diff options
Diffstat (limited to 'common')
46 files changed, 1957 insertions, 193 deletions
diff --git a/common/constants/constants.go b/common/constants/constants.go index 6afb9e283..e4f5a63a2 100644 --- a/common/constants/constants.go +++ b/common/constants/constants.go @@ -20,3 +20,24 @@ const ( ErrRemoteGetJSON = "error-remote-getjson" ErrRemoteGetCSV = "error-remote-getcsv" ) + +// Field/method names with special meaning. +const ( + FieldRelPermalink = "RelPermalink" + FieldPermalink = "Permalink" +) + +// IsFieldRelOrPermalink returns whether the given name is a RelPermalink or Permalink. +func IsFieldRelOrPermalink(name string) bool { + return name == FieldRelPermalink || name == FieldPermalink +} + +// Resource transformations. +const ( + ResourceTransformationFingerprint = "fingerprint" +) + +// IsResourceTransformationLinkChange returns whether the given name is a resource transformation that changes the permalink based on the content. +func IsResourceTransformationPermalinkHash(name string) bool { + return name == ResourceTransformationFingerprint +} diff --git a/common/hcontext/context.go b/common/hcontext/context.go new file mode 100644 index 000000000..9524ef284 --- /dev/null +++ b/common/hcontext/context.go @@ -0,0 +1,46 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcontext + +import "context" + +// ContextDispatcher is a generic interface for setting and getting values from a context. +type ContextDispatcher[T any] interface { + Set(ctx context.Context, value T) context.Context + Get(ctx context.Context) T +} + +// NewContextDispatcher creates a new ContextDispatcher with the given key. +func NewContextDispatcher[T any, R comparable](key R) ContextDispatcher[T] { + return keyInContext[T, R]{ + id: key, + } +} + +type keyInContext[T any, R comparable] struct { + zero T + id R +} + +func (f keyInContext[T, R]) Get(ctx context.Context) T { + v := ctx.Value(f.id) + if v == nil { + return f.zero + } + return v.(T) +} + +func (f keyInContext[T, R]) Set(ctx context.Context, value T) context.Context { + return context.WithValue(ctx, f.id, value) +} diff --git a/common/herrors/error_locator.go b/common/herrors/error_locator.go index b880fe045..1ece0cca4 100644 --- a/common/herrors/error_locator.go +++ b/common/herrors/error_locator.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -74,7 +74,6 @@ func ContainsMatcher(text string) func(m LineMatcher) int { // ErrorContext contains contextual information about an error. This will // typically be the lines surrounding some problem in a file. type ErrorContext struct { - // If a match will contain the matched line and up to 2 lines before and after. // Will be empty if no match. Lines []string diff --git a/common/herrors/error_locator_test.go b/common/herrors/error_locator_test.go index 6135657d8..62f15213d 100644 --- a/common/herrors/error_locator_test.go +++ b/common/herrors/error_locator_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/herrors/errors.go b/common/herrors/errors.go index 8e62b2c99..59739a86a 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import ( "os" "runtime" "runtime/debug" + "time" ) // PrintStackTrace prints the current stacktrace to w. @@ -47,6 +48,24 @@ func Recover(args ...any) { } } +// IsTimeoutError returns true if the given error is or contains a TimeoutError. +func IsTimeoutError(err error) bool { + return errors.Is(err, &TimeoutError{}) +} + +type TimeoutError struct { + Duration time.Duration +} + +func (e *TimeoutError) Error() string { + return fmt.Sprintf("timeout after %s", e.Duration) +} + +func (e *TimeoutError) Is(target error) bool { + _, ok := target.(*TimeoutError) + return ok +} + // IsFeatureNotAvailableError returns true if the given error is or contains a FeatureNotAvailableError. func IsFeatureNotAvailableError(err error) bool { return errors.Is(err, &FeatureNotAvailableError{}) diff --git a/common/herrors/errors_test.go b/common/herrors/errors_test.go index 223782e23..2f53a1e89 100644 --- a/common/herrors/errors_test.go +++ b/common/herrors/errors_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -42,5 +42,4 @@ func TestIsFeatureNotAvailableError(t *testing.T) { c.Assert(IsFeatureNotAvailableError(ErrFeatureNotAvailable), qt.Equals, true) c.Assert(IsFeatureNotAvailableError(&FeatureNotAvailableError{}), qt.Equals, true) c.Assert(IsFeatureNotAvailableError(errors.New("asdf")), qt.Equals, false) - } diff --git a/common/herrors/file_error.go b/common/herrors/file_error.go index f8bcecd34..32a6f0081 100644 --- a/common/herrors/file_error.go +++ b/common/herrors/file_error.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,13 +15,13 @@ package herrors import ( "encoding/json" - - godartsassv1 "github.com/bep/godartsass" - + "errors" "fmt" "io" "path/filepath" + godartsassv1 "github.com/bep/godartsass" + "github.com/bep/godartsass/v2" "github.com/bep/golibsass/libsass/libsasserrors" "github.com/gohugoio/hugo/common/paths" @@ -29,8 +29,6 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/spf13/afero" "github.com/tdewolff/parse/v2" - - "errors" ) // FileError represents an error when handling a file: Parsing a config file, @@ -48,6 +46,9 @@ type FileError interface { // UpdateContent updates the error with a new ErrorContext from the content of the file. UpdateContent(r io.Reader, linematcher LineMatcherFn) FileError + + // SetFilename sets the filename of the error. + SetFilename(filename string) FileError } // Unwrapper can unwrap errors created with fmt.Errorf. @@ -60,6 +61,11 @@ var ( _ Unwrapper = (*fileError)(nil) ) +func (fe *fileError) SetFilename(filename string) FileError { + fe.position.Filename = filename + return fe +} + func (fe *fileError) UpdatePosition(pos text.Position) FileError { oldFilename := fe.Position().Filename if pos.Filename != "" && fe.fileType == "" { @@ -115,7 +121,6 @@ func (fe *fileError) UpdateContent(r io.Reader, linematcher LineMatcherFn) FileE } return fe - } type fileError struct { @@ -181,7 +186,6 @@ func NewFileErrorFromName(err error, name string) FileError { } return &fileError{cause: err, fileType: fileType, position: pos} - } // NewFileErrorFromPos will use the filename and line number from pos to create a new FileError, wrapping err. @@ -192,7 +196,6 @@ func NewFileErrorFromPos(err error, pos text.Position) FileError { _, fileType = paths.FileAndExtNoDelimiter(filepath.Clean(pos.Filename)) } return &fileError{cause: err, fileType: fileType, position: pos} - } func NewFileErrorFromFileInErr(err error, fs afero.Fs, linematcher LineMatcherFn) FileError { @@ -249,7 +252,6 @@ func openFile(filename string, fs afero.Fs) (afero.File, string, error) { }); ok { realFilename = s.Filename() } - } f, err2 := fs.Open(filename) diff --git a/common/herrors/file_error_test.go b/common/herrors/file_error_test.go index 0b260a255..7aca08405 100644 --- a/common/herrors/file_error_test.go +++ b/common/herrors/file_error_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,12 +14,11 @@ package herrors import ( + "errors" "fmt" "strings" "testing" - "errors" - "github.com/gohugoio/hugo/common/text" qt "github.com/frankban/quicktest" @@ -48,7 +47,6 @@ func TestNewFileError(t *testing.T) { c.Assert(errorContext.Lines, qt.DeepEquals, []string{"line 30", "line 31", "line 32", "line 33", "line 34"}) c.Assert(errorContext.LinesPos, qt.Equals, 2) c.Assert(errorContext.ChromaLexer, qt.Equals, "go-html-template") - } func TestNewFileErrorExtractFromMessage(t *testing.T) { diff --git a/common/hreflect/helpers.go b/common/hreflect/helpers.go index 17afbf912..b5a8bacc9 100644 --- a/common/hreflect/helpers.go +++ b/common/hreflect/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -23,6 +23,7 @@ import ( "time" "github.com/gohugoio/hugo/common/htime" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" ) @@ -188,6 +189,20 @@ func IsTime(tp reflect.Type) bool { return false } +// IsValid returns whether v is not nil and a valid value. +func IsValid(v reflect.Value) bool { + if !v.IsValid() { + return false + } + + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: + return !v.IsNil() + } + + return true +} + // AsTime returns v as a time.Time if possible. // The given location is only used if the value implements AsTimeProvider (e.g. go-toml local). // A zero Time and false is returned if this isn't possible. @@ -217,7 +232,7 @@ func CallMethodByName(cxt context.Context, name string, v reflect.Value) []refle panic("not supported") } first := tp.In(0) - if first.Implements(ContextInterface) { + if IsContextType(first) { args = append(args, reflect.ValueOf(cxt)) } } @@ -236,4 +251,24 @@ func indirectInterface(v reflect.Value) reflect.Value { return v.Elem() } -var ContextInterface = reflect.TypeOf((*context.Context)(nil)).Elem() +var contextInterface = reflect.TypeOf((*context.Context)(nil)).Elem() + +var isContextCache = maps.NewCache[reflect.Type, bool]() + +type k string + +var contextTypeValue = reflect.TypeOf(context.WithValue(context.Background(), k("key"), 32)) + +// IsContextType returns whether tp is a context.Context type. +func IsContextType(tp reflect.Type) bool { + if tp == contextTypeValue { + return true + } + if tp == contextInterface { + return true + } + + return isContextCache.GetOrCreate(tp, func() bool { + return tp.Implements(contextInterface) + }) +} diff --git a/common/hreflect/helpers_test.go b/common/hreflect/helpers_test.go index d16b9b9b3..27b774337 100644 --- a/common/hreflect/helpers_test.go +++ b/common/hreflect/helpers_test.go @@ -14,6 +14,7 @@ package hreflect import ( + "context" "reflect" "testing" "time" @@ -40,6 +41,42 @@ func TestGetMethodByName(t *testing.T) { c.Assert(GetMethodIndexByName(tp, "Foo"), qt.Equals, -1) } +func TestIsContextType(t *testing.T) { + c := qt.New(t) + type k string + ctx := context.Background() + valueCtx := context.WithValue(ctx, k("key"), 32) + c.Assert(IsContextType(reflect.TypeOf(ctx)), qt.IsTrue) + c.Assert(IsContextType(reflect.TypeOf(valueCtx)), qt.IsTrue) +} + +func BenchmarkIsContextType(b *testing.B) { + type k string + b.Run("value", func(b *testing.B) { + ctx := context.Background() + ctxs := make([]reflect.Type, b.N) + for i := 0; i < b.N; i++ { + ctxs[i] = reflect.TypeOf(context.WithValue(ctx, k("key"), i)) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !IsContextType(ctxs[i]) { + b.Fatal("not context") + } + } + }) + + b.Run("background", func(b *testing.B) { + var ctxt reflect.Type = reflect.TypeOf(context.Background()) + for i := 0; i < b.N; i++ { + if !IsContextType(ctxt) { + b.Fatal("not context") + } + } + }) +} + func BenchmarkIsTruthFul(b *testing.B) { v := reflect.ValueOf("Hugo") diff --git a/common/hstrings/strings.go b/common/hstrings/strings.go index 88df97607..d9426ab5d 100644 --- a/common/hstrings/strings.go +++ b/common/hstrings/strings.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -122,3 +122,8 @@ func InSlicEqualFold(arr []string, el string) bool { } return false } + +type Tuple struct { + First string + Second string +} diff --git a/common/hstrings/strings_test.go b/common/hstrings/strings_test.go index 85068bdf9..d8e9e204a 100644 --- a/common/hstrings/strings_test.go +++ b/common/hstrings/strings_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,7 +33,6 @@ func TestStringEqualFold(t *testing.T) { c.Assert(StringEqualFold(s1).EqualFold("b"), qt.Equals, false) c.Assert(StringEqualFold(s1).Eq(s2), qt.Equals, true) c.Assert(StringEqualFold(s1).Eq("b"), qt.Equals, false) - } func TestGetOrCompileRegexp(t *testing.T) { @@ -42,7 +41,6 @@ func TestGetOrCompileRegexp(t *testing.T) { re, err := GetOrCompileRegexp(`\d+`) c.Assert(err, qt.IsNil) c.Assert(re.MatchString("123"), qt.Equals, true) - } func BenchmarkGetOrCompileRegexp(b *testing.B) { diff --git a/common/htime/integration_test.go b/common/htime/integration_test.go index e72c216d9..983fff1f7 100644 --- a/common/htime/integration_test.go +++ b/common/htime/integration_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/hugio/copy.go b/common/hugio/copy.go index 8dbadc48c..31d679dfc 100644 --- a/common/hugio/copy.go +++ b/common/hugio/copy.go @@ -16,6 +16,7 @@ package hugio import ( "fmt" "io" + iofs "io/fs" "path/filepath" "github.com/spf13/afero" @@ -60,12 +61,16 @@ func CopyDir(fs afero.Fs, from, to string, shouldCopy func(filename string) bool return fmt.Errorf("%q is not a directory", from) } - err = fs.MkdirAll(to, 0777) // before umask + err = fs.MkdirAll(to, 0o777) // before umask if err != nil { return err } - entries, _ := afero.ReadDir(fs, from) + d, err := fs.Open(from) + if err != nil { + return err + } + entries, _ := d.(iofs.ReadDirFile).ReadDir(-1) for _, entry := range entries { fromFilename := filepath.Join(from, entry.Name()) toFilename := filepath.Join(to, entry.Name()) diff --git a/common/hugio/hasBytesWriter.go b/common/hugio/hasBytesWriter.go index 7b7d7a5d7..5148c82f9 100644 --- a/common/hugio/hasBytesWriter.go +++ b/common/hugio/hasBytesWriter.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/hugio/hasBytesWriter_test.go b/common/hugio/hasBytesWriter_test.go index b1b8011d5..af53fa5dd 100644 --- a/common/hugio/hasBytesWriter_test.go +++ b/common/hugio/hasBytesWriter_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/hugio/readers.go b/common/hugio/readers.go index 60bd97992..feb1b1412 100644 --- a/common/hugio/readers.go +++ b/common/hugio/readers.go @@ -14,6 +14,7 @@ package hugio import ( + "bytes" "io" "strings" ) @@ -57,3 +58,22 @@ func NewReadSeekerNoOpCloser(r ReadSeeker) ReadSeekerNoOpCloser { func NewReadSeekerNoOpCloserFromString(content string) ReadSeekerNoOpCloser { return ReadSeekerNoOpCloser{strings.NewReader(content)} } + +// NewReadSeekerNoOpCloserFromString uses strings.NewReader to create a new ReadSeekerNoOpCloser +// from the given bytes slice. +func NewReadSeekerNoOpCloserFromBytes(content []byte) ReadSeekerNoOpCloser { + return ReadSeekerNoOpCloser{bytes.NewReader(content)} +} + +// NewReadSeekCloser creates a new ReadSeekCloser from the given ReadSeeker. +// The ReadSeeker will be seeked to the beginning before returned. +func NewOpenReadSeekCloser(r ReadSeekCloser) OpenReadSeekCloser { + return func() (ReadSeekCloser, error) { + r.Seek(0, io.SeekStart) + return r, nil + } +} + +// OpenReadSeekCloser allows setting some other way (than reading from a filesystem) +// to open or create a ReadSeekCloser. +type OpenReadSeekCloser func() (ReadSeekCloser, error) diff --git a/common/hugo/hugo.go b/common/hugo/hugo.go index 67d52f6c8..be43e2a38 100644 --- a/common/hugo/hugo.go +++ b/common/hugo/hugo.go @@ -35,6 +35,8 @@ import ( "github.com/spf13/afero" + iofs "io/fs" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" ) @@ -159,7 +161,12 @@ func GetExecEnviron(workDir string, cfg config.AllProvider, fs afero.Fs) []strin config.SetEnvVars(&env, "HUGO_PUBLISHDIR", filepath.Join(workDir, cfg.BaseConfig().PublishDir)) if fs != nil { - fis, err := afero.ReadDir(fs, files.FolderJSConfig) + var fis []iofs.DirEntry + d, err := fs.Open(files.FolderJSConfig) + if err == nil { + fis, err = d.(iofs.ReadDirFile).ReadDir(-1) + } + if err == nil { for _, fi := range fis { key := fmt.Sprintf("HUGO_FILE_%s", strings.ReplaceAll(strings.ToUpper(fi.Name()), ".", "_")) diff --git a/common/loggers/handlerdefault.go b/common/loggers/handlerdefault.go index bb48895bc..bc3c7eec2 100644 --- a/common/loggers/handlerdefault.go +++ b/common/loggers/handlerdefault.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -27,10 +27,9 @@ import ( "github.com/fatih/color" ) -var bold = color.New(color.Bold) - // levelColor mapping. var levelColor = [...]*color.Color{ + logg.LevelTrace: color.New(color.FgWhite), logg.LevelDebug: color.New(color.FgWhite), logg.LevelInfo: color.New(color.FgBlue), logg.LevelWarn: color.New(color.FgYellow), @@ -39,6 +38,7 @@ var levelColor = [...]*color.Color{ // levelString mapping. var levelString = [...]string{ + logg.LevelTrace: "TRACE", logg.LevelDebug: "DEBUG", logg.LevelInfo: "INFO ", logg.LevelWarn: "WARN ", diff --git a/common/loggers/handlersmisc.go b/common/loggers/handlersmisc.go index 5c9d6c091..55bf8b940 100644 --- a/common/loggers/handlersmisc.go +++ b/common/loggers/handlersmisc.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -69,7 +69,7 @@ func (h *logLevelCounter) HandleLog(e *logg.Entry) error { return nil } -var stopError = fmt.Errorf("stop") +var errStop = fmt.Errorf("stop") type logOnceHandler struct { threshold logg.Level @@ -87,7 +87,7 @@ func (h *logOnceHandler) HandleLog(e *logg.Entry) error { defer h.mu.Unlock() hash := identity.HashUint64(e.Level, e.Message, e.Fields) if h.seen[hash] { - return stopError + return errStop } h.seen[hash] = true return nil @@ -107,7 +107,7 @@ type stopHandler struct { func (h *stopHandler) HandleLog(e *logg.Entry) error { for _, handler := range h.handlers { if err := handler.HandleLog(e); err != nil { - if err == stopError { + if err == errStop { return nil } return err @@ -124,26 +124,13 @@ func (h *suppressStatementsHandler) HandleLog(e *logg.Entry) error { for _, field := range e.Fields { if field.Name == FieldNameStatementID { if h.statements[field.Value.(string)] { - return stopError + return errStop } } } return nil } -// replacer creates a new log handler that does string replacement in log messages. -func replacer(repl *strings.Replacer) logg.Handler { - return logg.HandlerFunc(func(e *logg.Entry) error { - e.Message = repl.Replace(e.Message) - for i, field := range e.Fields { - if s, ok := field.Value.(string); ok { - e.Fields[i].Value = repl.Replace(s) - } - } - return nil - }) -} - // whiteSpaceTrimmer creates a new log handler that trims whitespace from log messages and string fields. func whiteSpaceTrimmer() logg.Handler { return logg.HandlerFunc(func(e *logg.Entry) error { diff --git a/common/loggers/handlerterminal.go b/common/loggers/handlerterminal.go index e3d377bbf..53f6e41da 100644 --- a/common/loggers/handlerterminal.go +++ b/common/loggers/handlerterminal.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -81,7 +81,7 @@ func (h *noColoursHandler) HandleLog(e *logg.Entry) error { if strings.HasPrefix(field.Name, reservedFieldNamePrefix) { continue } - fmt.Fprintf(w, " %s %q", field.Name, field.Value) + fmt.Fprintf(w, " %s %v", field.Name, field.Value) } fmt.Fprintln(w) diff --git a/common/loggers/logger.go b/common/loggers/logger.go index bc64ae0e5..c4d81fb83 100644 --- a/common/loggers/logger.go +++ b/common/loggers/logger.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // @@ -68,11 +68,24 @@ func New(opts Options) Logger { errorsw := &strings.Builder{} logCounters := newLogLevelCounter() handlers := []logg.Handler{ - whiteSpaceTrimmer(), - logHandler, logCounters, } + if opts.Level == logg.LevelTrace { + // Trace is used during development only, and it's useful to + // only see the trace messages. + handlers = append(handlers, + logg.HandlerFunc(func(e *logg.Entry) error { + if e.Level != logg.LevelTrace { + return logg.ErrStopLogEntry + } + return nil + }), + ) + } + + handlers = append(handlers, whiteSpaceTrimmer(), logHandler) + if opts.HandlerPost != nil { var hookHandler logg.HandlerFunc = func(e *logg.Entry) error { opts.HandlerPost(e) @@ -127,6 +140,7 @@ func New(opts Options) Logger { out: opts.Stdout, level: opts.Level, logger: logger, + tracel: l.WithLevel(logg.LevelTrace), debugl: l.WithLevel(logg.LevelDebug), infol: l.WithLevel(logg.LevelInfo), warnl: l.WithLevel(logg.LevelWarn), @@ -145,11 +159,22 @@ func NewDefault() Logger { return New(opts) } +func NewTrace() Logger { + opts := Options{ + DistinctLevel: logg.LevelWarn, + Level: logg.LevelTrace, + Stdout: os.Stdout, + Stderr: os.Stdout, + } + return New(opts) +} + func LevelLoggerToWriter(l logg.LevelLogger) io.Writer { return logWriter{l: l} } type Logger interface { + Debug() logg.LevelLogger Debugf(format string, v ...any) Debugln(v ...any) Error() logg.LevelLogger @@ -174,6 +199,7 @@ type Logger interface { Warnf(format string, v ...any) Warnln(v ...any) Deprecatef(fail bool, format string, v ...any) + Trace(s logg.StringFunc) } type logAdapter struct { @@ -183,12 +209,17 @@ type logAdapter struct { out io.Writer level logg.Level logger logg.Logger + tracel logg.LevelLogger debugl logg.LevelLogger infol logg.LevelLogger warnl logg.LevelLogger errorl logg.LevelLogger } +func (l *logAdapter) Debug() logg.LevelLogger { + return l.debugl +} + func (l *logAdapter) Debugf(format string, v ...any) { l.debugl.Logf(format, v...) } @@ -294,6 +325,10 @@ func (l *logAdapter) Errorsf(id, format string, v ...any) { l.errorl.WithField(FieldNameStatementID, id).Logf(format, v...) } +func (l *logAdapter) Trace(s logg.StringFunc) { + l.tracel.Log(s) +} + func (l *logAdapter) sprint(v ...any) string { return strings.TrimRight(fmt.Sprintln(v...), "\n") } @@ -315,3 +350,19 @@ func (w logWriter) Write(p []byte) (n int, err error) { w.l.Log(logg.String(string(p))) return len(p), nil } + +func TimeTrackf(l logg.LevelLogger, start time.Time, fields logg.Fields, format string, a ...any) { + elapsed := time.Since(start) + if fields != nil { + l = l.WithFields(fields) + } + l.WithField("duration", elapsed).Logf(format, a...) +} + +func TimeTrackfn(fn func() (logg.LevelLogger, error)) error { + start := time.Now() + l, err := fn() + elapsed := time.Since(start) + l.WithField("duration", elapsed).Logf("") + return err +} diff --git a/common/loggers/logger_test.go b/common/loggers/logger_test.go index 6f589aafe..dcf94b123 100644 --- a/common/loggers/logger_test.go +++ b/common/loggers/logger_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // diff --git a/common/loggers/loggerglobal.go b/common/loggers/loggerglobal.go index 6fd474a69..c3e2970d0 100644 --- a/common/loggers/loggerglobal.go +++ b/common/loggers/loggerglobal.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // Some functions in this file (see comments) is based on the Go source code, // copyright The Go Authors and governed by a BSD-style license. // diff --git a/common/maps/cache.go b/common/maps/cache.go new file mode 100644 index 000000000..7e23a2662 --- /dev/null +++ b/common/maps/cache.go @@ -0,0 +1,90 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import "sync" + +// Cache is a simple thread safe cache backed by a map. +type Cache[K comparable, T any] struct { + m map[K]T + sync.RWMutex +} + +// NewCache creates a new Cache. +func NewCache[K comparable, T any]() *Cache[K, T] { + return &Cache[K, T]{m: make(map[K]T)} +} + +// Delete deletes the given key from the cache. +func (c *Cache[K, T]) Get(key K) (T, bool) { + c.RLock() + v, found := c.m[key] + c.RUnlock() + return v, found +} + +// GetOrCreate gets the value for the given key if it exists, or creates it if not. +func (c *Cache[K, T]) GetOrCreate(key K, create func() T) T { + c.RLock() + v, found := c.m[key] + c.RUnlock() + if found { + return v + } + c.Lock() + defer c.Unlock() + v, found = c.m[key] + if found { + return v + } + v = create() + c.m[key] = v + return v +} + +// Set sets the given key to the given value. +func (c *Cache[K, T]) Set(key K, value T) { + c.Lock() + c.m[key] = value + c.Unlock() +} + +// SliceCache is a simple thread safe cache backed by a map. +type SliceCache[T any] struct { + m map[string][]T + sync.RWMutex +} + +func NewSliceCache[T any]() *SliceCache[T] { + return &SliceCache[T]{m: make(map[string][]T)} +} + +func (c *SliceCache[T]) Get(key string) ([]T, bool) { + c.RLock() + v, found := c.m[key] + c.RUnlock() + return v, found +} + +func (c *SliceCache[T]) Append(key string, values ...T) { + c.Lock() + c.m[key] = append(c.m[key], values...) + c.Unlock() +} + +func (c *SliceCache[T]) Reset() { + c.Lock() + c.m = make(map[string][]T) + c.Unlock() +} diff --git a/common/maps/maps.go b/common/maps/maps.go index f0fd3d5ce..2686baad6 100644 --- a/common/maps/maps.go +++ b/common/maps/maps.go @@ -29,7 +29,7 @@ func ToStringMapE(in any) (map[string]any, error) { case Params: return vv, nil case map[string]string: - var m = map[string]any{} + m := map[string]any{} for k, v := range vv { m[k] = v } @@ -192,21 +192,20 @@ func (KeyRenamer) keyPath(k1, k2 string) string { } func (r KeyRenamer) renamePath(parentKeyPath string, m map[string]any) { - for key, val := range m { - keyPath := r.keyPath(parentKeyPath, key) - switch val.(type) { + for k, v := range m { + keyPath := r.keyPath(parentKeyPath, k) + switch vv := v.(type) { case map[any]any: - val = cast.ToStringMap(val) - r.renamePath(keyPath, val.(map[string]any)) + r.renamePath(keyPath, cast.ToStringMap(vv)) case map[string]any: - r.renamePath(keyPath, val.(map[string]any)) + r.renamePath(keyPath, vv) } newKey := r.getNewKey(keyPath) if newKey != "" { - delete(m, key) - m[newKey] = val + delete(m, k) + m[newKey] = v } } } diff --git a/common/maps/params.go b/common/maps/params.go index d94d16f9d..a8cbba555 100644 --- a/common/maps/params.go +++ b/common/maps/params.go @@ -61,7 +61,7 @@ func SetParams(dst, src Params) { // IsZero returns true if p is considered empty. func (p Params) IsZero() bool { - if p == nil || len(p) == 0 { + if len(p) == 0 { return true } @@ -74,7 +74,6 @@ func (p Params) IsZero() bool { } return false - } // MergeParamsWithStrategy transfers values from src to dst for new keys using the merge strategy given. @@ -93,7 +92,7 @@ func MergeParams(dst, src Params) { func (p Params) merge(ps ParamsMergeStrategy, pp Params) { ns, found := p.GetMergeStrategy() - var ms = ns + ms := ns if !found && ps != "" { ms = ps } @@ -248,7 +247,7 @@ const ( // CleanConfigStringMapString removes any processing instructions from m, // m will never be modified. func CleanConfigStringMapString(m map[string]string) map[string]string { - if m == nil || len(m) == 0 { + if len(m) == 0 { return m } if _, found := m[MergeStrategyKey]; !found { @@ -267,7 +266,7 @@ func CleanConfigStringMapString(m map[string]string) map[string]string { // CleanConfigStringMap is the same as CleanConfigStringMapString but for // map[string]any. func CleanConfigStringMap(m map[string]any) map[string]any { - if m == nil || len(m) == 0 { + if len(m) == 0 { return m } if _, found := m[MergeStrategyKey]; !found { @@ -291,7 +290,6 @@ func CleanConfigStringMap(m map[string]any) map[string]any { } return m2 - } func toMergeStrategy(v any) ParamsMergeStrategy { diff --git a/common/paths/path.go b/common/paths/path.go index 5d211c5e0..da99b16ac 100644 --- a/common/paths/path.go +++ b/common/paths/path.go @@ -16,14 +16,18 @@ package paths import ( "errors" "fmt" + "net/url" "path" "path/filepath" - "regexp" "strings" + "unicode" ) // FilePathSeparator as defined by os.Separator. -const FilePathSeparator = string(filepath.Separator) +const ( + FilePathSeparator = string(filepath.Separator) + slash = "/" +) // filepathPathBridge is a bridge for common functionality in filepath vs path type filepathPathBridge interface { @@ -72,6 +76,30 @@ func AbsPathify(workingDir, inPath string) string { return filepath.Join(workingDir, inPath) } +// AddTrailingSlash adds a trailing Unix styled slash (/) if not already +// there. +func AddTrailingSlash(path string) string { + if !strings.HasSuffix(path, "/") { + path += "/" + } + return path +} + +// AddLeadingSlash adds a leading Unix styled slash (/) if not already +// there. +func AddLeadingSlash(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return path +} + +// AddTrailingAndLeadingSlash adds a leading and trailing Unix styled slash (/) if not already +// there. +func AddLeadingAndTrailingSlash(path string) string { + return AddTrailingSlash(AddLeadingSlash(path)) +} + // MakeTitle converts the path given to a suitable title, trimming whitespace // and replacing hyphens with whitespace. func MakeTitle(inpath string) string { @@ -94,43 +122,6 @@ func makePathRelative(inPath string, possibleDirectories ...string) (string, err return inPath, errors.New("can't extract relative path, unknown prefix") } -// Should be good enough for Hugo. -var isFileRe = regexp.MustCompile(`.*\..{1,6}$`) - -// GetDottedRelativePath expects a relative path starting after the content directory. -// It returns a relative path with dots ("..") navigating up the path structure. -func GetDottedRelativePath(inPath string) string { - inPath = path.Clean(filepath.ToSlash(inPath)) - - if inPath == "." { - return "./" - } - - if !isFileRe.MatchString(inPath) && !strings.HasSuffix(inPath, "/") { - inPath += "/" - } - - if !strings.HasPrefix(inPath, "/") { - inPath = "/" + inPath - } - - dir, _ := filepath.Split(inPath) - - sectionCount := strings.Count(dir, "/") - - if sectionCount == 0 || dir == "/" { - return "./" - } - - var dottedPath string - - for i := 1; i < sectionCount; i++ { - dottedPath += "../" - } - - return dottedPath -} - // ExtNoDelimiter takes a path and returns the extension, excluding the delimiter, i.e. "md". func ExtNoDelimiter(in string) string { return strings.TrimPrefix(Ext(in), ".") @@ -167,12 +158,6 @@ func Filename(in string) (name string) { return } -// PathNoExt takes a path, strips out the extension, -// and returns the name of the file. -func PathNoExt(in string) string { - return strings.TrimSuffix(in, path.Ext(in)) -} - // FileAndExt returns the filename and any extension of a file path as // two separate strings. // @@ -252,16 +237,125 @@ func prettifyPath(in string, b filepathPathBridge) string { return b.Join(b.Dir(in), name, "index"+ext) } -type NamedSlice struct { - Name string - Slice []string +// CommonDir returns the common directory of the given paths. +func CommonDir(path1, path2 string) string { + if path1 == "" || path2 == "" { + return "" + } + + p1 := strings.Split(path1, "/") + p2 := strings.Split(path2, "/") + + var common []string + + for i := 0; i < len(p1) && i < len(p2); i++ { + if p1[i] == p2[i] { + common = append(common, p1[i]) + } else { + break + } + } + + return strings.Join(common, "/") +} + +// Sanitize sanitizes string to be used in Hugo's file paths and URLs, allowing only +// a predefined set of special Unicode characters. +// +// Spaces will be replaced with a single hyphen. +// +// This function is the core function used to normalize paths in Hugo. +// +// Note that this is the first common step for URL/path sanitation, +// the final URL/path may end up looking differently if the user has stricter rules defined (e.g. removePathAccents=true). +func Sanitize(s string) string { + var willChange bool + for i, r := range s { + willChange = !isAllowedPathCharacter(s, i, r) + if willChange { + break + } + } + + if !willChange { + // Prevent allocation when nothing changes. + return s + } + + target := make([]rune, 0, len(s)) + var ( + prependHyphen bool + wasHyphen bool + ) + + for i, r := range s { + isAllowed := isAllowedPathCharacter(s, i, r) + + if isAllowed { + // track explicit hyphen in input; no need to add a new hyphen if + // we just saw one. + wasHyphen = r == '-' + + if prependHyphen { + // if currently have a hyphen, don't prepend an extra one + if !wasHyphen { + target = append(target, '-') + } + prependHyphen = false + } + target = append(target, r) + } else if len(target) > 0 && !wasHyphen && unicode.IsSpace(r) { + prependHyphen = true + } + } + + return string(target) +} + +func isAllowedPathCharacter(s string, i int, r rune) bool { + if r == ' ' { + return false + } + // Check for the most likely first (faster). + isAllowed := unicode.IsLetter(r) || unicode.IsDigit(r) + isAllowed = isAllowed || r == '.' || r == '/' || r == '\\' || r == '_' || r == '#' || r == '+' || r == '~' || r == '-' || r == '@' + isAllowed = isAllowed || unicode.IsMark(r) + isAllowed = isAllowed || (r == '%' && i+2 < len(s) && ishex(s[i+1]) && ishex(s[i+2])) + return isAllowed } -func (n NamedSlice) String() string { - if len(n.Slice) == 0 { - return n.Name +// From https://golang.org/src/net/url/url.go +func ishex(c byte) bool { + switch { + case '0' <= c && c <= '9': + return true + case 'a' <= c && c <= 'f': + return true + case 'A' <= c && c <= 'F': + return true } - return fmt.Sprintf("%s%s{%s}", n.Name, FilePathSeparator, strings.Join(n.Slice, ",")) + return false +} + +var slashFunc = func(r rune) bool { + return r == '/' +} + +// Dir behaves like path.Dir without the path.Clean step. +// +// The returned path ends in a slash only if it is the root "/". +func Dir(s string) string { + dir, _ := path.Split(s) + if len(dir) > 1 && dir[len(dir)-1] == '/' { + return dir[:len(dir)-1] + } + return dir +} + +// FieldsSlash cuts s into fields separated with '/'. +func FieldsSlash(s string) []string { + f := strings.FieldsFunc(s, slashFunc) + return f } // DirFile holds the result from path.Split. @@ -274,3 +368,27 @@ type DirFile struct { func (df DirFile) String() string { return fmt.Sprintf("%s|%s", df.Dir, df.File) } + +// PathEscape escapes unicode letters in pth. +// Use URLEscape to escape full URLs including scheme, query etc. +// This is slightly faster for the common case. +// Note, there is a url.PathEscape function, but that also +// escapes /. +func PathEscape(pth string) string { + u, err := url.Parse(pth) + if err != nil { + panic(err) + } + return u.EscapedPath() +} + +// ToSlashTrimLeading is just a filepath.ToSlash with an added / prefix trimmer. +func ToSlashTrimLeading(s string) string { + return strings.TrimPrefix(filepath.ToSlash(s), "/") +} + +// ToSlashPreserveLeading converts the path given to a forward slash separated path +// and preserves the leading slash if present trimming any trailing slash. +func ToSlashPreserveLeading(s string) string { + return "/" + strings.Trim(filepath.ToSlash(s), "/") +} diff --git a/common/paths/path_test.go b/common/paths/path_test.go index 2400f16ab..3605bfc43 100644 --- a/common/paths/path_test.go +++ b/common/paths/path_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -75,44 +75,6 @@ func TestMakePathRelative(t *testing.T) { } } -func TestGetDottedRelativePath(t *testing.T) { - // on Windows this will receive both kinds, both country and western ... - for _, f := range []func(string) string{filepath.FromSlash, func(s string) string { return s }} { - doTestGetDottedRelativePath(f, t) - } -} - -func doTestGetDottedRelativePath(urlFixer func(string) string, t *testing.T) { - type test struct { - input, expected string - } - data := []test{ - {"", "./"}, - {urlFixer("/"), "./"}, - {urlFixer("post"), "../"}, - {urlFixer("/post"), "../"}, - {urlFixer("post/"), "../"}, - {urlFixer("tags/foo.html"), "../"}, - {urlFixer("/tags/foo.html"), "../"}, - {urlFixer("/post/"), "../"}, - {urlFixer("////post/////"), "../"}, - {urlFixer("/foo/bar/index.html"), "../../"}, - {urlFixer("/foo/bar/foo/"), "../../../"}, - {urlFixer("/foo/bar/foo"), "../../../"}, - {urlFixer("foo/bar/foo/"), "../../../"}, - {urlFixer("foo/bar/foo/bar"), "../../../../"}, - {"404.html", "./"}, - {"404.xml", "./"}, - {"/404.html", "./"}, - } - for i, d := range data { - output := GetDottedRelativePath(d.input) - if d.expected != output { - t.Errorf("Test %d failed. Expected %q got %q", i, d.expected, output) - } - } -} - func TestMakeTitle(t *testing.T) { type test struct { input, expected string @@ -226,3 +188,77 @@ func TestFileAndExt(t *testing.T) { } } } + +func TestSanitize(t *testing.T) { + c := qt.New(t) + tests := []struct { + input string + expected string + }{ + {" Foo bar ", "Foo-bar"}, + {"Foo.Bar/foo_Bar-Foo", "Foo.Bar/foo_Bar-Foo"}, + {"fOO,bar:foobAR", "fOObarfoobAR"}, + {"FOo/BaR.html", "FOo/BaR.html"}, + {"FOo/Ba---R.html", "FOo/Ba---R.html"}, /// See #10104 + {"FOo/Ba R.html", "FOo/Ba-R.html"}, + {"трям/трям", "трям/трям"}, + {"은행", "은행"}, + {"Банковский кассир", "Банковский-кассир"}, + // Issue #1488 + {"संस्कृत", "संस्कृत"}, + {"a%C3%B1ame", "a%C3%B1ame"}, // Issue #1292 + {"this+is+a+test", "this+is+a+test"}, // Issue #1290 + {"~foo", "~foo"}, // Issue #2177 + + } + + for _, test := range tests { + c.Assert(Sanitize(test.input), qt.Equals, test.expected) + } +} + +func BenchmarkSanitize(b *testing.B) { + const ( + allAlowedPath = "foo/bar" + spacePath = "foo bar" + ) + + // This should not allocate any memory. + b.Run("All allowed", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := Sanitize(allAlowedPath) + if got != allAlowedPath { + b.Fatal(got) + } + } + }) + + // This will allocate some memory. + b.Run("Spaces", func(b *testing.B) { + for i := 0; i < b.N; i++ { + got := Sanitize(spacePath) + if got != "foo-bar" { + b.Fatal(got) + } + } + }) +} + +func TestDir(t *testing.T) { + c := qt.New(t) + c.Assert(Dir("/a/b/c/d"), qt.Equals, "/a/b/c") + c.Assert(Dir("/a"), qt.Equals, "/") + c.Assert(Dir("/"), qt.Equals, "/") + c.Assert(Dir(""), qt.Equals, "") +} + +func TestFieldsSlash(t *testing.T) { + c := qt.New(t) + + c.Assert(FieldsSlash("a/b/c"), qt.DeepEquals, []string{"a", "b", "c"}) + c.Assert(FieldsSlash("/a/b/c"), qt.DeepEquals, []string{"a", "b", "c"}) + c.Assert(FieldsSlash("/a/b/c/"), qt.DeepEquals, []string{"a", "b", "c"}) + c.Assert(FieldsSlash("a/b/c/"), qt.DeepEquals, []string{"a", "b", "c"}) + c.Assert(FieldsSlash("/"), qt.DeepEquals, []string{}) + c.Assert(FieldsSlash(""), qt.DeepEquals, []string{}) +} diff --git a/common/paths/pathparser.go b/common/paths/pathparser.go new file mode 100644 index 000000000..842d9307b --- /dev/null +++ b/common/paths/pathparser.go @@ -0,0 +1,494 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "path" + "path/filepath" + "runtime" + "strings" + + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/hugofs/files" +) + +var defaultPathParser PathParser + +// PathParser parses a path into a Path. +type PathParser struct { + // Maps the language code to its index in the languages/sites slice. + LanguageIndex map[string]int +} + +// Parse parses component c with path s into Path using the default path parser. +func Parse(c, s string) *Path { + return defaultPathParser.Parse(c, s) +} + +// NormalizePathString returns a normalized path string using the very basic Hugo rules. +func NormalizePathStringBasic(s string) string { + // All lower case. + s = strings.ToLower(s) + + // Replace spaces with hyphens. + s = strings.ReplaceAll(s, " ", "-") + + return s +} + +// Parse parses component c with path s into Path using Hugo's content path rules. +func (parser PathParser) Parse(c, s string) *Path { + p, err := parser.parse(c, s) + if err != nil { + panic(err) + } + return p +} + +func (pp *PathParser) parse(component, s string) (*Path, error) { + ss := NormalizePathStringBasic(s) + + p, err := pp.doParse(component, ss) + if err != nil { + return nil, err + } + + if s != ss { + var err error + // Preserve the original case for titles etc. + p.unnormalized, err = pp.doParse(component, s) + + if err != nil { + return nil, err + } + } else { + p.unnormalized = p + } + + return p, nil +} + +func (pp *PathParser) doParse(component, s string) (*Path, error) { + p := &Path{ + component: component, + posContainerLow: -1, + posContainerHigh: -1, + posSectionHigh: -1, + posIdentifierLanguage: -1, + } + + hasLang := pp.LanguageIndex != nil + hasLang = hasLang && (component == files.ComponentFolderContent || component == files.ComponentFolderLayouts) + + if runtime.GOOS == "windows" { + s = path.Clean(filepath.ToSlash(s)) + if s == "." { + s = "" + } + } + + if s == "" { + s = "/" + } + + // Leading slash, no trailing slash. + if !strings.HasPrefix(s, "/") { + s = "/" + s + } + + if s != "/" && s[len(s)-1] == '/' { + s = s[:len(s)-1] + } + + p.s = s + slashCount := 0 + + for i := len(s) - 1; i >= 0; i-- { + c := s[i] + + switch c { + case '.': + if p.posContainerHigh == -1 { + var high int + if len(p.identifiers) > 0 { + high = p.identifiers[len(p.identifiers)-1].Low - 1 + } else { + high = len(p.s) + } + id := types.LowHigh{Low: i + 1, High: high} + if len(p.identifiers) == 0 { + p.identifiers = append(p.identifiers, id) + } else if len(p.identifiers) == 1 { + // Check for a valid language. + s := p.s[id.Low:id.High] + + if hasLang { + if _, found := pp.LanguageIndex[s]; found { + p.posIdentifierLanguage = 1 + p.identifiers = append(p.identifiers, id) + } + } + } + } + case '/': + slashCount++ + if p.posContainerHigh == -1 { + p.posContainerHigh = i + 1 + } else if p.posContainerLow == -1 { + p.posContainerLow = i + 1 + } + if i > 0 { + p.posSectionHigh = i + } + } + } + + isContentComponent := p.component == files.ComponentFolderContent || p.component == files.ComponentFolderArchetypes + isContent := isContentComponent && files.IsContentExt(p.Ext()) + + if isContent { + id := p.identifiers[len(p.identifiers)-1] + b := p.s[p.posContainerHigh : id.Low-1] + switch b { + case "index": + p.bundleType = PathTypeLeaf + case "_index": + p.bundleType = PathTypeBranch + default: + p.bundleType = PathTypeContentSingle + } + + if slashCount == 2 && p.IsLeafBundle() { + p.posSectionHigh = 0 + } + } + + return p, nil +} + +func ModifyPathBundleTypeResource(p *Path) { + if p.IsContent() { + p.bundleType = PathTypeContentResource + } else { + p.bundleType = PathTypeFile + } +} + +type PathType int + +const ( + // A generic resource, e.g. a JSON file. + PathTypeFile PathType = iota + + // All below are content files. + // A resource of a content type with front matter. + PathTypeContentResource + + // E.g. /blog/my-post.md + PathTypeContentSingle + + // All bewlow are bundled content files. + + // Leaf bundles, e.g. /blog/my-post/index.md + PathTypeLeaf + + // Branch bundles, e.g. /blog/_index.md + PathTypeBranch +) + +type Path struct { + s string + + posContainerLow int + posContainerHigh int + posSectionHigh int + + component string + bundleType PathType + + identifiers []types.LowHigh + + posIdentifierLanguage int + + trimLeadingSlash bool + + unnormalized *Path +} + +// TrimLeadingSlash returns a copy of the Path with the leading slash removed. +func (p Path) TrimLeadingSlash() *Path { + p.trimLeadingSlash = true + return &p +} + +func (p *Path) norm(s string) string { + if p.trimLeadingSlash { + s = strings.TrimPrefix(s, "/") + } + return s +} + +// IdentifierBase satifies identity.Identity. +func (p *Path) IdentifierBase() string { + return p.Base()[1:] +} + +// Component returns the component for this path (e.g. "content"). +func (p *Path) Component() string { + return p.component +} + +// Container returns the base name of the container directory for this path. +func (p *Path) Container() string { + if p.posContainerLow == -1 { + return "" + } + return p.norm(p.s[p.posContainerLow : p.posContainerHigh-1]) +} + +// ContainerDir returns the container directory for this path. +// For content bundles this will be the parent directory. +func (p *Path) ContainerDir() string { + if p.posContainerLow == -1 || !p.IsBundle() { + return p.Dir() + } + return p.norm(p.s[:p.posContainerLow-1]) +} + +// Section returns the first path element (section). +func (p *Path) Section() string { + if p.posSectionHigh <= 0 { + return "" + } + return p.norm(p.s[1:p.posSectionHigh]) +} + +// IsContent returns true if the path is a content file (e.g. mypost.md). +// Note that this will also return true for content files in a bundle. +func (p *Path) IsContent() bool { + return p.BundleType() >= PathTypeContentResource +} + +// isContentPage returns true if the path is a content file (e.g. mypost.md), +// but nof if inside a leaf bundle. +func (p *Path) isContentPage() bool { + return p.BundleType() >= PathTypeContentSingle +} + +// Name returns the last element of path. +func (p *Path) Name() string { + if p.posContainerHigh > 0 { + return p.s[p.posContainerHigh:] + } + return p.s +} + +// Name returns the last element of path withhout any extension. +func (p *Path) NameNoExt() string { + if i := p.identifierIndex(0); i != -1 { + return p.s[p.posContainerHigh : p.identifiers[i].Low-1] + } + return p.s[p.posContainerHigh:] +} + +// Name returns the last element of path withhout any language identifier. +func (p *Path) NameNoLang() string { + i := p.identifierIndex(p.posIdentifierLanguage) + if i == -1 { + return p.Name() + } + + return p.s[p.posContainerHigh:p.identifiers[i].Low-1] + p.s[p.identifiers[i].High:] +} + +// BaseNameNoIdentifier returns the logcical base name for a resource without any idenifier (e.g. no extension). +// For bundles this will be the containing directory's name, e.g. "blog". +func (p *Path) BaseNameNoIdentifier() string { + if p.IsBundle() { + return p.Container() + } + return p.NameNoIdentifier() +} + +// NameNoIdentifier returns the last element of path withhout any identifier (e.g. no extension). +func (p *Path) NameNoIdentifier() string { + if len(p.identifiers) > 0 { + return p.s[p.posContainerHigh : p.identifiers[len(p.identifiers)-1].Low-1] + } + return p.s[p.posContainerHigh:] +} + +// Dir returns all but the last element of path, typically the path's directory. +func (p *Path) Dir() (d string) { + if p.posContainerHigh > 0 { + d = p.s[:p.posContainerHigh-1] + } + if d == "" { + d = "/" + } + d = p.norm(d) + return +} + +// Path returns the full path. +func (p *Path) Path() (d string) { + return p.norm(p.s) +} + +// Unmormalized returns the Path with the original case preserved. +func (p *Path) Unmormalized() *Path { + return p.unnormalized +} + +// PathNoLang returns the Path but with any language identifier removed. +func (p *Path) PathNoLang() string { + return p.base(true, false) +} + +// PathNoIdentifier returns the Path but with any identifier (ext, lang) removed. +func (p *Path) PathNoIdentifier() string { + return p.base(false, false) +} + +// PathRel returns the path relativeto the given owner. +func (p *Path) PathRel(owner *Path) string { + ob := owner.Base() + if !strings.HasSuffix(ob, "/") { + ob += "/" + } + return strings.TrimPrefix(p.Path(), ob) +} + +// BaseRel returns the base path relative to the given owner. +func (p *Path) BaseRel(owner *Path) string { + ob := owner.Base() + if ob == "/" { + ob = "" + } + return p.Base()[len(ob)+1:] +} + +// For content files, Base returns the path without any identifiers (extension, language code etc.). +// Any 'index' as the last path element is ignored. +// +// For other files (Resources), any extension is kept. +func (p *Path) Base() string { + return p.base(!p.isContentPage(), p.IsBundle()) +} + +// BaseNoLeadingSlash returns the base path without the leading slash. +func (p *Path) BaseNoLeadingSlash() string { + return p.Base()[1:] +} + +func (p *Path) base(preserveExt, isBundle bool) string { + if len(p.identifiers) == 0 { + return p.norm(p.s) + } + + if preserveExt && len(p.identifiers) == 1 { + // Preserve extension. + return p.norm(p.s) + } + + id := p.identifiers[len(p.identifiers)-1] + high := id.Low - 1 + + if isBundle { + high = p.posContainerHigh - 1 + } + + if high == 0 { + high++ + } + + if !preserveExt { + return p.norm(p.s[:high]) + } + + // For txt files etc. we want to preserve the extension. + id = p.identifiers[0] + + return p.norm(p.s[:high] + p.s[id.Low-1:id.High]) +} + +func (p *Path) Ext() string { + return p.identifierAsString(0) +} + +func (p *Path) Lang() string { + return p.identifierAsString(1) +} + +func (p *Path) Identifier(i int) string { + return p.identifierAsString(i) +} + +func (p *Path) Identifiers() []string { + ids := make([]string, len(p.identifiers)) + for i, id := range p.identifiers { + ids[i] = p.s[id.Low:id.High] + } + return ids +} + +func (p *Path) IsHTML() bool { + return files.IsHTML(p.Ext()) +} + +func (p *Path) BundleType() PathType { + return p.bundleType +} + +func (p *Path) IsBundle() bool { + return p.bundleType >= PathTypeLeaf +} + +func (p *Path) IsBranchBundle() bool { + return p.bundleType == PathTypeBranch +} + +func (p *Path) IsLeafBundle() bool { + return p.bundleType == PathTypeLeaf +} + +func (p *Path) identifierAsString(i int) string { + i = p.identifierIndex(i) + if i == -1 { + return "" + } + + id := p.identifiers[i] + return p.s[id.Low:id.High] +} + +func (p *Path) identifierIndex(i int) int { + if i < 0 || i >= len(p.identifiers) { + return -1 + } + return i +} + +// HasExt returns true if the Unix styled path has an extension. +func HasExt(p string) bool { + for i := len(p) - 1; i >= 0; i-- { + if p[i] == '.' { + return true + } + if p[i] == '/' { + return false + } + } + return false +} diff --git a/common/paths/pathparser_test.go b/common/paths/pathparser_test.go new file mode 100644 index 000000000..3546b6605 --- /dev/null +++ b/common/paths/pathparser_test.go @@ -0,0 +1,351 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/hugofs/files" + + qt "github.com/frankban/quicktest" +) + +var testParser = &PathParser{ + LanguageIndex: map[string]int{ + "no": 0, + "en": 1, + }, +} + +func TestParse(t *testing.T) { + c := qt.New(t) + + tests := []struct { + name string + path string + assert func(c *qt.C, p *Path) + }{ + { + "Basic text file", + "/a/b.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Name(), qt.Equals, "b.txt") + c.Assert(p.Base(), qt.Equals, "/a/b.txt") + c.Assert(p.Container(), qt.Equals, "a") + c.Assert(p.Dir(), qt.Equals, "/a") + c.Assert(p.Ext(), qt.Equals, "txt") + c.Assert(p.IsContent(), qt.IsFalse) + }, + }, + { + "Basic text file, upper case", + "/A/B.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Name(), qt.Equals, "b.txt") + c.Assert(p.NameNoExt(), qt.Equals, "b") + c.Assert(p.NameNoIdentifier(), qt.Equals, "b") + c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b") + c.Assert(p.Base(), qt.Equals, "/a/b.txt") + c.Assert(p.Ext(), qt.Equals, "txt") + }, + }, + { + "Basic text file, 1 space in dir", + "/a b/c.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a-b/c.txt") + }, + }, + { + "Basic text file, 2 spaces in dir", + "/a b/c.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a--b/c.txt") + }, + }, + { + "Basic text file, 1 space in filename", + "/a/b c.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a/b-c.txt") + }, + }, + { + "Basic text file, 2 spaces in filename", + "/a/b c.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a/b--c.txt") + }, + }, + { + "Basic text file, mixed case and spaces, unnormalized", + "/a/Foo BAR.txt", + func(c *qt.C, p *Path) { + pp := p.Unmormalized() + c.Assert(pp, qt.IsNotNil) + c.Assert(pp.BaseNameNoIdentifier(), qt.Equals, "Foo BAR") + }, + }, + { + "Basic Markdown file", + "/a/b/c.md", + func(c *qt.C, p *Path) { + c.Assert(p.IsContent(), qt.IsTrue) + c.Assert(p.IsLeafBundle(), qt.IsFalse) + c.Assert(p.Name(), qt.Equals, "c.md") + c.Assert(p.Base(), qt.Equals, "/a/b/c") + c.Assert(p.Section(), qt.Equals, "a") + c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "c") + c.Assert(p.Path(), qt.Equals, "/a/b/c.md") + c.Assert(p.Dir(), qt.Equals, "/a/b") + c.Assert(p.Container(), qt.Equals, "b") + c.Assert(p.ContainerDir(), qt.Equals, "/a/b") + c.Assert(p.Ext(), qt.Equals, "md") + }, + }, + { + "Content resource", + "/a/b.md", + func(c *qt.C, p *Path) { + c.Assert(p.Name(), qt.Equals, "b.md") + c.Assert(p.Base(), qt.Equals, "/a/b") + c.Assert(p.BaseNoLeadingSlash(), qt.Equals, "a/b") + c.Assert(p.Section(), qt.Equals, "a") + c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b") + + // Reclassify it as a content resource. + ModifyPathBundleTypeResource(p) + c.Assert(p.BundleType(), qt.Equals, PathTypeContentResource) + c.Assert(p.IsContent(), qt.IsTrue) + c.Assert(p.Name(), qt.Equals, "b.md") + c.Assert(p.Base(), qt.Equals, "/a/b.md") + }, + }, + { + "No ext", + "/a/b", + func(c *qt.C, p *Path) { + c.Assert(p.Name(), qt.Equals, "b") + c.Assert(p.NameNoExt(), qt.Equals, "b") + c.Assert(p.Base(), qt.Equals, "/a/b") + c.Assert(p.Ext(), qt.Equals, "") + }, + }, + { + "No ext, trailing slash", + "/a/b/", + func(c *qt.C, p *Path) { + c.Assert(p.Name(), qt.Equals, "b") + c.Assert(p.Base(), qt.Equals, "/a/b") + c.Assert(p.Ext(), qt.Equals, "") + }, + }, + { + "Identifiers", + "/a/b.a.b.no.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Name(), qt.Equals, "b.a.b.no.txt") + c.Assert(p.NameNoIdentifier(), qt.Equals, "b.a.b") + c.Assert(p.NameNoLang(), qt.Equals, "b.a.b.txt") + c.Assert(p.Identifiers(), qt.DeepEquals, []string{"txt", "no"}) + c.Assert(p.Base(), qt.Equals, "/a/b.a.b.txt") + c.Assert(p.BaseNoLeadingSlash(), qt.Equals, "a/b.a.b.txt") + c.Assert(p.PathNoLang(), qt.Equals, "/a/b.a.b.txt") + c.Assert(p.Ext(), qt.Equals, "txt") + c.Assert(p.PathNoIdentifier(), qt.Equals, "/a/b.a.b") + }, + }, + { + "Home branch cundle", + "/_index.md", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/") + c.Assert(p.Path(), qt.Equals, "/_index.md") + c.Assert(p.Container(), qt.Equals, "") + c.Assert(p.ContainerDir(), qt.Equals, "/") + }, + }, + { + "Index content file in root", + "/a/index.md", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a") + c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "a") + c.Assert(p.Container(), qt.Equals, "a") + c.Assert(p.Container(), qt.Equals, "a") + c.Assert(p.ContainerDir(), qt.Equals, "") + c.Assert(p.Dir(), qt.Equals, "/a") + c.Assert(p.Ext(), qt.Equals, "md") + c.Assert(p.Identifiers(), qt.DeepEquals, []string{"md"}) + c.Assert(p.IsBranchBundle(), qt.IsFalse) + c.Assert(p.IsBundle(), qt.IsTrue) + c.Assert(p.IsLeafBundle(), qt.IsTrue) + c.Assert(p.Lang(), qt.Equals, "") + c.Assert(p.NameNoExt(), qt.Equals, "index") + c.Assert(p.NameNoIdentifier(), qt.Equals, "index") + c.Assert(p.NameNoLang(), qt.Equals, "index.md") + c.Assert(p.Section(), qt.Equals, "") + }, + }, + { + "Index content file with lang", + "/a/b/index.no.md", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a/b") + c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b") + c.Assert(p.Container(), qt.Equals, "b") + c.Assert(p.ContainerDir(), qt.Equals, "/a") + c.Assert(p.Dir(), qt.Equals, "/a/b") + c.Assert(p.Ext(), qt.Equals, "md") + c.Assert(p.Identifiers(), qt.DeepEquals, []string{"md", "no"}) + c.Assert(p.IsBranchBundle(), qt.IsFalse) + c.Assert(p.IsBundle(), qt.IsTrue) + c.Assert(p.IsLeafBundle(), qt.IsTrue) + c.Assert(p.Lang(), qt.Equals, "no") + c.Assert(p.NameNoExt(), qt.Equals, "index.no") + c.Assert(p.NameNoIdentifier(), qt.Equals, "index") + c.Assert(p.NameNoLang(), qt.Equals, "index.md") + c.Assert(p.PathNoLang(), qt.Equals, "/a/b/index.md") + c.Assert(p.Section(), qt.Equals, "a") + }, + }, + { + "Index branch content file", + "/a/b/_index.no.md", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a/b") + c.Assert(p.BaseNameNoIdentifier(), qt.Equals, "b") + c.Assert(p.Container(), qt.Equals, "b") + c.Assert(p.ContainerDir(), qt.Equals, "/a") + c.Assert(p.Ext(), qt.Equals, "md") + c.Assert(p.Identifiers(), qt.DeepEquals, []string{"md", "no"}) + c.Assert(p.IsBranchBundle(), qt.IsTrue) + c.Assert(p.IsBundle(), qt.IsTrue) + c.Assert(p.IsLeafBundle(), qt.IsFalse) + c.Assert(p.NameNoExt(), qt.Equals, "_index.no") + c.Assert(p.NameNoLang(), qt.Equals, "_index.md") + }, + }, + { + "Index root no slash", + "_index.md", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/") + c.Assert(p.Ext(), qt.Equals, "md") + c.Assert(p.Name(), qt.Equals, "_index.md") + }, + }, + { + "Index root", + "/_index.md", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/") + c.Assert(p.Ext(), qt.Equals, "md") + c.Assert(p.Name(), qt.Equals, "_index.md") + }, + }, + { + "Index first", + "/a/_index.md", + func(c *qt.C, p *Path) { + c.Assert(p.Section(), qt.Equals, "a") + }, + }, + { + "Index text file", + "/a/b/index.no.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a/b/index.txt") + c.Assert(p.Ext(), qt.Equals, "txt") + c.Assert(p.Identifiers(), qt.DeepEquals, []string{"txt", "no"}) + c.Assert(p.IsLeafBundle(), qt.IsFalse) + c.Assert(p.PathNoIdentifier(), qt.Equals, "/a/b/index") + }, + }, + { + "Empty", + "", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/") + c.Assert(p.Ext(), qt.Equals, "") + c.Assert(p.Name(), qt.Equals, "") + c.Assert(p.Path(), qt.Equals, "/") + }, + }, + { + "Slash", + "/", + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/") + c.Assert(p.Ext(), qt.Equals, "") + c.Assert(p.Name(), qt.Equals, "") + }, + }, + { + "Trim Leading Slash bundle", + "foo/bar/index.no.md", + func(c *qt.C, p *Path) { + c.Assert(p.Path(), qt.Equals, "/foo/bar/index.no.md") + pp := p.TrimLeadingSlash() + c.Assert(pp.Path(), qt.Equals, "foo/bar/index.no.md") + c.Assert(pp.PathNoLang(), qt.Equals, "foo/bar/index.md") + c.Assert(pp.Base(), qt.Equals, "foo/bar") + c.Assert(pp.Dir(), qt.Equals, "foo/bar") + c.Assert(pp.ContainerDir(), qt.Equals, "foo") + c.Assert(pp.Container(), qt.Equals, "bar") + c.Assert(pp.BaseNameNoIdentifier(), qt.Equals, "bar") + }, + }, + { + "Trim Leading Slash file", + "foo/bar.txt", + func(c *qt.C, p *Path) { + c.Assert(p.Path(), qt.Equals, "/foo/bar.txt") + pp := p.TrimLeadingSlash() + c.Assert(pp.Path(), qt.Equals, "foo/bar.txt") + c.Assert(pp.PathNoLang(), qt.Equals, "foo/bar.txt") + c.Assert(pp.Base(), qt.Equals, "foo/bar.txt") + c.Assert(pp.Dir(), qt.Equals, "foo") + c.Assert(pp.ContainerDir(), qt.Equals, "foo") + c.Assert(pp.Container(), qt.Equals, "foo") + c.Assert(pp.BaseNameNoIdentifier(), qt.Equals, "bar") + }, + }, + { + "File separator", + filepath.FromSlash("/a/b/c.txt"), + func(c *qt.C, p *Path) { + c.Assert(p.Base(), qt.Equals, "/a/b/c.txt") + c.Assert(p.Ext(), qt.Equals, "txt") + c.Assert(p.Name(), qt.Equals, "c.txt") + c.Assert(p.Path(), qt.Equals, "/a/b/c.txt") + }, + }, + } + for _, test := range tests { + c.Run(test.name, func(c *qt.C) { + test.assert(c, testParser.Parse(files.ComponentFolderContent, test.path)) + }) + } +} + +func TestHasExt(t *testing.T) { + c := qt.New(t) + + c.Assert(HasExt("/a/b/c.txt"), qt.IsTrue) + c.Assert(HasExt("/a/b.c/d.txt"), qt.IsTrue) + c.Assert(HasExt("/a/b/c"), qt.IsFalse) + c.Assert(HasExt("/a/b.c/d"), qt.IsFalse) +} diff --git a/common/paths/paths_integration_test.go b/common/paths/paths_integration_test.go new file mode 100644 index 000000000..62d40f527 --- /dev/null +++ b/common/paths/paths_integration_test.go @@ -0,0 +1,80 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths_test + +import ( + "testing" + + "github.com/gohugoio/hugo/hugolib" +) + +func TestRemovePathAccents(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[languages] +[languages.en] +weight = 1 +[languages.fr] +weight = 2 +removePathAccents = true +-- content/διακριτικός.md -- +-- content/διακριτικός.fr.md -- +-- layouts/_default/single.html -- +{{ .Language.Lang }}|Single. +-- layouts/_default/list.html -- +List +` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/en/διακριτικός/index.html", "en|Single") + b.AssertFileContent("public/fr/διακριτικος/index.html", "fr|Single") +} + +func TestDisablePathToLower(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term"] +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true +[languages] +[languages.en] +weight = 1 +[languages.fr] +weight = 2 +disablePathToLower = true +-- content/MySection/MyPage.md -- +-- content/MySection/MyPage.fr.md -- +-- content/MySection/MyBundle/index.md -- +-- content/MySection/MyBundle/index.fr.md -- +-- layouts/_default/single.html -- +{{ .Language.Lang }}|Single. +-- layouts/_default/list.html -- +{{ .Language.Lang }}|List. +` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/en/mysection/index.html", "en|List") + b.AssertFileContent("public/en/mysection/mypage/index.html", "en|Single") + b.AssertFileContent("public/fr/MySection/index.html", "fr|List") + b.AssertFileContent("public/fr/MySection/MyPage/index.html", "fr|Single") + b.AssertFileContent("public/en/mysection/mybundle/index.html", "en|Single") + b.AssertFileContent("public/fr/MySection/MyBundle/index.html", "fr|Single") +} diff --git a/common/paths/pathtype_string.go b/common/paths/pathtype_string.go new file mode 100644 index 000000000..7a99f8a03 --- /dev/null +++ b/common/paths/pathtype_string.go @@ -0,0 +1,27 @@ +// Code generated by "stringer -type=PathType"; DO NOT EDIT. + +package paths + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[PathTypeFile-0] + _ = x[PathTypeContentResource-1] + _ = x[PathTypeContentSingle-2] + _ = x[PathTypeLeaf-3] + _ = x[PathTypeBranch-4] +} + +const _PathType_name = "PathTypeFilePathTypeContentResourcePathTypeContentSinglePathTypeLeafPathTypeBranch" + +var _PathType_index = [...]uint8{0, 12, 35, 56, 68, 82} + +func (i PathType) String() string { + if i < 0 || i >= PathType(len(_PathType_index)-1) { + return "PathType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _PathType_name[_PathType_index[i]:_PathType_index[i+1]] +} diff --git a/common/paths/url.go b/common/paths/url.go index 093ba9ff7..4c4a7f2dc 100644 --- a/common/paths/url.go +++ b/common/paths/url.go @@ -184,3 +184,13 @@ func UrlToFilename(s string) (string, bool) { return p, true } + +// URLEscape escapes unicode letters. +func URLEscape(uri string) string { + // escape unicode letters + u, err := url.Parse(uri) + if err != nil { + panic(err) + } + return u.String() +} diff --git a/common/predicate/predicate.go b/common/predicate/predicate.go new file mode 100644 index 000000000..f9cb1bb2b --- /dev/null +++ b/common/predicate/predicate.go @@ -0,0 +1,72 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicate + +// P is a predicate function that tests whether a value of type T satisfies some condition. +type P[T any] func(T) bool + +// And returns a predicate that is a short-circuiting logical AND of this and the given predicates. +func (p P[T]) And(ps ...P[T]) P[T] { + return func(v T) bool { + for _, pp := range ps { + if !pp(v) { + return false + } + } + return p(v) + } +} + +// Or returns a predicate that is a short-circuiting logical OR of this and the given predicates. +func (p P[T]) Or(ps ...P[T]) P[T] { + return func(v T) bool { + for _, pp := range ps { + if pp(v) { + return true + } + } + return p(v) + } +} + +// Negate returns a predicate that is a logical negation of this predicate. +func (p P[T]) Negate() P[T] { + return func(v T) bool { + return !p(v) + } +} + +// Filter returns a new slice holding only the elements of s that satisfy p. +// Filter modifies the contents of the slice s and returns the modified slice, which may have a smaller length. +func (p P[T]) Filter(s []T) []T { + var n int + for _, v := range s { + if p(v) { + s[n] = v + n++ + } + } + return s[:n] +} + +// FilterCopy returns a new slice holding only the elements of s that satisfy p. +func (p P[T]) FilterCopy(s []T) []T { + var result []T + for _, v := range s { + if p(v) { + result = append(result, v) + } + } + return result +} diff --git a/common/predicate/predicate_test.go b/common/predicate/predicate_test.go new file mode 100644 index 000000000..1e1ec004b --- /dev/null +++ b/common/predicate/predicate_test.go @@ -0,0 +1,83 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicate_test + +import ( + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/predicate" +) + +func TestAdd(t *testing.T) { + c := qt.New(t) + + var p predicate.P[int] = intP1 + + c.Assert(p(1), qt.IsTrue) + c.Assert(p(2), qt.IsFalse) + + neg := p.Negate() + c.Assert(neg(1), qt.IsFalse) + c.Assert(neg(2), qt.IsTrue) + + and := p.And(intP2) + c.Assert(and(1), qt.IsFalse) + c.Assert(and(2), qt.IsFalse) + c.Assert(and(10), qt.IsTrue) + + or := p.Or(intP2) + c.Assert(or(1), qt.IsTrue) + c.Assert(or(2), qt.IsTrue) + c.Assert(or(10), qt.IsTrue) + c.Assert(or(11), qt.IsFalse) +} + +func TestFilter(t *testing.T) { + c := qt.New(t) + + var p predicate.P[int] = intP1 + p = p.Or(intP2) + + ints := []int{1, 2, 3, 4, 1, 6, 7, 8, 2} + + c.Assert(p.Filter(ints), qt.DeepEquals, []int{1, 2, 1, 2}) + c.Assert(ints, qt.DeepEquals, []int{1, 2, 1, 2, 1, 6, 7, 8, 2}) +} + +func TestFilterCopy(t *testing.T) { + c := qt.New(t) + + var p predicate.P[int] = intP1 + p = p.Or(intP2) + + ints := []int{1, 2, 3, 4, 1, 6, 7, 8, 2} + + c.Assert(p.FilterCopy(ints), qt.DeepEquals, []int{1, 2, 1, 2}) + c.Assert(ints, qt.DeepEquals, []int{1, 2, 3, 4, 1, 6, 7, 8, 2}) +} + +var intP1 = func(i int) bool { + if i == 10 { + return true + } + return i == 1 +} + +var intP2 = func(i int) bool { + if i == 10 { + return true + } + return i == 2 +} diff --git a/common/rungroup/rungroup.go b/common/rungroup/rungroup.go new file mode 100644 index 000000000..96ec57883 --- /dev/null +++ b/common/rungroup/rungroup.go @@ -0,0 +1,93 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rungroup + +import ( + "context" + + "golang.org/x/sync/errgroup" +) + +// Group is a group of workers that can be used to enqueue work and wait for +// them to finish. +type Group[T any] interface { + Enqueue(T) error + Wait() error +} + +type runGroup[T any] struct { + ctx context.Context + g *errgroup.Group + ch chan T +} + +// Config is the configuration for a new Group. +type Config[T any] struct { + NumWorkers int + Handle func(context.Context, T) error +} + +// Run creates a new Group with the given configuration. +func Run[T any](ctx context.Context, cfg Config[T]) Group[T] { + if cfg.NumWorkers <= 0 { + cfg.NumWorkers = 1 + } + if cfg.Handle == nil { + panic("Handle must be set") + } + + g, ctx := errgroup.WithContext(ctx) + // Buffered for performance. + ch := make(chan T, cfg.NumWorkers) + + for i := 0; i < cfg.NumWorkers; i++ { + g.Go(func() error { + for { + select { + case <-ctx.Done(): + return nil + case v, ok := <-ch: + if !ok { + return nil + } + if err := cfg.Handle(ctx, v); err != nil { + return err + } + } + } + }) + } + + return &runGroup[T]{ + ctx: ctx, + g: g, + ch: ch, + } +} + +// Enqueue enqueues a new item to be handled by the workers. +func (r *runGroup[T]) Enqueue(t T) error { + select { + case <-r.ctx.Done(): + return nil + case r.ch <- t: + } + return nil +} + +// Wait waits for all workers to finish and returns the first error. +func (r *runGroup[T]) Wait() error { + close(r.ch) + return r.g.Wait() +} diff --git a/common/rungroup/rungroup_test.go b/common/rungroup/rungroup_test.go new file mode 100644 index 000000000..ac902079e --- /dev/null +++ b/common/rungroup/rungroup_test.go @@ -0,0 +1,44 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rungroup + +import ( + "context" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestNew(t *testing.T) { + c := qt.New(t) + + var result int + adder := func(ctx context.Context, i int) error { + result += i + return nil + } + + g := Run[int]( + context.Background(), + Config[int]{ + Handle: adder, + }, + ) + + c.Assert(g, qt.IsNotNil) + g.Enqueue(32) + g.Enqueue(33) + c.Assert(g.Wait(), qt.IsNil) + c.Assert(result, qt.Equals, 65) +} diff --git a/common/terminal/colors.go b/common/terminal/colors.go index c4a78291e..8aa0e1af2 100644 --- a/common/terminal/colors.go +++ b/common/terminal/colors.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/types/css/csstypes.go b/common/types/css/csstypes.go index a31df00e7..061acfe64 100644 --- a/common/types/css/csstypes.go +++ b/common/types/css/csstypes.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/types/evictingqueue.go b/common/types/evictingqueue.go index 884762426..88add59d5 100644 --- a/common/types/evictingqueue.go +++ b/common/types/evictingqueue.go @@ -35,11 +35,11 @@ func NewEvictingStringQueue(size int) *EvictingStringQueue { } // Add adds a new string to the tail of the queue if it's not already there. -func (q *EvictingStringQueue) Add(v string) { +func (q *EvictingStringQueue) Add(v string) *EvictingStringQueue { q.mu.Lock() if q.set[v] { q.mu.Unlock() - return + return q } if len(q.set) == q.size { @@ -50,6 +50,17 @@ func (q *EvictingStringQueue) Add(v string) { q.set[v] = true q.vals = append(q.vals, v) q.mu.Unlock() + + return q +} + +func (q *EvictingStringQueue) Len() int { + if q == nil { + return 0 + } + q.mu.Lock() + defer q.mu.Unlock() + return len(q.vals) } // Contains returns whether the queue contains v. diff --git a/common/types/hstring/stringtypes.go b/common/types/hstring/stringtypes.go index 601218e0e..5e8e3a23d 100644 --- a/common/types/hstring/stringtypes.go +++ b/common/types/hstring/stringtypes.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/types/hstring/stringtypes_test.go b/common/types/hstring/stringtypes_test.go index 8fa1c9760..2f1f865c8 100644 --- a/common/types/hstring/stringtypes_test.go +++ b/common/types/hstring/stringtypes_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/common/types/types.go b/common/types/types.go index c36c51b3e..11683c196 100644 --- a/common/types/types.go +++ b/common/types/types.go @@ -92,5 +92,18 @@ type DevMarker interface { DevOnly() } +// Unwrapper is implemented by types that can unwrap themselves. +type Unwrapper interface { + // Unwrapv is for internal use only. + // It got its slightly odd name to prevent collisions with user types. + Unwrapv() any +} + +// LowHigh is typically used to represent a slice boundary. +type LowHigh struct { + Low int + High int +} + // This is only used for debugging purposes. var InvocationCounter atomic.Int64 diff --git a/common/urls/baseURL.go b/common/urls/baseURL.go index df26730ec..2958a2a04 100644 --- a/common/urls/baseURL.go +++ b/common/urls/baseURL.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,10 +23,12 @@ import ( // A BaseURL in Hugo is normally on the form scheme://path, but the // form scheme: is also valid (mailto:hugo@rules.com). type BaseURL struct { - url *url.URL - WithPath string - WithoutPath string - BasePath string + url *url.URL + WithPath string + WithPathNoTrailingSlash string + WithoutPath string + BasePath string + BasePathNoTrailingSlash string } func (b BaseURL) String() string { @@ -92,19 +94,19 @@ func NewBaseURLFromString(b string) (BaseURL, error) { return BaseURL{}, err } return newBaseURLFromURL(u) - } func newBaseURLFromURL(u *url.URL) (BaseURL, error) { - baseURL := BaseURL{url: u, WithPath: u.String()} - var baseURLNoPath = baseURL.URL() + // A baseURL should always have a trailing slash, see #11669. + if !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + baseURL := BaseURL{url: u, WithPath: u.String(), WithPathNoTrailingSlash: strings.TrimSuffix(u.String(), "/")} + baseURLNoPath := baseURL.URL() baseURLNoPath.Path = "" baseURL.WithoutPath = baseURLNoPath.String() - - basePath := u.Path - if basePath != "" && basePath != "/" { - baseURL.BasePath = basePath - } + baseURL.BasePath = u.Path + baseURL.BasePathNoTrailingSlash = strings.TrimSuffix(u.Path, "/") return baseURL, nil } diff --git a/common/urls/baseURL_test.go b/common/urls/baseURL_test.go index 95dc73339..ba337aac8 100644 --- a/common/urls/baseURL_test.go +++ b/common/urls/baseURL_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,17 +21,24 @@ import ( func TestBaseURL(t *testing.T) { c := qt.New(t) - b, err := NewBaseURLFromString("http://example.com") + + b, err := NewBaseURLFromString("http://example.com/") + c.Assert(err, qt.IsNil) + c.Assert(b.String(), qt.Equals, "http://example.com/") + + b, err = NewBaseURLFromString("http://example.com") c.Assert(err, qt.IsNil) - c.Assert(b.String(), qt.Equals, "http://example.com") + c.Assert(b.String(), qt.Equals, "http://example.com/") + c.Assert(b.WithPathNoTrailingSlash, qt.Equals, "http://example.com") + c.Assert(b.BasePath, qt.Equals, "/") p, err := b.WithProtocol("webcal://") c.Assert(err, qt.IsNil) - c.Assert(p.String(), qt.Equals, "webcal://example.com") + c.Assert(p.String(), qt.Equals, "webcal://example.com/") p, err = b.WithProtocol("webcal") c.Assert(err, qt.IsNil) - c.Assert(p.String(), qt.Equals, "webcal://example.com") + c.Assert(p.String(), qt.Equals, "webcal://example.com/") _, err = b.WithProtocol("mailto:") c.Assert(err, qt.Not(qt.IsNil)) @@ -57,11 +64,18 @@ func TestBaseURL(t *testing.T) { b, err = NewBaseURLFromString("") c.Assert(err, qt.IsNil) - c.Assert(b.String(), qt.Equals, "") + c.Assert(b.String(), qt.Equals, "/") // BaseURL with sub path b, err = NewBaseURLFromString("http://example.com/sub") c.Assert(err, qt.IsNil) - c.Assert(b.String(), qt.Equals, "http://example.com/sub") + c.Assert(b.String(), qt.Equals, "http://example.com/sub/") + c.Assert(b.WithPathNoTrailingSlash, qt.Equals, "http://example.com/sub") + c.Assert(b.BasePath, qt.Equals, "/sub/") + c.Assert(b.BasePathNoTrailingSlash, qt.Equals, "/sub") + + b, err = NewBaseURLFromString("http://example.com/sub/") + c.Assert(err, qt.IsNil) + c.Assert(b.String(), qt.Equals, "http://example.com/sub/") c.Assert(b.HostURL(), qt.Equals, "http://example.com") } |