Fix handling of HTML files without front matter

This means that any HTML file inside /content will be treated as a regular file.

If you want it processes with shortcodes and a layout, add front matter.

The defintion of an HTML file here is:

* File with extension .htm or .html
* With first non-whitespace character "<" that isn't a HTML comment.

This is in line with the documentation.

Fixes #7030
Fixes #7028
See #6789

8 files changed, 62 insertions, 243 deletions

diff --git a/hugolib/content_map_test.go b/hugolib/content_map_test.go
index 7a2829478..fb626b30c 100644
--- a/hugolib/content_map_test.go
+++ b/hugolib/content_map_test.go
@@ -54,7 +54,7 @@ func BenchmarkContentMap(b *testing.B) {
 				// real flow, so simulate this here.
 				meta["lang"] = lang
 				meta["path"] = meta.Filename()
-				meta["classifier"] = files.ClassifyContentFile(fi.Name())
+				meta["classifier"] = files.ClassifyContentFile(fi.Name(), meta.GetOpener())
 
 			})
 	}
@@ -115,7 +115,7 @@ func TestContentMap(t *testing.T) {
 				// real flow, so simulate this here.
 				meta["lang"] = lang
 				meta["path"] = meta.Filename()
-				meta["classifier"] = files.ClassifyContentFile(fi.Name())
+				meta["classifier"] = files.ClassifyContentFile(fi.Name(), meta.GetOpener())
 				meta["translationBaseName"] = helpers.Filename(fi.Name())
 
 			})
diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go
index fb9b6b83c..93a409f74 100644
--- a/hugolib/content_render_hooks_test.go
+++ b/hugolib/content_render_hooks_test.go
@@ -135,7 +135,7 @@ title: No Template
 	}
 	counters := &testCounters{}
 	b.Build(BuildCfg{testCounters: counters})
-	b.Assert(int(counters.contentRenderCounter), qt.Equals, 50)
+	b.Assert(int(counters.contentRenderCounter), qt.Equals, 43)
 
 	b.AssertFileContent("public/blog/p1/index.html", `
 <p>Cool Page|https://www.google.com|Title: Google's Homepage|Text: First Link|END</p>
diff --git a/hugolib/page.go b/hugolib/page.go
index 9db825256..f0f695227 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -449,12 +449,6 @@ func (p *pageState) initOutputFormat(isRenderingSite bool, idx int) error {
 		return err
 	}
 
-	if !p.renderable {
-		if _, err := p.Content(); err != nil {
-			return err
-		}
-	}
-
 	return nil
 
 }
@@ -679,8 +673,6 @@ func (p *pageState) mapContent(bucket *pagesMapBucket, meta *pageMeta) error {
 
 	s := p.shortcodeState
 
-	p.renderable = true
-
 	rn := &pageContentMap{
 		items: make([]interface{}, 0, 20),
 	}
@@ -703,12 +695,6 @@ Loop:
 
 		switch {
 		case it.Type == pageparser.TypeIgnore:
-		case it.Type == pageparser.TypeHTMLStart:
-			// This is HTML without front matter. It can still have shortcodes.
-			p.selfLayout = "__" + p.File().Filename()
-			p.renderable = false
-			p.s.BuildFlags.HasLateTemplate.CAS(false, true)
-			rn.AddBytes(it)
 		case it.IsFrontMatter():
 			f := pageparser.FormatFromFrontMatterType(it.Type)
 			m, err := metadecoders.Default.UnmarshalToMap(it.Val, f)
diff --git a/hugolib/page__content.go b/hugolib/page__content.go
index 013ab3072..91f26dc18 100644
--- a/hugolib/page__content.go
+++ b/hugolib/page__content.go
@@ -28,7 +28,6 @@ var (
 
 // The content related items on a Page.
 type pageContent struct {
-	renderable bool
 	selfLayout string
 	truncated  bool
 
@@ -52,7 +51,7 @@ func (p pageContent) contentToRender(renderedShortcodes map[string]string) []byt
 		case pageContentReplacement:
 			c = append(c, v.val...)
 		case *shortcode:
-			if !p.renderable || !v.insertPlaceholder() {
+			if !v.insertPlaceholder() {
 				// Insert the rendered shortcode.
 				renderedShortcode, found := renderedShortcodes[v.placeholder]
 				if !found {
diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go
index 330b0d75d..d7841f178 100644
--- a/hugolib/page__per_output.go
+++ b/hugolib/page__per_output.go
@@ -122,84 +122,76 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
 
 		isHTML := cp.p.m.markup == "html"
 
-		if p.renderable {
-			if !isHTML {
-				r, err := cp.renderContent(cp.workContent, true)
-				if err != nil {
-					return err
-				}
-
-				cp.workContent = r.Bytes()
-
-				if tocProvider, ok := r.(converter.TableOfContentsProvider); ok {
-					cfg := p.s.ContentSpec.Converters.GetMarkupConfig()
-					cp.tableOfContents = template.HTML(
-						tocProvider.TableOfContents().ToHTML(
-							cfg.TableOfContents.StartLevel,
-							cfg.TableOfContents.EndLevel,
-							cfg.TableOfContents.Ordered,
-						),
-					)
-				} else {
-					tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
-					cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-					cp.workContent = tmpContent
-				}
+		if !isHTML {
+			r, err := cp.renderContent(cp.workContent, true)
+			if err != nil {
+				return err
 			}
 
-			if cp.placeholdersEnabled {
-				// ToC was accessed via .Page.TableOfContents in the shortcode,
-				// at a time when the ToC wasn't ready.
-				cp.contentPlaceholders[tocShortcodePlaceholder] = string(cp.tableOfContents)
+			cp.workContent = r.Bytes()
+
+			if tocProvider, ok := r.(converter.TableOfContentsProvider); ok {
+				cfg := p.s.ContentSpec.Converters.GetMarkupConfig()
+				cp.tableOfContents = template.HTML(
+					tocProvider.TableOfContents().ToHTML(
+						cfg.TableOfContents.StartLevel,
+						cfg.TableOfContents.EndLevel,
+						cfg.TableOfContents.Ordered,
+					),
+				)
+			} else {
+				tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
+				cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
+				cp.workContent = tmpContent
 			}
+		}
 
-			if p.cmap.hasNonMarkdownShortcode || cp.placeholdersEnabled {
-				// There are one or more replacement tokens to be replaced.
-				cp.workContent, err = replaceShortcodeTokens(cp.workContent, cp.contentPlaceholders)
-				if err != nil {
-					return err
-				}
-			}
+		if cp.placeholdersEnabled {
+			// ToC was accessed via .Page.TableOfContents in the shortcode,
+			// at a time when the ToC wasn't ready.
+			cp.contentPlaceholders[tocShortcodePlaceholder] = string(cp.tableOfContents)
+		}
 
-			if cp.p.source.hasSummaryDivider {
-				if isHTML {
-					src := p.source.parsed.Input()
+		if p.cmap.hasNonMarkdownShortcode || cp.placeholdersEnabled {
+			// There are one or more replacement tokens to be replaced.
+			cp.workContent, err = replaceShortcodeTokens(cp.workContent, cp.contentPlaceholders)
+			if err != nil {
+				return err
+			}
+		}
 
-					// Use the summary sections as they are provided by the user.
-					if p.source.posSummaryEnd != -1 {
-						cp.summary = helpers.BytesToHTML(src[p.source.posMainContent:p.source.posSummaryEnd])
-					}
+		if cp.p.source.hasSummaryDivider {
+			if isHTML {
+				src := p.source.parsed.Input()
 
-					if cp.p.source.posBodyStart != -1 {
-						cp.workContent = src[cp.p.source.posBodyStart:]
-					}
+				// Use the summary sections as they are provided by the user.
+				if p.source.posSummaryEnd != -1 {
+					cp.summary = helpers.BytesToHTML(src[p.source.posMainContent:p.source.posSummaryEnd])
+				}
 
-				} else {
-					summary, content, err := splitUserDefinedSummaryAndContent(cp.p.m.markup, cp.workContent)
-					if err != nil {
-						cp.p.s.Log.ERROR.Printf("Failed to set user defined summary for page %q: %s", cp.p.pathOrTitle(), err)
-					} else {
-						cp.workContent = content
-						cp.summary = helpers.BytesToHTML(summary)
-					}
+				if cp.p.source.posBodyStart != -1 {
+					cp.workContent = src[cp.p.source.posBodyStart:]
 				}
-			} else if cp.p.m.summary != "" {
-				b, err := cp.renderContent([]byte(cp.p.m.summary), false)
+
+			} else {
+				summary, content, err := splitUserDefinedSummaryAndContent(cp.p.m.markup, cp.workContent)
 				if err != nil {
-					return err
+					cp.p.s.Log.ERROR.Printf("Failed to set user defined summary for page %q: %s", cp.p.pathOrTitle(), err)
+				} else {
+					cp.workContent = content
+					cp.summary = helpers.BytesToHTML(summary)
 				}
-				html := cp.p.s.ContentSpec.TrimShortHTML(b.Bytes())
-				cp.summary = helpers.BytesToHTML(html)
 			}
-
-			cp.content = helpers.BytesToHTML(cp.workContent)
-
+		} else if cp.p.m.summary != "" {
+			b, err := cp.renderContent([]byte(cp.p.m.summary), false)
+			if err != nil {
+				return err
+			}
+			html := cp.p.s.ContentSpec.TrimShortHTML(b.Bytes())
+			cp.summary = helpers.BytesToHTML(html)
 		}
 
-		if !p.renderable {
-			err := cp.addSelfTemplate()
-			return err
-		}
+		cp.content = helpers.BytesToHTML(cp.workContent)
 
 		return nil
 
@@ -207,8 +199,7 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
 
 	// Recursive loops can only happen in content files with template code (shortcodes etc.)
 	// Avoid creating new goroutines if we don't have to.
-	needTimeout := !p.renderable || p.shortcodeState.hasShortcodes()
-	needTimeout = needTimeout || cp.renderHooks != nil
+	needTimeout := p.shortcodeState.hasShortcodes() || cp.renderHooks != nil
 
 	if needTimeout {
 		cp.initMain = parent.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (interface{}, error) {
@@ -428,15 +419,6 @@ func (p *pageContentOutput) setWordCounts(isCJKLanguage bool) {
 	}
 }
 
-func (p *pageContentOutput) addSelfTemplate() error {
-	self := p.p.selfLayoutForOutput(p.f)
-	err := p.p.s.Tmpl().(tpl.TemplateManager).AddLateTemplate(self, string(p.workContent))
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 // A callback to signal that we have inserted a placeholder into the rendered
 // content. This avoids doing extra replacement work.
 func (p *pageContentOutput) enablePlaceholders() {
diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go
index b1041707b..0d3bfff18 100644
--- a/hugolib/shortcode.go
+++ b/hugolib/shortcode.go
@@ -408,7 +408,7 @@ func renderShortcode(
 }
 
 func (s *shortcodeHandler) hasShortcodes() bool {
-	return len(s.shortcodes) > 0
+	return s != nil && len(s.shortcodes) > 0
 }
 
 func (s *shortcodeHandler) renderShortcodesForPage(p *pageState, f output.Format) (map[string]string, bool, error) {
diff --git a/hugolib/site_test.go b/hugolib/site_test.go
index 66b54e352..0b05aac12 100644
--- a/hugolib/site_test.go
+++ b/hugolib/site_test.go
@@ -15,7 +15,6 @@ package hugolib
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -24,8 +23,6 @@ import (
 
 	"github.com/markbates/inflect"
 
-	"github.com/gohugoio/hugo/helpers"
-
 	qt "github.com/frankban/quicktest"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/resources/page"
@@ -502,70 +499,6 @@ func doTestSectionNaming(t *testing.T, canonify, uglify, pluralize bool) {
 
 }
 
-func TestSkipRender(t *testing.T) {
-	t.Parallel()
-	sources := [][2]string{
-		{filepath.FromSlash("sect/doc1.html"), "---\nmarkup: markdown\n---\n# title\nsome *content*"},
-		{filepath.FromSlash("sect/doc2.html"), "<!doctype html><html><body>more content</body></html>"},
-		{filepath.FromSlash("sect/doc3.md"), "# doc3\n*some* content"},
-		{filepath.FromSlash("sect/doc4.md"), "---\ntitle: doc4\n---\n# doc4\n*some content*"},
-		{filepath.FromSlash("sect/doc5.html"), "<!doctype html><html>{{ template \"head\" }}<body>body5</body></html>"},
-		{filepath.FromSlash("sect/doc6.html"), "<!doctype html><html>{{ template \"head_abs\" }}<body>body5</body></html>"},
-		{filepath.FromSlash("doc7.html"), "<html><body>doc7 content</body></html>"},
-		{filepath.FromSlash("sect/doc8.html"), "---\nmarkup: md\n---\n# title\nsome *content*"},
-		// Issue #3021
-		{filepath.FromSlash("doc9.html"), "<html><body>doc9: {{< myshortcode >}}</body></html>"},
-	}
-
-	cfg, fs := newTestCfg()
-
-	cfg.Set("verbose", true)
-	cfg.Set("canonifyURLs", true)
-	cfg.Set("uglyURLs", true)
-	cfg.Set("baseURL", "http://auth/bub")
-
-	for _, src := range sources {
-		writeSource(t, fs, filepath.Join("content", src[0]), src[1])
-
-	}
-
-	writeSource(t, fs, filepath.Join("layouts", "_default/single.html"), "{{.Content}}")
-	writeSource(t, fs, filepath.Join("layouts", "head"), "<head><script src=\"script.js\"></script></head>")
-	writeSource(t, fs, filepath.Join("layouts", "head_abs"), "<head><script src=\"/script.js\"></script></head>")
-	writeSource(t, fs, filepath.Join("layouts", "shortcodes", "myshortcode.html"), "SHORT")
-
-	buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{})
-
-	tests := []struct {
-		doc      string
-		expected string
-	}{
-		{filepath.FromSlash("public/sect/doc1.html"), "<h1 id=\"title\">title</h1>\n<p>some <em>content</em></p>\n"},
-		{filepath.FromSlash("public/sect/doc2.html"), "<!doctype html><html><body>more content</body></html>"},
-		{filepath.FromSlash("public/sect/doc3.html"), "<h1 id=\"doc3\">doc3</h1>\n<p><em>some</em> content</p>\n"},
-		{filepath.FromSlash("public/sect/doc4.html"), "<h1 id=\"doc4\">doc4</h1>\n<p><em>some content</em></p>\n"},
-		{filepath.FromSlash("public/sect/doc5.html"), "<!doctype html><html><head><script src=\"script.js\"></script></head><body>body5</body></html>"},
-		{filepath.FromSlash("public/sect/doc6.html"), "<!doctype html><html><head><script src=\"http://auth/bub/script.js\"></script></head><body>body5</body></html>"},
-		{filepath.FromSlash("public/doc7.html"), "<html><body>doc7 content</body></html>"},
-		{filepath.FromSlash("public/sect/doc8.html"), "<h1 id=\"title\">title</h1>\n<p>some <em>content</em></p>\n"},
-		{filepath.FromSlash("public/doc9.html"), "<html><body>doc9: SHORT</body></html>"},
-	}
-
-	for _, test := range tests {
-		file, err := fs.Destination.Open(test.doc)
-		if err != nil {
-			helpers.PrintFs(fs.Destination, "public", os.Stdout)
-			t.Fatalf("Did not find %s in target.", test.doc)
-		}
-
-		content := helpers.ReaderToString(file)
-
-		if content != test.expected {
-			t.Errorf("%s content expected:\n%q\ngot:\n%q", test.doc, test.expected, content)
-		}
-	}
-}
-
 func TestAbsURLify(t *testing.T) {
 	t.Parallel()
 	sources := [][2]string{
diff --git a/hugolib/template_test.go b/hugolib/template_test.go
index 99e57c75f..9f04aabdd 100644
--- a/hugolib/template_test.go
+++ b/hugolib/template_test.go
@@ -245,87 +245,6 @@ Page Content
 
 }
 
-func TestTemplateLateTemplates(t *testing.T) {
-	t.Parallel()
-	b := newTestSitesBuilder(t).WithSimpleConfigFile().Running()
-
-	numPages := 500 // To get some parallelism
-	homeTempl := `
-Len RegularPages: {{ len site.RegularPages }}
-{{ range site.RegularPages }}
-Link: {{ .RelPermalink }} Len Content: {{ len .Content }}
-{{ end }}
-`
-	pageTemplate := `<!doctype html>
-<html lang="en">
-<head>
-  <meta charset="utf-8">
-  <title>{{ .RelPermalink }}</title>
-  <meta name="description" content="The HTML5 Herald">
-  <meta name="author" content="SitePoint">
-  <link rel="stylesheet" href="css/styles.css?v=1.0">
-</head>
-<body>
-  <h1>{{ .RelPermalink }}</h1>
-  <p>Shortcode: {{< shortcode >}}</p>
-  <p>Partial: {{ partial "mypartial.html" . }}</p>
-  <script src="js/scripts.js"></script>
-</body>
-</html>
-`
-
-	b.WithTemplatesAdded(
-		"index.html", homeTempl,
-		"partials/mypartial.html", `this my partial`,
-	)
-
-	// Make sure we get some parallelism.
-	for i := 0; i < numPages; i++ {
-		b.WithContent(fmt.Sprintf("page%d.html", i+1), pageTemplate)
-	}
-
-	b.Build(BuildCfg{})
-
-	b.AssertFileContent("public/index.html", fmt.Sprintf(`
-Len RegularPages: %d
-Link: /page3/ Len Content: 0
-Link: /page22/ Len Content: 0
-`, numPages))
-
-	for i := 0; i < numPages; i++ {
-		b.AssertFileContent(fmt.Sprintf("public/page%d/index.html", i+1),
-			fmt.Sprintf(`<title>/page%d/</title>`, i+1),
-			`<p>Shortcode: Shortcode: Hello</p>`,
-			"<p>Partial: this my partial</p>",
-		)
-	}
-
-	b.EditFiles(
-		"layouts/partials/mypartial.html", `this my changed partial`,
-		"layouts/index.html", (homeTempl + "CHANGED"),
-	)
-	for i := 0; i < 5; i++ {
-		b.EditFiles(fmt.Sprintf("content/page%d.html", i+1), pageTemplate+"CHANGED")
-	}
-
-	b.Build(BuildCfg{})
-	b.AssertFileContent("public/index.html", fmt.Sprintf(`
-Len RegularPages: %d
-Link: /page3/ Len Content: 0
-Link: /page2/ Len Content: 0
-CHANGED
-`, numPages))
-	for i := 0; i < 5; i++ {
-		b.AssertFileContent(fmt.Sprintf("public/page%d/index.html", i+1),
-			fmt.Sprintf(`<title>/page%d/</title>`, i+1),
-			`<p>Shortcode: Shortcode: Hello</p>`,
-			"<p>Partial: this my changed partial</p>",
-			"CHANGED",
-		)
-	}
-
-}
-
 func TestTemplateManyBaseTemplates(t *testing.T) {
 	t.Parallel()
 	b := newTestSitesBuilder(t).WithSimpleConfigFile()