5 files changed, 76 insertions, 4 deletions
diff --git a/parser/pageparser/item.go b/parser/pageparser/item.go
index f23341ea8..52546be41 100644
--- a/parser/pageparser/item.go
+++ b/parser/pageparser/item.go
@@ -18,6 +18,8 @@ import (
 	"fmt"
 	"regexp"
 	"strconv"
+
+	"github.com/yuin/goldmark/util"
 )
 
 type Item struct {
@@ -64,7 +66,11 @@ func (i Item) ValTyped() any {
 }
 
 func (i Item) IsText() bool {
-	return i.Type == tText
+	return i.Type == tText || i.Type == tIndentation
+}
+
+func (i Item) IsIndentation() bool {
+	return i.Type == tIndentation
 }
 
 func (i Item) IsNonWhitespace() bool {
@@ -125,6 +131,8 @@ func (i Item) String() string {
 		return "EOF"
 	case i.Type == tError:
 		return string(i.Val)
+	case i.Type == tIndentation:
+		return fmt.Sprintf("%s:[%s]", i.Type, util.VisualizeSpaces(i.Val))
 	case i.Type > tKeywordMarker:
 		return fmt.Sprintf("<%s>", i.Val)
 	case len(i.Val) > 50:
@@ -159,6 +167,8 @@ const (
 	tScParam
 	tScParamVal
 
+	tIndentation
+
 	tText // plain text
 
 	// preserved for later - keywords come after this
diff --git a/parser/pageparser/itemtype_string.go b/parser/pageparser/itemtype_string.go
index 632afaecc..b0b849ade 100644
--- a/parser/pageparser/itemtype_string.go
+++ b/parser/pageparser/itemtype_string.go
@@ -4,9 +4,36 @@ package pageparser
 
 import "strconv"
 
-const _ItemType_name = "tErrortEOFTypeHTMLStartTypeLeadSummaryDividerTypeFrontMatterYAMLTypeFrontMatterTOMLTypeFrontMatterJSONTypeFrontMatterORGTypeEmojiTypeIgnoretLeftDelimScNoMarkuptRightDelimScNoMarkuptLeftDelimScWithMarkuptRightDelimScWithMarkuptScClosetScNametScNameInlinetScParamtScParamValtTexttKeywordMarker"
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[tError-0]
+	_ = x[tEOF-1]
+	_ = x[TypeLeadSummaryDivider-2]
+	_ = x[TypeFrontMatterYAML-3]
+	_ = x[TypeFrontMatterTOML-4]
+	_ = x[TypeFrontMatterJSON-5]
+	_ = x[TypeFrontMatterORG-6]
+	_ = x[TypeEmoji-7]
+	_ = x[TypeIgnore-8]
+	_ = x[tLeftDelimScNoMarkup-9]
+	_ = x[tRightDelimScNoMarkup-10]
+	_ = x[tLeftDelimScWithMarkup-11]
+	_ = x[tRightDelimScWithMarkup-12]
+	_ = x[tScClose-13]
+	_ = x[tScName-14]
+	_ = x[tScNameInline-15]
+	_ = x[tScParam-16]
+	_ = x[tScParamVal-17]
+	_ = x[tIndentation-18]
+	_ = x[tText-19]
+	_ = x[tKeywordMarker-20]
+}
+
+const _ItemType_name = "tErrortEOFTypeLeadSummaryDividerTypeFrontMatterYAMLTypeFrontMatterTOMLTypeFrontMatterJSONTypeFrontMatterORGTypeEmojiTypeIgnoretLeftDelimScNoMarkuptRightDelimScNoMarkuptLeftDelimScWithMarkuptRightDelimScWithMarkuptScClosetScNametScNameInlinetScParamtScParamValtIndentationtTexttKeywordMarker"
 
-var _ItemType_index = [...]uint16{0, 6, 10, 23, 45, 64, 83, 102, 120, 129, 139, 159, 180, 202, 225, 233, 240, 253, 261, 272, 277, 291}
+var _ItemType_index = [...]uint16{0, 6, 10, 32, 51, 70, 89, 107, 116, 126, 146, 167, 189, 212, 220, 227, 240, 248, 259, 271, 276, 290}
 
 func (i ItemType) String() string {
 	if i < 0 || i >= ItemType(len(_ItemType_index)-1) {
diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go
index 20965ac0e..770f26eb9 100644
--- a/parser/pageparser/pagelexer.go
+++ b/parser/pageparser/pagelexer.go
@@ -120,6 +120,7 @@ func (l *pageLexer) next() rune {
 	runeValue, runeWidth := utf8.DecodeRune(l.input[l.pos:])
 	l.width = runeWidth
 	l.pos += l.width
+
 	return runeValue
 }
 
@@ -137,8 +138,34 @@ func (l *pageLexer) backup() {
 
 // sends an item back to the client.
 func (l *pageLexer) emit(t ItemType) {
+	defer func() {
+		l.start = l.pos
+	}()
+
+	if t == tText {
+		// Identify any trailing whitespace/intendation.
+		// We currently only care about the last one.
+		for i := l.pos - 1; i >= l.start; i-- {
+			b := l.input[i]
+			if b != ' ' && b != '\t' && b != '\r' && b != '\n' {
+				break
+			}
+			if i == l.start && b != '\n' {
+				l.items = append(l.items, Item{tIndentation, l.start, l.input[l.start:l.pos], false})
+				return
+			} else if b == '\n' && i < l.pos-1 {
+				l.items = append(l.items, Item{t, l.start, l.input[l.start : i+1], false})
+				l.items = append(l.items, Item{tIndentation, i + 1, l.input[i+1 : l.pos], false})
+				return
+			} else if b == '\n' && i == l.pos-1 {
+				break
+			}
+
+		}
+	}
+
 	l.items = append(l.items, Item{t, l.start, l.input[l.start:l.pos], false})
-	l.start = l.pos
+
 }
 
 // sends a string item back to the client.
diff --git a/parser/pageparser/pageparser.go b/parser/pageparser/pageparser.go
index bfbedcf26..67abefc30 100644
--- a/parser/pageparser/pageparser.go
+++ b/parser/pageparser/pageparser.go
@@ -149,6 +149,11 @@ func (t *Iterator) Backup() {
 	t.lastPos--
 }
 
+// Pos returns the current position in the input.
+func (t *Iterator) Pos() int {
+	return t.lastPos
+}
+
 // check for non-error and non-EOF types coming next
 func (t *Iterator) IsValueNext() bool {
 	i := t.Peek()
diff --git a/parser/pageparser/pageparser_shortcode_test.go b/parser/pageparser/pageparser_shortcode_test.go
index 54580217c..ce1297573 100644
--- a/parser/pageparser/pageparser_shortcode_test.go
+++ b/parser/pageparser/pageparser_shortcode_test.go
@@ -51,6 +51,9 @@ var shortCodeLexerTests = []lexerTest{
 
 	{"simple with markup", `{{% sc1 %}}`, []Item{tstLeftMD, tstSC1, tstRightMD, tstEOF}},
 	{"with spaces", `{{<     sc1     >}}`, []Item{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}},
+	{"indented on new line", "Hello\n    {{% sc1 %}}", []Item{nti(tText, "Hello\n"), nti(tIndentation, "    "), tstLeftMD, tstSC1, tstRightMD, tstEOF}},
+	{"indented on new line tab", "Hello\n\t{{% sc1 %}}", []Item{nti(tText, "Hello\n"), nti(tIndentation, "\t"), tstLeftMD, tstSC1, tstRightMD, tstEOF}},
+	{"indented on first line", "    {{% sc1 %}}", []Item{nti(tIndentation, "    "), tstLeftMD, tstSC1, tstRightMD, tstEOF}},
 	{"mismatched rightDelim", `{{< sc1 %}}`, []Item{
 		tstLeftNoMD, tstSC1,
 		nti(tError, "unrecognized character in shortcode action: U+0025 '%'. Note: Parameters with non-alphanumeric args must be quoted"),