WIP section parsing cleanup

TODO.txt

@@ -92,6 +92,21 @@
         4 01.00.00.01
         4 01.00.00.00
 
+* Sat Sep 16 11:28 2017: add test for short overline with good underline
+
+
+  Like 04.01.00.03
+
+  ==
+  ABC
+  ===
+
+* Thu Sep 14 23:53 2017: add StartLine, EndLine to System Messages for Bad Section Tests
+
+* Thu Sep 14 23:39 2017: Remove "Node" from Node types in all nodes.json files
+
+  It's redundant.
+
 * Sat Jun 17 01:10 2017: remove -def-list- from test names
 
   do the same for enum list

doc/implementation.rst

@@ -1,7 +1,7 @@
 ============================================================
 Implementation of the Go reStructuredText Parser and Tooling
 ============================================================
-:Modified: Mon Jun 19 01:12 2017
+:Modified: Thu Sep 14 11:38 2017
 
 --------
 Overview
@@ -400,7 +400,7 @@ shown in the other part of the test suite above:
 
 .. code:: console
 
-   $ rst2pseudoxml 10-test-list-option/00-short-posix/10.00.00.00-three-short-options.rst
+   $ rst2pseudoxml --halt=5 10-test-list-option/00-short-posix/10.00.00.00-three-short-options.rst
 
 In this case, the output is the same, so the reStructuredText source is good.
 

pkg/document/node.go

@@ -720,9 +720,19 @@ func (s SystemMessagesNode) MarshalJSON() ([]byte, error) {
 // SystemMessageNode are messages generated by the parser. System messages are leveled by severity and can be one of either
 // Warning, Error, Info, and Severe.
 type SystemMessageNode struct {
-	Type          NodeType `json:"type"`
-	Line          int      `json:"line,omitempty"`
-	StartPosition int      `json:"StartPosition,omitempty"`
+	Type NodeType `json:"type"`
+
+	// The line containing the problem resulted in the system message
+	Line int `json:"line,omitempty"`
+
+	// The start line of the message block
+	StartLine int `json:"startLine,omitempty"`
+
+	// The end line of the message block
+	EndLine int `json:"endLine,omitempty"`
+
+	// The character position of the start of the problem that resulted in a system message
+	StartPosition int `json:"StartPosition,omitempty"`
 
 	// The type of parser message that generated the systemMessage.
 	MessageType string `json:"messageType"`
@@ -765,6 +775,12 @@ func (s SystemMessageNode) MarshalJSON() ([]byte, error) {
 	if s.Line > 0 {
 		buffer.WriteString(fmt.Sprintf("\"line\": %d,", s.Line))
 	}
+	if s.StartLine > 0 {
+		buffer.WriteString(fmt.Sprintf("\"startLine\": %d,", s.StartLine))
+	}
+	if s.EndLine > 0 {
+		buffer.WriteString(fmt.Sprintf("\"endLine\": %d,", s.EndLine))
+	}
 	if s.StartPosition > 0 {
 		buffer.WriteString(fmt.Sprintf("\"startPosition\": %d,", s.StartPosition))
 	}
@@ -1156,7 +1172,9 @@ func (d DefinitionNode) String() string { return fmt.Sprintf("%#v", d) }
 func (d DefinitionNode) MarshalJSON() ([]byte, error) {
 	buffer := bytes.NewBufferString("{")
 	buffer.WriteString(fmt.Sprintf("\"type\": %q,", d.Type.String()))
-	buffer.WriteString(fmt.Sprintf("\"line\": %d,", d.Line))
+	if d.Line > 0 {
+		buffer.WriteString(fmt.Sprintf("\"line\": %d,", d.Line))
+	}
 	b, err := json.Marshal(d.NodeList)
 	if err != nil {
 		return nil, err

pkg/parser/buffer.go

@@ -147,6 +147,15 @@ func (t *tokenBuffer) next(pos int) *tok.Item {
 }
 
 func (t *tokenBuffer) peekLine(line int) (toks []*tok.Item) {
+	// peek the parser until line > p.line
+	x := 1
+	for {
+		pt := t.peek(x)
+		if pt == nil || pt.Type == tok.EOF || pt.Line > line {
+			break
+		}
+		x++
+	}
 	for x := 0; x < len(t.buf)-1; x++ {
 		if t.buf[x] != nil && t.buf[x].Line == line {
 			toks = append(toks, t.buf[x])

pkg/parser/inline_markup.go

@@ -10,6 +10,7 @@ import (
 func (p *Parser) inlineEmphasis(i *tok.Item, titleCheck bool) {
 	// Make sure inline markup is not in a section title
 	isInTitle := p.isInlineMarkupInSectionTitle(i)
+	// p.DumpExit(isInTitle)
 	if titleCheck && isInTitle {
 		return
 	}

pkg/parser/paragraph.go

@@ -87,7 +87,7 @@ outer:
 			p.backup()
 			break outer
 		}
-		p.DumpExit(p.Nodes)
+		// p.DumpExit(p.Nodes)
 		p.Msg("Continuing...")
 	}
 	p.Msgr("number of indents", "p.indents.len", p.indents.len())

pkg/parser/section.go

@@ -15,65 +15,77 @@ type sectionParseSubState struct {
 }
 
 func parseSectionTitle(s *sectionParseSubState, p *Parser, item *tok.Item) bool {
-	p.Msg("next type == tok.Title")
+	p.Msg("parsing section title with overline and underline")
 
 	// Section with overline
-	pBack := p.peekBack(1)
-	tLen := p.token.Length
-
-	if tLen < 3 && tLen != s.sectionSpace.Length {
-		p.next(2)
-		bTok := p.peekBack(1)
-		if bTok != nil && bTok.Type == tok.Space {
-			p.next(2)
-			return p.systemMessage(mes.SectionWarningUnexpectedTitleOverlineOrTransition)
-		}
+	overline := p.token
+	title := p.peekSkip(tok.Space)
+	und := p.peekLine(title.Line + 1)
+	var underline *tok.Item
+	if len(und) > 0 {
+
+		underline = und[0]
+	}
+
+	// pBack := p.peekBack(1)
+	// tLen := p.token.Length
+	// p.DumpExit(underline)
+	// p.DumpExit(title)
+
+	// p.DumpExit(overline)
+	if overline.Length < 3 && overline.Length != title.Length {
+		// p.next(2)
+		// if bTok != nil && bTok.Type == tok.Space {
+		// p.next(2)
+		// return p.systemMessage(mes.SectionWarningUnexpectedTitleOverlineOrTransition)
+		// }
 		return p.systemMessage(mes.SectionWarningOverlineTooShortForTitle)
-	} else if pBack != nil && pBack.Type == tok.Space {
-		// Indented section (error) The section title has an indented overline
-		return p.systemMessage(mes.SectionErrorUnexpectedSectionTitleOrTransition)
+	} else if underline == nil || underline.Type != tok.SectionAdornment {
+		return p.systemMessage(mes.SectionErrorIncompleteSectionTitle)
+	} else if underline.Length < 3 && underline.Length != title.Length {
+		return p.systemMessage(mes.SectionWarningUnderlineTooShortForTitle)
+		// } else if pBack != nil && pBack.Type == tok.Space {
+		// // Indented section (error) The section title has an indented overline
+		// return p.systemMessage(mes.SectionErrorUnexpectedSectionTitleOrTransition)
 	}
 
 	s.sectionOverAdorn = item
 	p.next(1)
 
 loop:
 	for {
-		switch tTok := p.token; tTok.Type {
+		switch cTok := p.token; cTok.Type {
 		case tok.Title:
-			s.sectionTitle = doc.NewTitleNodeWithText(tTok)
+			s.sectionTitle = doc.NewTitleNodeWithText(cTok)
 			p.next(1)
 		case tok.Space:
-			s.sectionIndent = tTok
+			s.sectionIndent = cTok
 			p.next(1)
 		case tok.SectionAdornment:
-			s.sectionUnderAdorn = tTok
+			s.sectionUnderAdorn = cTok
 			break loop
 		}
 	}
 	return true
 }
 
 func parseSectionTitleNoOverline(s *sectionParseSubState, p *Parser, i *tok.Item) bool {
-	tLen := p.token.Length
-	pBack := p.peekBack(1)
-	p.Msgr("last item type", "type", pBack.Type)
-	if pBack.Type == tok.Space {
-		pBack := p.peekBack(2)
-		if pBack != nil && pBack.Type == tok.Title {
-			// The section underline is indented
-			return p.systemMessage(mes.SectionErrorUnexpectedSectionTitle)
-		}
-	} else if pBack.Type == tok.SectionAdornment && tLen < 3 && tLen != pBack.Length {
-		p.DumpExit(pBack)
+	title := p.peekBack(1)
+	underlineLen := p.token.Length
+	// if title.Type == tok.Space {
+	// title := p.peekBack(2)
+	// if title != nil && title.Type == tok.Title {
+	// // The section underline is indented
+	// return p.systemMessage(mes.SectionErrorUnexpectedSectionTitle)
+	// }
+	// } else if title.Type == tok.SectionAdornment && underlineLen < 3 && underlineLen != title.Length {
+	if underlineLen < 3 && underlineLen != title.Length {
 		// Short underline
 		return p.systemMessage(mes.SectionWarningUnderlineTooShortForTitle)
 	}
-
 	// Section OKAY
-	s.sectionTitle = doc.NewTitleNodeWithText(pBack)
+	s.sectionTitle = doc.NewTitleNodeWithText(title)
 	s.sectionUnderAdorn = i
-
 	return true
 }
 
@@ -125,28 +137,38 @@ func parseSectionTitleWithInlineMarkupAndNoOverline(s *sectionParseSubState, p *
 
 func sectionOK(s *sectionParseSubState, p *Parser, i *tok.Item) bool {
 	if s.sectionSpace != nil && s.sectionSpace.Type == tok.Text {
+		// p.DumpExit(s)
 		//
 		// If sectionSpace is set to a tok.Text,
 		// * The underline is missing, therefore we generate an error based on what follows the tok.Text
 		// * There is no blankline after the underline
 		//
+
 		// Check for an underline...
-		if und := p.peekLine(s.sectionSpace.Line - 1); len(und) == 1 && und[0].Type == tok.SectionAdornment {
-			// There is no blankline after the underline, this is a good section
-			return true
-		}
+		// underline := p.peekLine(s.sectionSpace.Line - 1)
+		// if len(underline) == 1 && underline[0].Type == tok.SectionAdornment && underline[0].Length > 2 {
+		// // There is no blankline after the underline, this is a good section
+		// return true
+		// }
 		p.Msg("IN HERE 4")
-		tLen := p.token.Length
-		p.next(2) // Move the token buffer past the error tokens
+		// p.next(2) // Move the token buffer past the error tokens
 
-		if tLen < 3 && tLen != s.sectionSpace.Length {
-			p.backup()
+		if tLen := p.token.Length; tLen < 3 && tLen != s.sectionSpace.Length {
+			// p.backup()
 			return p.systemMessage(mes.SectionWarningOverlineTooShortForTitle)
 		} else if t := p.peek(1); t != nil && t.Type == tok.BlankLine {
 			return p.systemMessage(mes.SectionErrorMissingMatchingUnderlineForOverline)
 		}
 
 		return p.systemMessage(mes.SectionErrorIncompleteSectionTitle)
+	} else if s.sectionSpace != nil && s.sectionSpace.Type == tok.BlankLine {
+		// Section title with underline followed by blankline and no overline
+		p.Msg("IN HERE ALPHA")
+		p.DumpExit(p.token)
+		if tLen := p.token.Length; tLen < 3 && tLen != s.sectionSpace.Length {
+			// p.backup()
+			return p.systemMessage(mes.SectionWarningUnderlineTooShortForTitle)
+		}
 	} else if s.sectionSpace != nil && s.sectionSpace.Type == tok.SectionAdornment {
 		p.Msg("IN HERE 5")
 		// Missing section title
@@ -169,19 +191,36 @@ func parseSection(s *sectionParseSubState, p *Parser, i *tok.Item) bool {
 	pBack := p.peekBack(1)
 	pBackTitle := p.peekBackTo(tok.Title)
 
-	if !sectionOK(s, p, i) {
-		return false
+	// Section has no overline
+	// p.DumpExit(i)
+	// Section has overline
+	overline := p.token
+	title := p.peek(1)
+	// underline := p.peek(2)
+	if title != nil && title.Type == tok.Space {
+		title = p.peekSkip(tok.Space)
+		// underline = p.peekLine(title.Line + 1)[0]
 	}
 
-	if s.sectionSpace != nil && s.sectionSpace.Type == tok.Title {
+	// if !sectionOK(s, p, i) {
+	// return false
+	// }
+
+	// p.DumpExit(overline)
+	// p.DumpExit(title)
+	// p.DumpExit(underline)
+
+	if pBack != nil && pBack.Type == tok.Title && p.token.Type == tok.SectionAdornment { // || pBack.Type == tok.Space) {
+		// Title with underline and no overline
+		p.Msg("IN HERE 3")
+		return parseSectionTitleNoOverline(s, p, i)
+	} else if overline.Type == tok.SectionAdornment && (title.Type == tok.Title || title.Type == tok.Text) { //&& underline.Type == tok.SectionAdornment {
 		p.Msg("IN HERE 1")
+		// section title with underline and overline
 		return parseSectionTitle(s, p, i)
 	} else if pBackTitle != nil && p.isInlineMarkupInSectionTitle(pBackTitle) {
 		p.Msg("IN HERE 2")
 		return parseSectionTitleWithInlineMarkupAndNoOverline(s, p, i)
-	} else if pBack != nil && (pBack.Type == tok.Title || pBack.Type == tok.Space) {
-		p.Msg("IN HERE 3")
-		return parseSectionTitleNoOverline(s, p, i)
 	}
 
 	p.Msg("IN HERE 8")
@@ -235,7 +274,8 @@ func checkSectionLengths(s *sectionParseSubState, p *Parser, sec *doc.SectionNod
 func (p *Parser) section(i *tok.Item) {
 	p.Msgr("have item", "item", i)
 
-	s := &sectionParseSubState{sectionSpace: p.peekSkip(tok.Space)}
+	// s := &sectionParseSubState{sectionSpace: p.peekSkip(tok.Space)}
+	s := &sectionParseSubState{}
 	if !parseSection(s, p, i) {
 		p.Msg("Failed to parse section!")
 		return