Parsing yaml metadata headers in markdown files (#191)

This PR updates our markdown parser to support parsing yaml headers and
tests to make sure that some of the features we'd like to use from it
work.

Author: vmarcella

go.mod

@@ -42,10 +42,12 @@ require (
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
+	github.com/yuin/goldmark-meta v1.1.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -94,6 +94,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
 github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
 github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
+github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
+github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0=
 golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
@@ -123,6 +125,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/parsers/markdown.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/Azure/InnovationEngine/internal/logging"
 	"github.com/yuin/goldmark"
+	meta "github.com/yuin/goldmark-meta"
 	"github.com/yuin/goldmark/ast"
 	"github.com/yuin/goldmark/extension"
 	"github.com/yuin/goldmark/parser"
@@ -17,7 +18,7 @@ import (
 )
 
 var markdownParser = goldmark.New(
-	goldmark.WithExtensions(extension.GFM),
+	goldmark.WithExtensions(extension.GFM, meta.New(meta.WithStoresInDocument())),
 	goldmark.WithParserOptions(
 		parser.WithAutoHeadingID(),
 		parser.WithBlockParsers(),
@@ -33,6 +34,10 @@ func ParseMarkdownIntoAst(source []byte) ast.Node {
 	return document
 }
 
+func ExtractYamlMetadataFromAst(node ast.Node) map[string]interface{} {
+	return node.OwnerDocument().Meta()
+}
+
 // The representation of an expected output block in a markdown file. This is
 // for scenarios that have expected output that should be validated against the
 // actual output.
@@ -237,7 +242,6 @@ func convertScenarioVariablesToMap(variableBlock string) map[string]string {
 				variableMap[key] = value
 			}
 		}
-
 	}
 
 	return variableMap

internal/parsers/markdown_test.go

@@ -10,7 +10,6 @@ func TestParsingMarkdownHeaders(t *testing.T) {
 		markdown := []byte(`# Hello World`)
 		document := ParseMarkdownIntoAst(markdown)
 		title, err := ExtractScenarioTitleFromAst(document, markdown)
-
 		if err != nil {
 			t.Errorf("Error parsing title: %s", err)
 		}
@@ -24,7 +23,6 @@ func TestParsingMarkdownHeaders(t *testing.T) {
 		markdown := []byte("# Hello World \n # Hello again")
 		document := ParseMarkdownIntoAst(markdown)
 		title, err := ExtractScenarioTitleFromAst(document, markdown)
-
 		if err != nil {
 			t.Errorf("Error parsing title: %s", err)
 		}
@@ -50,8 +48,60 @@ func TestParsingMarkdownHeaders(t *testing.T) {
 	})
 }
 
-func TestParsingMarkdownCodeBlocks(t *testing.T) {
+func TestParsingYamlMetadata(t *testing.T) {
+	t.Run("Markdown with valid yaml metadata", func(t *testing.T) {
+		markdown := []byte(`---
+    key: value
+    array: [1, 2, 3]
+    ---
+    `)
+
+		document := ParseMarkdownIntoAst(markdown)
+		metadata := ExtractYamlMetadataFromAst(document)
+
+		if metadata["key"] != "value" {
+			t.Errorf("Metadata is wrong: %v", metadata)
+		}
+
+		array := metadata["array"].([]interface{})
+		if array[0] != 1 || array[1] != 2 || array[2] != 3 {
+			t.Errorf("Metadata is wrong: %v", metadata)
+		}
+	})
+
+	t.Run("Markdown without yaml metadata", func(t *testing.T) {
+		markdown := []byte(`# Hello World.`)
+		document := ParseMarkdownIntoAst(markdown)
+		metadata := ExtractYamlMetadataFromAst(document)
+
+		if len(metadata) != 0 {
+			t.Errorf("Metadata should be empty")
+		}
+	})
+
+	t.Run("yaml with nested properties", func(t *testing.T) {
+		markdown := []byte(`---
+    nested:
+      key: value
+    key.value: otherValue
+    ---
+    `)
 
+		document := ParseMarkdownIntoAst(markdown)
+		metadata := ExtractYamlMetadataFromAst(document)
+
+		nested := metadata["nested"].(map[interface{}]interface{})
+		if nested["key"] != "value" {
+			t.Errorf("Metadata is wrong: %v", metadata)
+		}
+
+		if metadata["key.value"] != "otherValue" {
+			t.Errorf("Metadata is wrong: %v", metadata)
+		}
+	})
+}
+
+func TestParsingMarkdownCodeBlocks(t *testing.T) {
 	t.Run("Markdown with a valid bash code block", func(t *testing.T) {
 		markdown := []byte(fmt.Sprintf("# Hello World\n ```bash\n%s\n```", "echo Hello"))
 
@@ -74,13 +124,16 @@ func TestParsingMarkdownCodeBlocks(t *testing.T) {
 			)
 		}
 	})
-
 }
 
 func TestParsingMarkdownExpectedSimilarty(t *testing.T) {
-
 	t.Run("Markdown with a expected_similarty tag using float", func(t *testing.T) {
-		markdown := []byte(fmt.Sprintf("```bash\n%s\n```\n<!--expected_similarity=0.8-->\n```\nHello\n```\n", "echo Hello"))
+		markdown := []byte(
+			fmt.Sprintf(
+				"```bash\n%s\n```\n<!--expected_similarity=0.8-->\n```\nHello\n```\n",
+				"echo Hello",
+			),
+		)
 
 		document := ParseMarkdownIntoAst(markdown)
 		codeBlocks := ExtractCodeBlocksFromAst(document, markdown, []string{"bash"})
@@ -92,16 +145,23 @@ func TestParsingMarkdownExpectedSimilarty(t *testing.T) {
 		block := codeBlocks[0].ExpectedOutput
 		expectedFloat := .8
 		if block.ExpectedSimilarity != expectedFloat {
-			t.Errorf("ExpectedSimilarity is wrong, got %f, expected %f", block.ExpectedSimilarity, expectedFloat)
+			t.Errorf(
+				"ExpectedSimilarity is wrong, got %f, expected %f",
+				block.ExpectedSimilarity,
+				expectedFloat,
+			)
 		}
 	})
-
 }
 
 func TestParsingMarkdownExpectedRegex(t *testing.T) {
-
 	t.Run("Markdown with a expected_similarty tag using regex", func(t *testing.T) {
-		markdown := []byte(fmt.Sprintf("```bash\n%s\n```\n<!--expected_similarity=\"Foo \\w+\"-->\n```\nFoo Bar\n```\n", "echo 'Foo Bar'"))
+		markdown := []byte(
+			fmt.Sprintf(
+				"```bash\n%s\n```\n<!--expected_similarity=\"Foo \\w+\"-->\n```\nFoo Bar\n```\n",
+				"echo 'Foo Bar'",
+			),
+		)
 
 		document := ParseMarkdownIntoAst(markdown)
 		codeBlocks := ExtractCodeBlocksFromAst(document, markdown, []string{"bash"})
@@ -121,5 +181,4 @@ func TestParsingMarkdownExpectedRegex(t *testing.T) {
 			t.Errorf("ExpectedRegex is wrong, got %q, expected %q", stringRegex, expectedRegex)
 		}
 	})
-
 }