FrontMatter: new subparser parser run from Markdown parser

In this version, just extracting "foo" in "title: foo" written
in Yaml. JSON(;;;) and TOML(+++) are not supported yet.

Close universal-ctags#3032.

Signed-off-by: Masatake YAMATO <[email protected]>

Units/parser-rmarkdown.r/frontmatter.d/args.ctags

@@ -0,0 +1,3 @@
+--sort=no
+--extras=+g
+--fields=+KenlE

Units/parser-rmarkdown.r/frontmatter.d/expected.tags

@@ -0,0 +1,12 @@
+S1	input.rmd	/^# S1$/;"	chapter	line:4	language:Markdown	end:17
+xyX	input.rmd	/^```{r xyX}$/;"	chunklabel	line:6	language:RMarkdown	extras:subparser
+S2	input.rmd	/^# S2$/;"	chapter	line:18	language:Markdown	end:28
+__anon9d4e183a0100	input.rmd	/^```{r, cache = TRUE, dependson = "xyX"}$/;"	chunklabel	line:20	language:RMarkdown	extras:subparser,anonymous
+__anon9d4e183a0200	input.rmd	/^```{python}$/;"	chunklabel	line:24	language:RMarkdown	extras:subparser,anonymous
+S3	input.rmd	/^# S3$/;"	chapter	line:29	language:Markdown	end:30
+x	input.rmd	/^x <- 1$/;"	globalVar	line:8	language:R	extras:guest	end:8
+foo	input.rmd	/^foo <- function () {$/;"	function	line:9	language:R	extras:guest	end:12
+y	input.rmd	/^    y <- 2$/;"	functionVar	line:10	language:R	function:foo	extras:guest	end:10
+X	input.rmd	/^X <- func()$/;"	globalVar	line:14	language:R	extras:guest	end:14
+f	input.rmd	/^def f():$/;"	function	line:25	language:Python	extras:guest	end:27
+example for u-ctags	input.rmd	/^title: example for u-ctags$/;"	title	line:2	language:FrontMatter	extras:subparser

Units/parser-rmarkdown.r/frontmatter.d/input.rmd

@@ -0,0 +1,30 @@
+---
+title: example for u-ctags
+---
+# S1
+
+```{r xyX}
+
+x <- 1
+foo <- function () {
+    y <- 2
+    return(y)
+}
+
+X <- func()
+
+```
+
+# S2
+
+```{r, cache = TRUE, dependson = "xyX"}
+mean(X)
+```
+
+```{python}
+def f():
+   g()
+   return 3
+```
+# S3
+

Units/parser-rmarkdown.r/frontmatter.d/languages

@@ -0,0 +1 @@
+Yaml

docs/news.rst

@@ -412,6 +412,7 @@ The following parsers have been added:
 * Elixir *optlib*
 * Elm *optlib*
 * Falcon
+* FrontMatter *only YAML syntax, running as a guest  on R?Markdown*
 * FunctionParameters *perl based subparser*
 * Gdbinit script *optlib*
 * GemSpec *Ruby based subparser*

main/parsers_p.h

@@ -30,7 +30,8 @@
 #define YAML_PARSER_LIST						\
 	YamlParser,									\
 	AnsiblePlaybookParser, \
-	OpenAPIParser
+	OpenAPIParser, \
+	YamlFrontMatter
 #else
 #define YAML_PARSER_LIST
 #endif
@@ -85,6 +86,7 @@
 	FalconParser, \
 	FlexParser, \
 	FortranParser, \
+	FrontMatterParser, \
 	FunctionParametersParser, \
 	FyppParser,	   \
 	GdbinitParser, \

parsers/frontmatter.c

@@ -0,0 +1,75 @@
+/*
+*
+*   Copyright (c) 2022, Masatake YAMATO
+*   Copyright (c) 2022, Red Hat, K.K.
+*
+*   This source code is released for free distribution under the terms of the
+*   GNU General Public License version 2 or (at your option) any later version.
+*
+* This module contains functions for extracting language objects in FrontMatter.
+*
+* https://gohugo.io/content-management/front-matter
+*/
+
+/*
+*   INCLUDE FILES
+*/
+#include "general.h"	/* must always come first */
+
+#include "frontmatter.h"
+
+#include "entry.h"
+#include "parse.h"
+#include "promise.h"
+#include "read.h"
+
+#include <string.h>
+
+/*
+*   DATA DEFINITIONS
+*/
+static kindDefinition FrontMatterKinds [] = {
+	{ true, 't', "title", "titles", },
+};
+
+/*
+*   FUNCTION DEFINITIONS
+*/
+static void findFrontMatterTags (void)
+{
+	const unsigned char *line = readLineFromInputFile ();
+
+	if (line == NULL)
+		return;
+
+#ifdef HAVE_LIBYAML
+	if (strcmp("---", (const char *)line) == 0)
+	{
+		line = readLineFromInputFile ();
+		unsigned long endOffset = strlen((const char *)line);
+		if (line)
+		{
+			long startLineNum = getInputLineNumber ();
+			while ((line = readLineFromInputFile()))
+				endOffset = strlen((const char *)line);
+
+			long endLineNum = getInputLineNumber ();
+
+			makePromise ("YamlFrontMatter", startLineNum, 0,
+						 endLineNum, endOffset, startLineNum);
+		}
+		return;
+	}
+#endif
+}
+
+extern parserDefinition* FrontMatterParser (void)
+{
+	parserDefinition* def = parserNew ("FrontMatter");
+	def->kindTable      = FrontMatterKinds;
+	def->kindCount  = ARRAY_SIZE (FrontMatterKinds);
+
+	def->parser     = findFrontMatterTags;
+
+	return def;
+}

parsers/frontmatter.h

@@ -0,0 +1,19 @@
+/*
+ *   Copyright (c) 2022, Masatake YAMATO <[email protected]>
+ *
+ *   This source code is released for free distribution under the terms of the
+ *   GNU General Public License version 2 or (at your option) any later version.
+ *
+ *   Frontmatter parser interface exported to the other parsers
+ */
+
+#ifndef CTAGS_FRONTMATTER_H
+#define CTAGS_FRONTMATTER_H
+
+#include "general.h"
+
+typedef enum {
+	FRONTMATTER_TITLE_KIND,
+} frontmatterKind;
+
+#endif

parsers/markdown.c

@@ -260,7 +260,18 @@ static void findMarkdownTags (void)
 		if (lineNum == 1 || inPreambule)
 		{
 			if (line[pos] == '-' && line[pos + 1] == '-' && line[pos + 2] == '-')
+			{
+				if (inPreambule)
+				{
+					long endLineNumber = lineNum;
+					if (startLineNumber < endLineNumber)
+						makePromise ("FrontMatter", startLineNumber, 0,
+									 endLineNumber, 0, startSourceLineNumber);
+				}
+				else
+					startSourceLineNumber = startLineNumber = lineNum;
 				inPreambule = !inPreambule;
+			}
 		}
 
 		if (inPreambule)
@@ -401,5 +412,19 @@ extern parserDefinition* MarkdownParser (void)
 	def->defaultScopeSeparator = "\"\"";
 	def->parser = findMarkdownTags;
 
+	/*
+	 * This setting (useMemoryStreamInput) is for running
+	 * Yaml parser from YamlFrontMatter as subparser.
+	 * YamlFrontMatter is run from FrontMatter as a gust parser.
+	 * FrontMatter is run from Markdown as a guest parser.
+	 * This stacked structure hits the limitation of the main
+	 * part: subparser's requirement for memory based input stream
+	 * is not propagated to the main part.
+	 *
+	 * TODO: instead of setting useMemoryStreamInput here, we
+	 * should remove the limitation.
+	 */
+	def->useMemoryStreamInput = true;
+
 	return def;
 }