/**
 * Functions to work with DDOC macros.
 *
 * Provide functionalities to perform various macro-related operations, including:
 * - Expand a text, with $(D expand).
 * - Expand a macro, with $(D expandMacro);
 * - Parse macro files (.ddoc), with $(D parseMacrosFile);
 * - Parse a "Macros:" section, with $(D parseKeyValuePair);
 * To work with embedded documentation ('.dd' files), see $(D ddoc.standalone).
 *
 * Most functions provide two interfaces.
 * One takes an $(D OutputRange) to write to, and the other one is
 * a convenience wrapper around it, which returns a string.
 * It uses an $(D std.array.Appender) as the output range.
 *
 * Most functions take a 'macros' parameter. The user is not required to pass
 * the standard D macros in it if he wants HTML output, the same macros that
 * are hardwired into DDOC are hardwired into libddoc (B, I, D_CODE, etc...).
 *
 * Note:
 * The code can contains embedded code, which will be highlighted by
 * macros substitution (see corresponding DDOC macros).
 * However, the substitution is *NOT* performed by this module, you should
 * call $(D ddoc.highlight.highlight) first.
 * If you forget to do so, libddoc will consider this as a developper
 * mistake, and will kindly inform you with an assertion error.
 *
 * Copyright: © 2014 Economic Modeling Specialists, Intl.
 * Authors: Brian Schott, Mathias 'Geod24' Lang
 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
 */
module ddoc.macros;

///
unittest
{
	import std.conv : text;
	import ddoc.lexer : Lexer;

	// Ddoc has some hardwired macros, which will be automatically searched.
	// List here: dlang.org/ddoc.html
	auto l1 = Lexer(`A simple $(B Hello $(I world))`);
	immutable r1 = expand(l1, null);
	assert(r1 == `A simple <b>Hello <i>world</i></b>`, r1);

	// Example on how to parse ddoc file / macros sections.
	KeyValuePair[] pairs;
	auto lm2 = Lexer(`GREETINGS  =  Hello $(B $0)
			  IDENTITY = $0`);
	// Acts as we are parsing a ddoc file.
	assert(parseKeyValuePair(lm2, pairs));
	// parseKeyValuePair parses up to the first invalid token, or until
	// a section is reached. It returns false on parsing failure.
	assert(lm2.empty, lm2.front.text);
	assert(pairs.length == 2, text("Expected length 2, got: ", pairs.length));
	string[string] m2;
	foreach (kv; pairs)
		m2[kv[0]] = kv[1];
	// Macros are not expanded until the final call site.
	// This allow for forward reference of macro and recursive macros.
	assert(m2.get(`GREETINGS`, null) == `Hello $(B $0)`, m2.get(`GREETINGS`, null));
	assert(m2.get(`IDENTITY`, null) == `$0`, m2.get(`IDENTITY`, null));

	// There are some more specialized functions in this module, such as
	// expandMacro which expects the lexer to be placed on a macro, and
	// will consume the input (unlike expand, which exhaust a copy).
	auto l2 = Lexer(`$(GREETINGS $(IDENTITY John Doe))`);
	immutable r2 = expand(l2, m2);
	assert(r2 == `Hello <b>John Doe</b>`, r2);

	// Note that the expansions are not processed recursively.
	// Hence, it's possible to have DDOC-formatted code inside DDOC.
	auto l3 = Lexer(`This $(DOLLAR)(MACRO) do not expand recursively.`);
	immutable r3 = expand(l3, null);
	immutable e3 = `This $(MACRO) do not expand recursively.`;
	assert(e3 == r3, r3);
}

import ddoc.lexer;
import std.exception;
import std.range;
import std.algorithm;
import std.stdio;
import std.typecons : Tuple;

alias KeyValuePair = Tuple!(string, string);

/// The set of ddoc's predefined macros.
immutable string[string] DEFAULT_MACROS;

shared static this()
{
	DEFAULT_MACROS = [`B` : `<b>$0</b>`, `I` : `<i>$0</i>`, `U` : `<u>$0</u>`,
		`P` : `<p>$0</p>`, `DL` : `<dl>$0</dl>`, `DT` : `<dt>$0</dt>`,
		`DD` : `<dd>$0</dd>`, `TABLE` : `<table>$0</table>`, `TR` : `<tr>$0</tr>`,
		`TH` : `<th>$0</th>`, `TD` : `<td>$0</td>`, `OL` : `<ol>$0</ol>`,
		`UL` : `<ul>$0</ul>`, `LI` : `<li>$0</li>`,
		`LINK` : `<a href="$0">$0</a>`, `LINK2` : `<a href="$1">$+</a>`,
		`LPAREN` : `(`, `RPAREN` : `)`, `DOLLAR` : `$`, `BACKTICK` : "`",
		`DEPRECATED` : `$0`, `RED` : `<font color=red>$0</font>`,
		`BLUE` : `<font color=blue>$0</font>`,
		`GREEN` : `<font color=green>$0</font>`,
		`YELLOW` : `<font color=yellow>$0</font>`,
		`BLACK` : `<font color=black>$0</font>`,
		`WHITE` : `<font color=white>$0</font>`,

		`D_CODE` : `<pre class="d_code">$0</pre>`,
		`D_INLINECODE` : `<pre style="display:inline;" class="d_inline_code">$0</pre>`,
		`D_COMMENT` : `$(GREEN $0)`, `D_STRING` : `$(RED $0)`,
		`D_KEYWORD` : `$(BLUE $0)`, `D_PSYMBOL` : `$(U $0)`,
		`D_PARAM` : `$(I $0)`, `DDOC` : `<html>
  <head>
    <META http-equiv="content-type" content="text/html; charset=utf-8">
    <title>$(TITLE)</title>
  </head>
  <body>
  <h1>$(TITLE)</h1>
  $(BODY)
  <hr>$(SMALL Page generated by $(LINK2 https://github.com/economicmodeling/libddoc, libddoc). $(COPYRIGHT))
  </body>
</html>`,

		`DDOC_BACKQUOTED` : `$(D_INLINECODE $0)`, `DDOC_COMMENT` : `<!-- $0 -->`,
		`DDOC_DECL` : `$(DT $(BIG $0))`, `DDOC_DECL_DD` : `$(DD $0)`,
		`DDOC_DITTO` : `$(BR)$0`, `DDOC_SECTIONS` : `$0`,
		`DDOC_SUMMARY` : `$0$(BR)$(BR)`, `DDOC_DESCRIPTION` : `$0$(BR)$(BR)`,
		`DDOC_AUTHORS` : "$(B Authors:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_BUGS` : "$(RED BUGS:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_COPYRIGHT` : "$(B Copyright:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_DATE` : "$(B Date:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_DEPRECATED` : "$(RED Deprecated:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_EXAMPLES` : "$(B Examples:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_HISTORY` : "$(B History:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_LICENSE` : "$(B License:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_RETURNS` : "$(B Returns:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_SEE_ALSO` : "$(B See Also:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_STANDARDS` : "$(B Standards:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_THROWS` : "$(B Throws:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_VERSION` : "$(B Version:)$(BR)\n$0$(BR)$(BR)",
		`DDOC_SECTION_H` : `$(B $0)$(BR)$(BR)`, `DDOC_SECTION` : `$0$(BR)$(BR)`,
		`DDOC_MEMBERS` : `$(DL $0)`,
		`DDOC_MODULE_MEMBERS` : `$(DDOC_MEMBERS $0)`,
		`DDOC_CLASS_MEMBERS` : `$(DDOC_MEMBERS $0)`,
		`DDOC_STRUCT_MEMBERS` : `$(DDOC_MEMBERS $0)`,
		`DDOC_ENUM_MEMBERS` : `$(DDOC_MEMBERS $0)`,
		`DDOC_TEMPLATE_MEMBERS` : `$(DDOC_MEMBERS $0)`,
		`DDOC_ENUM_BASETYPE` : `$0`,
		`DDOC_PARAMS` : "$(B Params:)$(BR)\n$(TABLE $0)$(BR)",
		`DDOC_PARAM_ROW` : `$(TR $0)`, `DDOC_PARAM_ID` : `$(TD $0)`,
		`DDOC_PARAM_DESC` : `$(TD $0)`, `DDOC_BLANKLINE` : `$(BR)$(BR)`,

		`DDOC_ANCHOR` : `<a name="$1"></a>`, `DDOC_PSYMBOL` : `$(U $0)`,
		`DDOC_PSUPER_SYMBOL` : `$(U $0)`, `DDOC_KEYWORD` : `$(B $0)`,
		`DDOC_PARAM` : `$(I $0)`, `ESCAPES` : `/</&lt;/
/>/&gt;/
&/&amp;/`,];
}

/**
 * Write the text from the lexer to the $(D OutputRange), and expand any macro in it..
 *
 * expand takes a $(D ddoc.Lexer), and will, until it's empty, write it's expanded version to $(D output).
 *
 * Params:
 * input = A reference to the lexer to use. When expandMacros successfully returns, it will be empty.
 * macros = A list of DDOC macros to use for expansion. This override the previous definitions, hardwired in
 *		DDOC. Which means if an user provides a macro such as $(D macros["B"] = "<h1>$0</h1>";),
 *		it will be used, otherwise the default $(D macros["B"] = "<b>$0</b>";) will be used.
 *		To undefine hardwired macros, just set them to an empty string: $(D macros["B"] = "";).
 * removeUnknown = Set to true to make unknown macros disappear from the output or false to make them output unprocessed.
 * output = An object satisfying $(D std.range.isOutputRange), usually a $(D std.array.Appender).
 */
void expand(O)(Lexer input, in string[string] macros, O output, bool removeUnknown = true) if (isOutputRange!(O,
		string))
{
	// First, we need to turn every embedded code into a $(D_CODE)
	while (!input.empty)
	{
		assert(input.front.type != Type.embedded, callHighlightMsg);
		if (input.front.type == Type.dollar)
		{
			input.popFront();
			if (input.front.type == Type.lParen)
			{
				auto mac = Lexer(matchParenthesis(input), true);
				if (!mac.empty)
				{
					if (!expandMacroImpl(mac, macros, output) && !removeUnknown)
					{
						output.put("$");
						output.put("(");
						foreach (val; mac)
							output.put(val.text);
						output.put(")");
					}
				}
			}
			else
				output.put("$");
		}
		else
		{
			output.put(input.front.text);
			input.popFront();
		}
	}
}

/// Ditto
string expand(Lexer input, string[string] macros, bool removeUnknown = true)
{
	import std.array : appender;

	auto app = appender!string();
	expand(input, macros, app, removeUnknown);
	return app.data;
}

unittest
{
	auto lex = Lexer(`Dat logo: $(LOGO dlang, Beautiful dlang logo)`);
	immutable r = expand(lex, [`LOGO` : `<img src="images/$1_logo.png" alt="$2">`]);
	immutable exp = `Dat logo: <img src="images/dlang_logo.png" alt="Beautiful dlang logo">`;
	assert(r == exp, r);
}

unittest
{
	auto lex = Lexer(`$(DIV, Evil)`);
	immutable r = expand(lex, [`DIV` : `<div $1>$+</div>`]);
	immutable exp = `<div >Evil</div>`;
	assert(r == exp, r);
}

unittest
{
	auto lex = Lexer(`$(B this) $(UNKN $(B is)) unknown!`);
	immutable r = expand(lex, [`B` : `<b>$0</b>`], false);
	immutable exp = `<b>this</b> $(UNKN $(B is)) unknown!`;
	assert(r == exp, r);
}

/**
 * Expand a macro, and write the result to an $(D OutputRange).
 *
 * It's the responsability of the caller to ensure that the lexer contains the
 * beginning of a macro. The front of the input should be either a dollar
 * followed an opening parenthesis, or an opening parenthesis.
 *
 * If the macro does not have a closing parenthesis, input will be exhausted
 * and a $(D DdocException) will be thrown.
 *
 * Params:
 * input = A reference to a lexer with front pointing to the macro.
 * macros = Additional macros to use, in addition of DDOC's ones.
 * output = An $(D OutputRange) to write to.
 */
void expandMacro(O)(ref Lexer input, in string[string] macros, O output) if (
		isOutputRange!(O, string))
	in
{
	import std.conv : text;

	assert(input.front.type == Type.dollar || input.front.type == Type.lParen,
		text("$ or ( expected, not ", input.front.type));
}
do
{
	import std.conv : text;

	if (input.front.type == Type.dollar)
		input.popFront();
	assert(input.front.type == Type.lParen, text(input.front.type));
	auto l = Lexer(matchParenthesis(input), true);
	expandMacroImpl(l, macros, output);
}

/// Ditto
string expandMacro(ref Lexer input, in string[string] macros)
in
{
	import std.conv : text;

	assert(input.front.type == Type.dollar || input.front.type == Type.lParen,
		text("$ or ( expected, not ", input.front.type));
}
do
{
	import std.array : appender;

	auto app = appender!string();
	expandMacro(input, macros, app);
	return app.data;
}

///
unittest
{
	import ddoc.lexer : Lexer;
	import std.array : appender;

	auto macros = [
		"IDENTITY" : "$0", "HWORLD" : "$(IDENTITY Hello world!)",
		"ARGS" : "$(IDENTITY $1 $+)", "GREETINGS" : "$(IDENTITY $(ARGS Hello,$0))",
	];

	auto l1 = Lexer(`$(HWORLD)`);
	immutable r1 = expandMacro(l1, macros);
	assert(r1 == "Hello world!", r1);

	auto l2 = Lexer(`$(B $(IDENTITY $(GREETINGS John Malkovich)))`);
	immutable r2 = expandMacro(l2, macros);
	assert(r2 == "<b>Hello John Malkovich</b>", r2);
}

/// A simple example, with recursive macros:
unittest
{
	import ddoc.lexer : Lexer;

	auto lex = Lexer(`$(MYTEST Un,jour,mon,prince,viendra)`);
	auto macros = [`MYTEST` : `$1 $(MYTEST $+)`];
	// Note: There's also a version of expand that takes an OutputRange.
	immutable result = expand(lex, macros);
	assert(result == `Un jour mon prince viendra `, result);
}

unittest
{
	auto macros = [
		"D" : "<b>$0</b>", "P" : "<p>$(D $0)</p>", "KP" : "<b>$1</b><i>$+</i>",
		"LREF" : `<a href="#$1">$(D $1)</a>`
	];
	auto l = Lexer(`$(D something $(KP a, b) $(P else), abcd) $(LREF byLineAsync)`c);
	immutable expected = `<b>something <b>a</b><i>b</i> <p><b>else</b></p>, abcd</b> <a href="#byLineAsync"><b>byLineAsync</b></a>`;
	auto result = appender!string();
	expand(l, macros, result);
	assert(result.data == expected, result.data);
}

unittest
{
	auto l1 = Lexer("Do you have a $(RPAREN) problem with $(LPAREN) me?");
	immutable r1 = expand(l1, null);
	assert(r1 == "Do you have a ) problem with ( me?", r1);

	auto l2 = Lexer("And (with $(LPAREN) me) ?");
	immutable r2 = expand(l2, null);
	assert(r2 == "And (with ( me) ?", r2);

	auto l3 = Lexer("What about $(TEST me) ?");
	immutable r3 = expand(l3, ["TEST" : "($0"]);
	assert(r3 == "What about (me ?", r3);
}

/**
 * Parses macros files, usually with extension .ddoc.
 *
 * Macros files are files that only contains macros definitions.
 * Newline after a macro is part of this macro, so a blank line between
 * macro A and macro B will lead to macro A having a trailing newline.
 * If you wish to split your file in blocks, terminate each block with
 * a dummy macro, e.g: '_' (underscore).
 *
 * Params:
 * paths = A variadic array with paths to ddoc files.
 *
 * Returns:
 * An associative array containing all the macros parsed from the files.
 * In case of multiple definitions, macros are overriden.
 */
string[string] parseMacrosFile(R)(R paths) if (isInputRange!(R))
{
	import std.exception : enforceEx;
	import std.file : readText;
	import std.conv : text;

	string[string] ret;
	foreach (file; paths)
	{
		KeyValuePair[] pairs;
		auto txt = readText(file);
		auto lexer = Lexer(txt, true);
		parseKeyValuePair(lexer, pairs);
		enforceEx!DdocException(lexer.empty, text("Unparsed data (",
			lexer.offset, "): ", lexer.text[lexer.offset .. $]));
		foreach (kv; pairs)
			ret[kv[0]] = kv[1];
	}
	return ret;
}

/**
 * Parses macros (or params) declaration list until the lexer is empty.
 *
 * Macros are simple Key/Value pair. So, a macro is declared as: NAME=VALUE.
 * Any number of whitespace (space / tab) can precede and follow the equal sign.
 *
 * Params:
 * lexer = A reference to lexer consisting solely of macros definition (if $(D stopAtSection) is false),
 *	   or consisting of a macro followed by other sections.
 *	   Consequently, at the end of the parsing, the lexer will be empty or may point to a section.
 * pairs = A reference to an array of $(D KeyValuePair), where the macros will be stored.
 *
 * Returns: true if the parsing succeeded.
 */
bool parseKeyValuePair(ref Lexer lexer, ref KeyValuePair[] pairs)
{
	import std.array : appender;
	import std.conv : text;

	string prevKey, key;
	string prevValue, value;
	size_t start;
	while (!lexer.empty)
	{
		// If parseAsKeyValuePair returns true, we stopped on a newline.
		// If it returns false, we're either on a section (header),
		// or the continuation of a macro.
		if (!parseAsKeyValuePair(lexer, key, value))
		{
			if (prevKey == null) // First pass and invalid data
				return false;
			if (lexer.front.type == Type.header)
				break;
			assert(lexer.offset >= prevValue.length);
			if (prevValue.length == 0)
				start = tokOffset(lexer);
			while (!lexer.empty && lexer.front.type != Type.newline)
				lexer.popFront();
			prevValue = lexer.text[start .. lexer.offset];
		}
		else
		{
			// New macro, we can save the previous one.
			// The only case when key would not be defined is on first pass.
			if (prevKey)
				pairs ~= KeyValuePair(prevKey, prevValue);
			prevKey = key;
			prevValue = value;
			key = value = null;
			start = tokOffset(lexer) - prevValue.length;
		}
		if (!lexer.empty)
		{
			assert(lexer.front.type == Type.newline, text("Front: ",
				lexer.front.type, ", text: ", lexer.text[lexer.offset .. $]));
			lexer.popFront();
		}
	}

	if (prevKey)
		pairs ~= KeyValuePair(prevKey, prevValue);

	return true;
}

private:
// upperArgs is a string[11] actually, or null.
bool expandMacroImpl(O)(Lexer input, in string[string] macros, O output)
{
	import std.conv : text;

	//debug writeln("Expanding: ", input.text);
	// Check if the macro exist and get it's value.
	if (input.front.type != Type.word)
		return false;
	string macroName = input.front.text;
	//debug writeln("[EXPAND] Macro name: ", input.front.text);
	string macroValue = lookup(macroName, macros);
	// No point loosing time if the macro is undefined.
	if (macroValue is null)
		return false;
	//debug writeln("[EXPAND] Macro value: ", macroValue);
	input.popFront();

	// Special case for $(DDOC). It's ugly, but it gets the job done.
	if (input.empty && macroName == "BODY")
	{
		output.put(lookup("BODY", macros));
		return true;
	}

	// Collect the arguments
	if (!input.empty && (input.front.type == Type.whitespace || input.front.type == Type.newline))
		input.popFront();
	string[11] arguments;
	collectMacroArguments(input, arguments);

	// First pass
	auto argOutput = appender!string();
	if (!replaceArgs(macroValue, arguments, argOutput))
		return true;

	// Second pass
	replaceMacs(argOutput.data, macros, output);
	return true;
}

unittest
{
	auto a1 = appender!string();
	expandMacroImpl(Lexer(`B value`), null, a1);
	assert(a1.data == `<b>value</b>`, a1.data);

	auto a2 = appender!string();
	expandMacroImpl(Lexer(`IDENTITY $(B value)`), ["IDENTITY" : "$0"], a2);
	assert(a2.data == `<b>value</b>`, a2.data);
}

// Try to parse a line as a KeyValuePair, returns false if it fails
private bool parseAsKeyValuePair(ref Lexer olexer, ref string key, ref string value)
{
	auto lexer = olexer;
	while (!lexer.empty && (lexer.front.type == Type.whitespace || lexer.front.type == Type.newline))
		lexer.popFront();
	if (!lexer.empty && lexer.front.type == Type.word)
	{
		key = lexer.front.text;
		lexer.popFront();
	}
	else
		return false;
	while (!lexer.empty && lexer.front.type == Type.whitespace)
		lexer.popFront();
	if (!lexer.empty && lexer.front.type == Type.equals)
		lexer.popFront();
	else
		return false;
	while (!lexer.empty && lexer.front.type == Type.whitespace)
		lexer.popFront();
	assert(lexer.offset > 0, "Something is wrong with the lexer");
	// Offset points to the END of the token, not the beginning.
	immutable size_t start = tokOffset(lexer);
	while (!lexer.empty && lexer.front.type != Type.newline)
	{
		assert(lexer.front.type != Type.header);
		lexer.popFront();
	}
	immutable size_t end = lexer.offset - ((start != lexer.offset
		&& lexer.offset != lexer.text.length) ? 1 : 0);
	value = lexer.text[start .. end];
	olexer = lexer;
	return true;
}

// Note: For macro $(NAME arg1,arg2), collectMacroArguments receive "arg1,arg2".
size_t collectMacroArguments(Lexer input, ref string[11] args)
{
	import std.conv : text;

	size_t argPos = 1;
	size_t argStart = tokOffset(input);
	args[] = null;
	if (input.empty)
		return 0;
	args[0] = input.text[tokOffset(input) .. $];
	while (!input.empty)
	{
		assert(input.front.type != Type.embedded, callHighlightMsg);
		switch (input.front.type)
		{
		case Type.comma:
			if (argPos <= 9)
				args[argPos++] = input.text[argStart .. (input.offset - 1)];
			input.popFront();
			stripWhitespace(input);
			argStart = tokOffset(input);
			// Set the $+ parameter.
			if (argPos == 2)
				args[10] = input.text[tokOffset(input) .. $];
			break;
		case Type.lParen:
			// Advance the lexer to the matching parenthesis.
			matchParenthesis(input);
			break;
			// TODO: Implement ", ' and <-- pairing.
		default:
			input.popFront();
		}
	}
	assert(argPos >= 1 && argPos <= 10, text(argPos));
	if (argPos <= 9)
		args[argPos] = input.text[argStart .. input.offset];
	return argPos;
}

unittest
{
	import std.conv : text;

	string[11] args;

	auto l1 = Lexer(`Hello, world`);
	auto c1 = collectMacroArguments(l1, args);
	assert(c1 == 2, text(c1));
	assert(args[0] == `Hello, world`, args[0]);
	assert(args[1] == `Hello`, args[1]);
	assert(args[2] == `world`, args[2]);
	for (size_t i = 3; i < 10; ++i)
		assert(args[i] is null, args[i]);
	assert(args[10] == `world`, args[10]);

	auto l2 = Lexer(`goodbye,cruel,world,I,will,happily,return,home`);
	auto c2 = collectMacroArguments(l2, args);
	assert(c2 == 8, text(c2));
	assert(args[0] == `goodbye,cruel,world,I,will,happily,return,home`, args[0]);
	assert(args[1] == `goodbye`, args[1]);
	assert(args[2] == `cruel`, args[2]);
	assert(args[3] == `world`, args[3]);
	assert(args[4] == `I`, args[4]);
	assert(args[5] == `will`, args[5]);
	assert(args[6] == `happily`, args[6]);
	assert(args[7] == `return`, args[7]);
	assert(args[8] == `home`, args[8]);
	assert(args[9] is null, args[9]);
	assert(args[10] == `cruel,world,I,will,happily,return,home`, args[10]);

	// It's not as easy as a split !
	auto l3 = Lexer(`this,(is,(just,two),args)`);
	auto c3 = collectMacroArguments(l3, args);
	assert(c3 == 2, text(c3));
	assert(args[0] == `this,(is,(just,two),args)`, args[0]);
	assert(args[1] == `this`, args[1]);
	assert(args[2] == `(is,(just,two),args)`, args[2]);
	for (size_t i = 3; i < 10; ++i)
		assert(args[i] is null, args[i]);
	assert(args[10] == `(is,(just,two),args)`, args[10]);

	auto l4 = Lexer(``);
	auto c4 = collectMacroArguments(l4, args);
	assert(c4 == 0, text(c4));
	for (size_t i = 0; i < 11; ++i)
		assert(args[i] is null, args[i]);

	import std.string : split;

	enum first = `I,am,happy,to,join,with,you,today,in,what,will,go,down,in,history,as,the,greatest,demonstration,for,freedom,in,the,history,of,our,nation.`;
	auto l5 = Lexer(first);
	auto c5 = collectMacroArguments(l5, args);
	assert(c5 == 10, text(c5));
	assert(args[0] == first, args[0]);
	foreach (idx, word; first.split(",")[0 .. 9])
		assert(args[idx + 1] == word, text(word, " != ", args[idx + 1]));
	assert(args[10] == first[2 .. $], args[10]);

	// TODO: ", ', {, <--, matched and unmatched.
}

// Where the grunt work is done...

bool replaceArgs(O)(string val, in string[11] args, O output)
{
	import std.conv : text;
	import std.ascii : isDigit;

	bool hasEnd;
	auto lex = Lexer(val, true);
	while (!lex.empty)
	{
		assert(lex.front.type != Type.embedded, callHighlightMsg);
		switch (lex.front.type)
		{
		case Type.dollar:
			lex.popFront();
			// It could be $1_test
			if (isDigit(lex.front.text[0]))
			{
				auto idx = lex.front.text[0] - '0';
				assert(idx >= 0 && idx <= 9, text(idx));
				// Missing argument
				if (args[idx] is null)
				{
					lex.popFront();
					continue;
				}
				output.put(args[idx]);
				output.put(lex.front.text[1 .. $]);
				lex.popFront();
			}
			else if (lex.front.text == "+")
			{
				if (args == string[11].init)
					return false;

				lex.popFront();
				output.put(args[10]);
			}
			else
			{
				output.put("$");
			}
			break;
		case Type.lParen:
			output.put("(");
			if (!replaceArgs(matchParenthesis(lex, &hasEnd), args, output))
				return false;
			if (hasEnd)
				output.put(")");
			break;
		default:
			output.put(lex.front.text);
			lex.popFront();
		}
	}
	return true;
}

unittest
{
	string[11] args;

	auto a1 = appender!string;
	args[0] = "Some kind of test, I guess";
	args[1] = "Some kind of test";
	args[2] = " I guess";
	assert(replaceArgs("$(MY $(SUPER $(MACRO $0)))", args, a1));
	assert(a1.data == "$(MY $(SUPER $(MACRO Some kind of test, I guess)))", a1.data);

	auto a2 = appender!string;
	args[] = null;
	args[0] = "Some,kind,of,test";
	args[1] = "Some";
	args[2] = "kind";
	args[3] = "of";
	args[4] = "test";
	args[10] = "kind,of,test";
	assert(replaceArgs("$(SOME $(MACRO $1 $+))", args, a2));
	assert(a2.data == "$(SOME $(MACRO Some kind,of,test))", a2.data);

	auto a3 = appender!string;
	args[] = null;
	args[0] = "Some,kind";
	args[1] = "Some";
	args[2] = "kind";
	args[10] = "kind";
	assert(replaceArgs("$(SOME $(MACRO $1 $2 $3))", args, a3), a3.data);

	auto a4 = appender!string;
	args[] = null;
	args[0] = "Some kind of test, I guess";
	assert(replaceArgs("$(MACRO $0 $1)", args, a4));
	assert(a4.data == "$(MACRO Some kind of test, I guess )", a4.data);
}

void replaceMacs(O)(string val, in string[string] macros, O output)
{
	//debug writeln("[REPLACE] Arguments replaced: ", val);
	bool hasEnd;
	auto lex = Lexer(val, true);
	while (!lex.empty)
	{
		assert(lex.front.type != Type.embedded, callHighlightMsg);
		switch (lex.front.type)
		{
		case Type.dollar:
			lex.popFront();
			if (lex.front.type == Type.lParen)
				expandMacro(lex, macros, output);
			else
				output.put("$");
			break;
		case Type.lParen:
			output.put("(");
			auto par = matchParenthesis(lex, &hasEnd);
			expand(Lexer(par), macros, output);
			if (hasEnd)
				output.put(")");
			break;
		default:
			output.put(lex.front.text);
			lex.popFront();
		}
	}
}

// Some utilities functions

/**
 * Must be called with a parenthesis as the front item of $(D lexer).
 * Will move the lexer forward until a matching parenthesis is met,
 * taking nesting into account.
 * If no matching parenthesis is met, returns null (and $(D lexer) will be empty).
 */
string matchParenthesis(ref Lexer lexer, bool* hasEnd = null)
in
{
	import std.conv : text;

	assert(lexer.front.type == Type.lParen, text(lexer.front));
	assert(lexer.offset);
}
do
{
	size_t count;
	size_t start = lexer.offset;
	do
	{
		if (lexer.front.type == Type.rParen)
			--count;
		else if (lexer.front.type == Type.lParen)
			++count;
		lexer.popFront();
	}
	while (count > 0 && !lexer.empty);
	size_t end = (lexer.empty) ? lexer.text.length : tokOffset(lexer);
	if (hasEnd !is null)
		*hasEnd = (count == 0);
	if (count == 0)
		end -= 1;
	return lexer.text[start .. end];
}

unittest
{
	auto l1 = Lexer(`(Hello) World`);
	immutable r1 = matchParenthesis(l1);
	assert(r1 == "Hello", r1);
	assert(!l1.empty);

	auto l2 = Lexer(`()`);
	immutable r2 = matchParenthesis(l2);
	assert(r2 == "", r2);
	assert(l2.empty);

	auto l3 = Lexer(`(())`);
	immutable r3 = matchParenthesis(l3);
	assert(r3 == "()", r3);
	assert(l3.empty);

	auto l4 = Lexer(`W (He(l)lo)`);
	l4.popFront();
	l4.popFront();
	immutable r4 = matchParenthesis(l4);
	assert(r4 == "He(l)lo", r4);
	assert(l4.empty);

	auto l5 = Lexer(` @(Hello())   ()`);
	l5.popFront();
	l5.popFront();
	immutable r5 = matchParenthesis(l5);
	assert(r5 == "Hello()", r5);
	assert(!l5.empty);

	auto l6 = Lexer(`(Hello()   (`);
	immutable r6 = matchParenthesis(l6);
	assert(r6 == "Hello()   (", r6);
	assert(l6.empty);
}

package size_t tokOffset(in Lexer lex)
{
	return lex.offset - lex.front.text.length;
}

unittest
{
	import std.conv : text;

	auto lex = Lexer(`My  (friend) $ lives abroad`);
	auto expected = [0, 2, 4, 5, 11, 12, 13, 14, 15, 20, 21];
	while (!lex.empty)
	{
		assert(expected.length > 0, "Test and results are not in sync");
		assert(tokOffset(lex) == expected[0], text(lex.front, " : ",
			tokOffset(lex), " -- ", expected[0]));
		lex.popFront();
		expected = expected[1 .. $];
	}
}

string lookup(in string name, in string[string] macros, string defVal = null)
{
	auto p = name in macros;
	if (p is null)
		return DEFAULT_MACROS.get(name, defVal);
	return *p;
}

/// Returns: The number of offset skipped.
package size_t stripWhitespace(ref Lexer lexer)
{
	size_t start = lexer.offset;
	while (!lexer.empty && (lexer.front.type == Type.whitespace || lexer.front.type == Type.newline))
	{
		start = lexer.offset;
		lexer.popFront();
	}
	return start;
}

enum callHighlightMsg = "You should call ddoc.hightlight.hightlight(string) or ddoc.unhighlight.unhighlight(string) first.";