Show HTML entities in article text as their characters

README.md

@@ -21,6 +21,7 @@ A wiki for phpBB with version history, side-by-side compare, and a moderation qu
 - **Watch article** on every article page, for signed-in members: they are notified when a new version of a watched article goes live (a moderator approved an edit, or a trusted editor's edit went live directly), never about their own edit. It is a notification type in phpBB's notification settings, so members choose the bell, email or both. With the ACP setting **Watch edited articles** (on by default, under **Extensions > Wiki > Settings**), saving an edit also starts watching the article. Watches are removed with the article and with the member.
 - Wikipedia-style headings: `== Heading ==` to `====== Heading ======` in an article become section headings with anchors, `[[Article#Section]]` and `[[#Section]]` link to them, and a table of contents shows after the intro when an article has four or more headings (`__TOC__` places it, `__FORCETOC__` forces it, `__NOTOC__` hides it). A first line `#REDIRECT [[Article]]` sets the redirect for members who may set redirects. This works in wiki articles only; in a forum post `== text ==` stays plain text.
 - Tables, written as HTML in an article: `<table>`, `<caption>`, `<thead>`, `<tbody>`, `<tfoot>`, `<tr>`, `<th>` and `<td>`, with `colspan` and `rowspan` on a cell. BBCode and `[[links]]` work inside cells. Any other attribute (`style`, `onclick`, `class`) is dropped, and any other element (`script`, `iframe`) stays plain text. Like headings, tables work in wiki articles only; in a forum post `<table>` stays plain text. After upgrading, disable and enable the extension so the board rebuilds its cache.
+- HTML entities written in an article, such as `&nbsp;`, `&#32;` or `&mdash;`, show as the characters they stand for, as on MediaWiki, in the text and in headings, link texts and link targets. `&lt;b&gt;` shows as "<b>", never as markup, and an unknown entity stays as written. Like headings, this works in wiki articles only; a forum post shows entities as written. After upgrading, disable and enable the extension so the board rebuilds its cache. Articles saved earlier show their entities as written until they are saved again or reparsed with `php bin/phpbbcli.php reparser:reparse phpbbmodders_wiki_article`.
 - Templates: `{{Name}}`, `{{Name|first|second}}` and `{{Name|key=value}}` in an article include the live version of the article `Template:Name`, so a notice or infobox is written once and edited in one place. See [Templates](#templates).
 - Categories: `[[Category:Name]]` in an article puts it in a category. The article lists its categories at the bottom, `/wiki/Category:Name` lists the articles in a category, and `/wiki/categories` lists every category. See [Categories](#categories).
 - Search: `/wiki/search` and a search box on the overview find articles by their title, description and text, with highlighted snippets. It has its own index, kept current as articles are saved, approved, set offline and deleted, and three backends to choose from in the ACP (native, MySQL fulltext, PostgreSQL fulltext). The board's own search can show the matching articles in a block of their own, with an "Also search the Wiki" option. See [Search](#search).

event/wikilink_listener.php

@@ -114,7 +114,9 @@ public function configure_wikilink($event)
 		// parser reads it only while a wiki article is saved (see toggle_wiki_markup()); a sort key
 		// after a bar is accepted and ignored.
 		$category = $configurator->tags->add('WIKICATEGORY');
-		$category->attributes->add('name')->filterChain->append('\phpbbmodders\wiki\wiki\categories::normalise');
+		$category_name = $category->attributes->add('name');
+		$category_name->filterChain->append('\phpbbmodders\wiki\wiki\markup::decode_entities_in_name');
+		$category_name->filterChain->append('\phpbbmodders\wiki\wiki\categories::normalise');
 		$category->template = '<span class="wikicategory" data-wiki-category="{@name}"></span>';
 
 		$configurator->Preg->match(
@@ -126,7 +128,9 @@ public function configure_wikilink($event)
 		// It is added first, and WIKILINK leaves "File:" names to it.
 		$file_tag = $configurator->tags->add('WIKIFILE');
 		$file_tag->attributes->add('name');
-		$file_tag->attributes->add('alt')->required = false;
+		$file_alt = $file_tag->attributes->add('alt');
+		$file_alt->required = false;
+		$file_alt->filterChain->append('\phpbbmodders\wiki\wiki\markup::decode_entities');
 		$file_tag->template = '<span class="wikifile" data-wiki-file="{@name}" data-wiki-alt="{@alt}"></span>';
 
 		$configurator->Preg->match(
@@ -139,7 +143,8 @@ public function configure_wikilink($event)
 		// table of contents are added when the article is shown (see markup::finish()).
 		$heading = $configurator->tags->add('WIKIHEADING');
 		$heading->attributes->add('marks');
-		$heading->attributes->add('title');
+		// Entities in captured text are decoded here, since the entity plugin never sees it
+		$heading->attributes->add('title')->filterChain->append('\phpbbmodders\wiki\wiki\markup::decode_entities');
 		$heading->template = '<xsl:choose>'
 			. '<xsl:when test="string-length(@marks) = 2"><h2 class="wikiheading"><xsl:value-of select="@title"/></h2></xsl:when>'
 			. '<xsl:when test="string-length(@marks) = 3"><h3 class="wikiheading"><xsl:value-of select="@title"/></h3></xsl:when>'
@@ -173,9 +178,12 @@ public function configure_wikilink($event)
 		$configurator->Preg->match(self::template_pattern(), 'WIKITEMPLATE');
 
 		$tag = $configurator->tags->add('WIKILINK');
-		$tag->attributes->add('name')->required = false;
-		$tag->attributes->add('section')->required = false;
-		$tag->attributes->add('text')->required = false;
+		foreach (array('name' => 'decode_entities_in_name', 'section' => 'decode_entities', 'text' => 'decode_entities') as $attribute => $decode)
+		{
+			$link_attribute = $tag->attributes->add($attribute);
+			$link_attribute->required = false;
+			$link_attribute->filterChain->append('\phpbbmodders\wiki\wiki\markup::' . $decode);
+		}
 		$tag->template = '<a class="wikilink" data-wiki-article="{@name}" data-wiki-section="{@section}">'
 			. '<xsl:choose><xsl:when test="@text"><xsl:value-of select="@text"/></xsl:when>'
 			. '<xsl:otherwise><xsl:value-of select="@name"/><xsl:if test="@section">#<xsl:value-of select="@section"/></xsl:if></xsl:otherwise></xsl:choose></a>';
@@ -185,6 +193,12 @@ public function configure_wikilink($event)
 			'WIKILINK'
 		);
 
+		// HTML entities written in an article, such as &nbsp;, &#32; or &mdash;, become
+		// the characters they stand for, as on MediaWiki. The character is output as
+		// text, so &lt; shows as "<" and never becomes markup. The parser reads them
+		// only while a wiki article is saved (see toggle_wiki_markup()).
+		$configurator->HTMLEntities;
+
 		// <table> written as HTML. Only these elements and the colspan and rowspan of a cell
 		// are allowed; s9e refuses scripts, styles and event attributes itself. The parser
 		// reads them only while a wiki article is saved (see toggle_wiki_markup()).
@@ -228,6 +242,16 @@ public function toggle_wiki_markup($event)
 				$parser->disableTag($tag);
 			}
 		}
+
+		// Entities, only in text as written in an article (see markup::set_entities())
+		if (\phpbbmodders\wiki\wiki\markup::is_active() && \phpbbmodders\wiki\wiki\markup::entities())
+		{
+			$parser->enableTag('HE');
+		}
+		else
+		{
+			$parser->disableTag('HE');
+		}
 	}
 
 	public function link_wikilinks($event)

tests/unit/markup_test.php

@@ -377,4 +377,42 @@ public static function link_texts(): array
 			'own text, same page'	=> array('[[#History|below]]', 'below'),
 		);
 	}
+
+	public function test_entities_in_captured_text_are_decoded_only_while_wiki_markup_is_on()
+	{
+		$text = 'Head—ing  x  y <b> < &bogus; 
';
+		try
+		{
+			// A forum post: left as written
+			markup::set_active(false);
+			$this->assertSame($text, markup::decode_entities($text));
+
+			// Text as written in an article: decoded, unknown and control entities kept
+			markup::set_active(true);
+			$this->assertSame("Head\u{2014}ing \u{00A0}x  y <b> &lt; &bogus; &#10;", markup::decode_entities($text));
+
+			// Template text, already decoded once on its way to the parser: left alone
+			markup::set_entities(false);
+			$this->assertSame($text, markup::decode_entities($text));
+		}
+		finally
+		{
+			markup::set_active(false);
+			markup::set_entities(true);
+		}
+	}
+
+	public function test_a_non_breaking_space_in_an_article_name_is_a_space()
+	{
+		markup::set_active(true);
+		try
+		{
+			$this->assertSame('Main Page', markup::decode_entities_in_name('Main&nbsp;Page'));
+			$this->assertSame("A\u{2014}B", markup::decode_entities_in_name('A&mdash;B'));
+		}
+		finally
+		{
+			markup::set_active(false);
+		}
+	}
 }

wiki/markup.php

@@ -26,6 +26,65 @@ class markup
 	/** @var bool Whether the text parser should read wiki markup right now */
 	protected static $active = false;
 
+	/** @var bool Whether, with wiki markup on, entities such as   become characters */
+	protected static $entities = true;
+
+	/**
+	 * Say whether the next parse should turn HTML entities into the characters they
+	 * stand for. It should for text as written in an article; template text has
+	 * already been decoded once on its way to the parser, so it should not.
+	 *
+	 * @param bool $entities
+	 * @return void
+	 */
+	public static function set_entities($entities)
+	{
+		self::$entities = (bool) $entities;
+	}
+
+	/**
+	 * @return bool Whether entities become characters while wiki markup is on
+	 */
+	public static function entities()
+	{
+		return self::$entities;
+	}
+
+	/**
+	 * Attribute filter for text the parser captures in one piece, such as a heading or
+	 * a link's text, which the entity plugin never sees: turns the entities in it into
+	 * their characters, at the same times the plugin does. As in the plugin, an unknown
+	 * entity, or one for a control character, is left as written.
+	 *
+	 * @param string $value
+	 * @return string
+	 */
+	public static function decode_entities($value)
+	{
+		if (!self::$active || !self::$entities)
+		{
+			return $value;
+		}
+
+		return preg_replace_callback('/&(?:[a-z][a-z0-9]*|#[0-9]+|#x[0-9a-f]+);/i', function ($match) {
+			$char = html_entity_decode($match[0], ENT_HTML5 | ENT_QUOTES, 'UTF-8');
+
+			return ($char === $match[0] || ord($char) < 32) ? $match[0] : $char;
+		}, $value);
+	}
+
+	/**
+	 * decode_entities() for an article's name, where a non-breaking space counts as a
+	 * space, as in MediaWiki's page names.
+	 *
+	 * @param string $value
+	 * @return string
+	 */
+	public static function decode_entities_in_name($value)
+	{
+		return str_replace("\u{00A0}", ' ', self::decode_entities($value));
+	}
+
 	/**
 	 * Switch wiki markup on for the next parse (an article save or preview), or off.
 	 *

wiki/templates.php

@@ -878,14 +878,18 @@ public function page_view($article_text, $bbcode_uid)
 			$uid = $bitfield = '';
 			$flags = 0;
 			$was_active = markup::is_active();
+			$was_entities = markup::entities();
 			markup::set_active(true);
+			// Already decoded once on the way here; entities must not be decoded again
+			markup::set_entities(false);
 			try
 			{
 				generate_text_for_storage($documentation, $uid, $bitfield, $flags, true, true, true);
 			}
 			finally
 			{
 				markup::set_active($was_active);
+				markup::set_entities($was_entities);
 			}
 			$html = generate_text_for_display($documentation, $uid, $bitfield, $flags);
 		}
@@ -1029,14 +1033,18 @@ protected function render($text)
 		$text = $open . $text . $close;
 
 		$was_active = markup::is_active();
+		$was_entities = markup::entities();
 		markup::set_active(true);
+		// Template text was decoded once on its way here; entities must not be decoded again
+		markup::set_entities(false);
 		try
 		{
 			generate_text_for_storage($text, $uid, $bitfield, $flags, true, true, true);
 		}
 		finally
 		{
 			markup::set_active($was_active);
+			markup::set_entities($was_entities);
 		}
 
 		// While this text is rendered, what it still holds is part of the article being shown