feat: Add streaming XML tokenizer with error handling and listeners

include/xtrpg/xml/tokenizer/TokenizationError.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+namespace xtrpg::xml::tokenizer {
+
+enum class TokenizationError {
+  /**
+   * Indicates that there is currently no error.
+   */
+  NONE,
+
+  /**
+   * Indicates that the internal buffer has exhausted it's maximum content size.
+   * This would be an indication that either a tagname, attribute name or
+   * attribute value are too excessive in length to safely processes.
+   */
+  BUFFER_SIZE_EXHAUSTED,
+
+  /**
+   * Indicates that the input contains an invalid XML token sequence.
+   */
+  MALFORMED_INPUT
+};
+}

include/xtrpg/xml/tokenizer/XmlStreamTokenizer.hpp

@@ -0,0 +1,75 @@
+#pragma once
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "xtrpg/xml/tokenizer/TokenizationError.hpp"
+#include "xtrpg/xml/tokenizer/XmlTokenListener.hpp"
+
+#ifndef __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS
+#define __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS 999999
+#endif
+
+namespace xtrpg::xml::tokenizer {
+/**
+ * A processor class that processing a stream of XML data and fires off
+ * tokenization events to a registered listener.
+ */
+class XmlStreamTokenizer {
+public:
+  /**
+   * Consumes the data on the provided stream until it's exhausted.
+   */
+  void process(std::istream &stream);
+
+  /**
+   * Defines a instance to act as the listener for this class.
+   */
+  void setListener(std::shared_ptr<XmlTokenListener> listener) {
+    _listener = listener;
+  }
+
+private:
+  enum class State {
+    TEXT,
+    AFTER_OPEN,
+    START_TAG_NAME,
+    START_TAG_BODY,
+    ATTRIBUTE_NAME,
+    ATTRIBUTE_AFTER_NAME,
+    ATTRIBUTE_VALUE_START,
+    ATTRIBUTE_VALUE,
+    END_TAG_NAME,
+    END_TAG_BODY,
+    DECLARATION_NAME,
+    DECLARATION_BODY,
+    DECLARATION_ATTRIBUTE_NAME,
+    DECLARATION_ATTRIBUTE_AFTER_NAME,
+    DECLARATION_ATTRIBUTE_VALUE_START,
+    DECLARATION_ATTRIBUTE_VALUE,
+    DECLARATION_QUESTION,
+    SELF_CLOSING,
+    SPECIAL,
+    COMMENT,
+    CDATA
+  };
+
+  /**
+   * Unprocessed stream data that's carried over between process calls.
+   */
+  std::string _buffer;
+
+  std::string _attributeName;
+  std::string _specialPrefix;
+  State _state{State::TEXT};
+  char _quote{'\0'};
+
+  TokenizationError _error{TokenizationError::NONE};
+
+  std::weak_ptr<XmlTokenListener> _listener;
+};
+
+} // namespace xtrpg::xml::tokenizer

include/xtrpg/xml/tokenizer/XmlTokenListener.hpp

@@ -0,0 +1,59 @@
+#pragma once
+
+#include <string>
+
+#include "xtrpg/xml/tokenizer/TokenizationError.hpp"
+
+namespace xtrpg::xml::tokenizer {
+/**
+ * Represents a class that is capable of processing a stream of XML Token
+ * events.
+ */
+class XmlTokenListener {
+public:
+  virtual ~XmlTokenListener() = default;
+
+  /**
+   * Indication to the listener instance that it should begin processing a new
+   * XML Tag Node, with the given tagname. If the listener already has an open
+   * tag then it should assign the existing tag to be the parent of this new
+   * tag.
+   */
+  virtual void openTag(std::string_view tagname) = 0;
+
+  /**
+   * Indication that the current tag has finished processing and focus should be
+   * returned to it's parent.
+   */
+  virtual void closeTag() = 0;
+
+  /**
+   * Indication to the listener that it should begin processing a new XML
+   * Declaration Tag Node, with the given tagname.
+   */
+  virtual void openDeclaration(std::string_view tagname) = 0;
+
+  /**
+   * Indication that the current declaration tag has finished processing and
+   * focus should be returned to it's parent.
+   */
+  virtual void closeDeclaration() = 0;
+
+  /**
+   * Defines an attribute (key/value pair) that should be assigned to the
+   * current tag or declaration tag.
+   */
+  virtual void setAttribute(std::string_view name, std::string_view value) = 0;
+
+  /**
+   * Defines a block of text that should be applied as a Raw Text Child Node of
+   * the current tag.
+   */
+  virtual void appendText(std::string_view content) = 0;
+
+  /**
+   * Defines an error state of the tokenizer.
+   */
+  virtual void onError(TokenizationError error) = 0;
+};
+} // namespace xtrpg::xml::tokenizer