diff --git a/include/xtrpg/xml/tokenizer/TokenizationError.hpp b/include/xtrpg/xml/tokenizer/TokenizationError.hpp new file mode 100644 index 0000000..bbe6fb6 --- /dev/null +++ b/include/xtrpg/xml/tokenizer/TokenizationError.hpp @@ -0,0 +1,23 @@ +#pragma once + +namespace xtrpg::xml::tokenizer { + +enum class TokenizationError { + /** + * Indicates that there is currently no error. + */ + NONE, + + /** + * Indicates that the internal buffer has exhausted it's maximum content size. + * This would be an indication that either a tagname, attribute name or + * attribute value are too excessive in length to safely processes. + */ + BUFFER_SIZE_EXHAUSTED, + + /** + * Indicates that the input contains an invalid XML token sequence. + */ + MALFORMED_INPUT +}; +} \ No newline at end of file diff --git a/include/xtrpg/xml/tokenizer/XmlStreamTokenizer.hpp b/include/xtrpg/xml/tokenizer/XmlStreamTokenizer.hpp new file mode 100644 index 0000000..70a9747 --- /dev/null +++ b/include/xtrpg/xml/tokenizer/XmlStreamTokenizer.hpp @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "xtrpg/xml/tokenizer/TokenizationError.hpp" +#include "xtrpg/xml/tokenizer/XmlTokenListener.hpp" + +#ifndef __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS +#define __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS 999999 +#endif + +namespace xtrpg::xml::tokenizer { +/** + * A processor class that processing a stream of XML data and fires off + * tokenization events to a registered listener. + */ +class XmlStreamTokenizer { +public: + /** + * Consumes the data on the provided stream until it's exhausted. + */ + void process(std::istream &stream); + + /** + * Defines a instance to act as the listener for this class. + */ + void setListener(std::shared_ptr listener) { + _listener = listener; + } + +private: + enum class State { + TEXT, + AFTER_OPEN, + START_TAG_NAME, + START_TAG_BODY, + ATTRIBUTE_NAME, + ATTRIBUTE_AFTER_NAME, + ATTRIBUTE_VALUE_START, + ATTRIBUTE_VALUE, + END_TAG_NAME, + END_TAG_BODY, + DECLARATION_NAME, + DECLARATION_BODY, + DECLARATION_ATTRIBUTE_NAME, + DECLARATION_ATTRIBUTE_AFTER_NAME, + DECLARATION_ATTRIBUTE_VALUE_START, + DECLARATION_ATTRIBUTE_VALUE, + DECLARATION_QUESTION, + SELF_CLOSING, + SPECIAL, + COMMENT, + CDATA + }; + + /** + * Unprocessed stream data that's carried over between process calls. + */ + std::string _buffer; + + std::string _attributeName; + std::string _specialPrefix; + State _state{State::TEXT}; + char _quote{'\0'}; + + TokenizationError _error{TokenizationError::NONE}; + + std::weak_ptr _listener; +}; + +} // namespace xtrpg::xml::tokenizer \ No newline at end of file diff --git a/include/xtrpg/xml/tokenizer/XmlTokenListener.hpp b/include/xtrpg/xml/tokenizer/XmlTokenListener.hpp new file mode 100644 index 0000000..1bd3938 --- /dev/null +++ b/include/xtrpg/xml/tokenizer/XmlTokenListener.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include + +#include "xtrpg/xml/tokenizer/TokenizationError.hpp" + +namespace xtrpg::xml::tokenizer { +/** + * Represents a class that is capable of processing a stream of XML Token + * events. + */ +class XmlTokenListener { +public: + virtual ~XmlTokenListener() = default; + + /** + * Indication to the listener instance that it should begin processing a new + * XML Tag Node, with the given tagname. If the listener already has an open + * tag then it should assign the existing tag to be the parent of this new + * tag. + */ + virtual void openTag(std::string_view tagname) = 0; + + /** + * Indication that the current tag has finished processing and focus should be + * returned to it's parent. + */ + virtual void closeTag() = 0; + + /** + * Indication to the listener that it should begin processing a new XML + * Declaration Tag Node, with the given tagname. + */ + virtual void openDeclaration(std::string_view tagname) = 0; + + /** + * Indication that the current declaration tag has finished processing and + * focus should be returned to it's parent. + */ + virtual void closeDeclaration() = 0; + + /** + * Defines an attribute (key/value pair) that should be assigned to the + * current tag or declaration tag. + */ + virtual void setAttribute(std::string_view name, std::string_view value) = 0; + + /** + * Defines a block of text that should be applied as a Raw Text Child Node of + * the current tag. + */ + virtual void appendText(std::string_view content) = 0; + + /** + * Defines an error state of the tokenizer. + */ + virtual void onError(TokenizationError error) = 0; +}; +} // namespace xtrpg::xml::tokenizer \ No newline at end of file diff --git a/src/xml/tokenizer/XmlStreamTokenizer.cpp b/src/xml/tokenizer/XmlStreamTokenizer.cpp new file mode 100644 index 0000000..157cea1 --- /dev/null +++ b/src/xml/tokenizer/XmlStreamTokenizer.cpp @@ -0,0 +1,385 @@ +#include "xtrpg/xml/tokenizer/XmlStreamTokenizer.hpp" + +#include +#include + +namespace { +bool isNameCharacter(const char character) { + return std::isalnum(static_cast(character)) != 0 || + character == '_' || character == '-' || character == ':' || + character == '.'; +} + +bool isWhitespace(const char character) { + return std::isspace(static_cast(character)) != 0; +} +} // namespace + +namespace xtrpg::xml::tokenizer { + +void XmlStreamTokenizer::process(std::istream &stream) { + // If the tokenizer is already in an error state then re-issue the same error. + if (TokenizationError::NONE != this->_error) { + if (const auto listener = this->_listener.lock()) { + listener->onError(this->_error); + } + return; + } + + const auto listener = this->_listener.lock(); + const auto fail = [&](const TokenizationError error) { + this->_error = error; + if (listener) { + listener->onError(error); + } + }; + const auto appendText = [&](const std::string_view text) { + if (!text.empty() && listener) { + listener->appendText(text); + } + }; + const auto openStartTag = [&]() { + if (this->_buffer.empty()) { + fail(TokenizationError::MALFORMED_INPUT); + return; + } + if (listener) { + listener->openTag(this->_buffer); + } + this->_buffer.clear(); + }; + const auto openDeclaration = [&]() { + if (this->_buffer.empty()) { + fail(TokenizationError::MALFORMED_INPUT); + return; + } + if (listener) { + listener->openDeclaration(this->_buffer); + } + this->_buffer.clear(); + }; + const auto bufferExceeded = [&]() { + fail(TokenizationError::BUFFER_SIZE_EXHAUSTED); + }; + + this->_buffer.reserve(__TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS); + this->_attributeName.reserve(128); + this->_specialPrefix.reserve(7); + + std::array input; + while (this->_error == TokenizationError::NONE && + (stream.read(input.data(), input.size()) || stream.gcount() > 0)) { + const auto count = stream.gcount(); + for (std::streamsize index = 0; + this->_error == TokenizationError::NONE && index < count; ++index) { + const char character = input[static_cast(index)]; + switch (this->_state) { + case State::TEXT: + if (character == '<') { + appendText(this->_buffer); + this->_buffer.clear(); + this->_state = State::AFTER_OPEN; + } else { + this->_buffer += character; + if (this->_buffer.size() >= __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + appendText(this->_buffer); + this->_buffer.clear(); + } + } + break; + case State::AFTER_OPEN: + if (character == '/') { + this->_buffer.clear(); + this->_state = State::END_TAG_NAME; + } else if (character == '?') { + this->_buffer.clear(); + this->_state = State::DECLARATION_NAME; + } else if (character == '!') { + this->_specialPrefix.clear(); + this->_state = State::SPECIAL; + } else if (isNameCharacter(character)) { + this->_buffer = character; + this->_state = State::START_TAG_NAME; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::START_TAG_NAME: + if (isNameCharacter(character)) { + this->_buffer += character; + if (this->_buffer.size() > __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + bufferExceeded(); + } + } else if (isWhitespace(character)) { + openStartTag(); + this->_state = State::START_TAG_BODY; + } else if (character == '>') { + openStartTag(); + this->_state = State::TEXT; + } else if (character == '/') { + openStartTag(); + this->_state = State::SELF_CLOSING; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::START_TAG_BODY: + if (isWhitespace(character)) { + break; + } + if (character == '>') { + this->_state = State::TEXT; + } else if (character == '/') { + this->_state = State::SELF_CLOSING; + } else if (isNameCharacter(character)) { + this->_buffer = character; + this->_state = State::ATTRIBUTE_NAME; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::ATTRIBUTE_NAME: + if (isNameCharacter(character)) { + this->_buffer += character; + if (this->_buffer.size() > __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + bufferExceeded(); + } + } else if (isWhitespace(character)) { + this->_attributeName = this->_buffer; + this->_buffer.clear(); + this->_state = State::ATTRIBUTE_AFTER_NAME; + } else if (character == '=') { + this->_attributeName = this->_buffer; + this->_buffer.clear(); + this->_state = State::ATTRIBUTE_VALUE_START; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::ATTRIBUTE_AFTER_NAME: + if (isWhitespace(character)) { + break; + } + if (character == '=') { + this->_state = State::ATTRIBUTE_VALUE_START; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::ATTRIBUTE_VALUE_START: + if (isWhitespace(character)) { + break; + } + if (character == '\'' || character == '"') { + this->_quote = character; + this->_buffer.clear(); + this->_state = State::ATTRIBUTE_VALUE; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::ATTRIBUTE_VALUE: + if (character == this->_quote) { + if (listener) { + listener->setAttribute(this->_attributeName, this->_buffer); + } + this->_attributeName.clear(); + this->_buffer.clear(); + this->_state = State::START_TAG_BODY; + } else { + this->_buffer += character; + if (this->_buffer.size() > __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + bufferExceeded(); + } + } + break; + case State::END_TAG_NAME: + if (isNameCharacter(character)) { + this->_buffer += character; + if (this->_buffer.size() > __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + bufferExceeded(); + } + } else if (isWhitespace(character)) { + this->_state = State::END_TAG_BODY; + } else if (character == '>') { + if (this->_buffer.empty()) { + fail(TokenizationError::MALFORMED_INPUT); + } else { + if (listener) { + listener->closeTag(); + } + this->_buffer.clear(); + this->_state = State::TEXT; + } + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::END_TAG_BODY: + if (isWhitespace(character)) { + break; + } + if (character == '>' && !this->_buffer.empty()) { + if (listener) { + listener->closeTag(); + } + this->_buffer.clear(); + this->_state = State::TEXT; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::DECLARATION_NAME: + if (isNameCharacter(character)) { + this->_buffer += character; + if (this->_buffer.size() > __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + bufferExceeded(); + } + } else if (isWhitespace(character)) { + openDeclaration(); + this->_state = State::DECLARATION_BODY; + } else if (character == '?') { + openDeclaration(); + this->_state = State::DECLARATION_QUESTION; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::DECLARATION_BODY: + if (isWhitespace(character)) { + break; + } + if (character == '?') { + this->_state = State::DECLARATION_QUESTION; + } else if (isNameCharacter(character)) { + this->_buffer = character; + this->_state = State::DECLARATION_ATTRIBUTE_NAME; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::DECLARATION_ATTRIBUTE_NAME: + if (isNameCharacter(character)) { + this->_buffer += character; + if (this->_buffer.size() > __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + bufferExceeded(); + } + } else if (isWhitespace(character)) { + this->_attributeName = this->_buffer; + this->_buffer.clear(); + this->_state = State::DECLARATION_ATTRIBUTE_AFTER_NAME; + } else if (character == '=') { + this->_attributeName = this->_buffer; + this->_buffer.clear(); + this->_state = State::DECLARATION_ATTRIBUTE_VALUE_START; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::DECLARATION_ATTRIBUTE_AFTER_NAME: + if (isWhitespace(character)) { + break; + } + if (character == '=') { + this->_state = State::DECLARATION_ATTRIBUTE_VALUE_START; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::DECLARATION_ATTRIBUTE_VALUE_START: + if (isWhitespace(character)) { + break; + } + if (character == '\'' || character == '"') { + this->_quote = character; + this->_buffer.clear(); + this->_state = State::DECLARATION_ATTRIBUTE_VALUE; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::DECLARATION_ATTRIBUTE_VALUE: + if (character == this->_quote) { + if (listener) { + listener->setAttribute(this->_attributeName, this->_buffer); + } + this->_attributeName.clear(); + this->_buffer.clear(); + this->_state = State::DECLARATION_BODY; + } else { + this->_buffer += character; + if (this->_buffer.size() > __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + bufferExceeded(); + } + } + break; + case State::DECLARATION_QUESTION: + if (character == '>') { + if (listener) { + listener->closeDeclaration(); + } + this->_state = State::TEXT; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::SELF_CLOSING: + if (character == '>') { + if (listener) { + listener->closeTag(); + } + this->_state = State::TEXT; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::SPECIAL: + this->_specialPrefix += character; + if (this->_specialPrefix == "--") { + this->_buffer.clear(); + this->_state = State::COMMENT; + } else if (this->_specialPrefix == "[CDATA[") { + this->_buffer.clear(); + this->_state = State::CDATA; + } else if (std::string_view("--").starts_with(this->_specialPrefix) || + std::string_view("[CDATA[").starts_with( + this->_specialPrefix)) { + break; + } else { + fail(TokenizationError::MALFORMED_INPUT); + } + break; + case State::COMMENT: + this->_buffer += character; + if (this->_buffer.size() >= 3 && this->_buffer.ends_with("-->")) { + this->_buffer.clear(); + this->_state = State::TEXT; + } else if (this->_buffer.size() > + __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + bufferExceeded(); + } + break; + case State::CDATA: + this->_buffer += character; + if (this->_buffer.size() >= 3 && this->_buffer.ends_with("]]>")) { + this->_buffer.resize(this->_buffer.size() - 3); + appendText(this->_buffer); + this->_buffer.clear(); + this->_state = State::TEXT; + } else if (this->_buffer.size() >= + __TOKENIZER_MAX_BUFFER_SIZE_IN_CHARS) { + const auto textSize = this->_buffer.size() - 2; + appendText(std::string_view(this->_buffer.data(), textSize)); + const char penultimate = this->_buffer[this->_buffer.size() - 2]; + const char last = this->_buffer[this->_buffer.size() - 1]; + this->_buffer.clear(); + this->_buffer += penultimate; + this->_buffer += last; + } + break; + } + } + } +} + +} // namespace xtrpg::xml::tokenizer \ No newline at end of file