From 4c0583c91f7ca458c3e0bdee17657c05abe53ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Sun, 18 Jun 2017 18:15:04 +0200 Subject: [PATCH] Refactored tokenizer to lazily evaluate file locations. --- lib/pddlparse/include/pddlparse/Context.h | 2 +- lib/pddlparse/include/pddlparse/Exception.h | 35 ++-- .../include/pddlparse/detail/parsing/Action.h | 8 +- .../pddlparse/detail/parsing/Description.h | 4 +- .../include/pddlparse/detail/parsing/Domain.h | 10 +- .../pddlparse/detail/parsing/Expressions.h | 12 +- .../pddlparse/detail/parsing/Problem.h | 10 +- .../src/pddlparse/detail/Requirements.cpp | 8 +- .../src/pddlparse/detail/parsing/Action.cpp | 24 +-- .../src/pddlparse/detail/parsing/Constant.cpp | 2 +- .../pddlparse/detail/parsing/Description.cpp | 18 +-- .../src/pddlparse/detail/parsing/Domain.cpp | 30 ++-- .../src/pddlparse/detail/parsing/Effect.cpp | 4 +- .../pddlparse/detail/parsing/InitialState.cpp | 2 +- .../pddlparse/detail/parsing/Precondition.cpp | 2 +- .../pddlparse/detail/parsing/Predicate.cpp | 2 +- .../detail/parsing/PrimitiveType.cpp | 6 +- .../src/pddlparse/detail/parsing/Problem.cpp | 36 ++--- .../pddlparse/detail/parsing/Requirement.cpp | 2 +- .../src/pddlparse/detail/parsing/Type.cpp | 2 +- .../pddlparse/detail/parsing/Unsupported.cpp | 2 +- .../src/pddlparse/detail/parsing/Variable.cpp | 2 +- lib/tokenize/include/tokenize/Location.h | 41 ++++- lib/tokenize/include/tokenize/Stream.h | 26 +-- .../include/tokenize/StreamPosition.h | 23 +++ lib/tokenize/include/tokenize/Tokenizer.h | 10 +- .../include/tokenize/TokenizerException.h | 8 +- lib/tokenize/src/tokenize/Location.cpp | 150 ++++++++++++++++++ lib/tokenize/src/tokenize/Stream.cpp | 67 +------- lib/tokenize/tests/TestTokenizer.cpp | 132 ++++++++------- 30 files changed, 420 insertions(+), 260 deletions(-) create mode 100644 lib/tokenize/include/tokenize/StreamPosition.h create mode 100644 lib/tokenize/src/tokenize/Location.cpp diff --git a/lib/pddlparse/include/pddlparse/Context.h b/lib/pddlparse/include/pddlparse/Context.h index a4cf884..b08139e 100644 --- a/lib/pddlparse/include/pddlparse/Context.h +++ b/lib/pddlparse/include/pddlparse/Context.h @@ -23,7 +23,7 @@ struct Context } // TODO: replace std::string with char * - using WarningCallback = std::function; + using WarningCallback = std::function; Context() = default; ~Context() = default; diff --git a/lib/pddlparse/include/pddlparse/Exception.h b/lib/pddlparse/include/pddlparse/Exception.h index e8a04df..07555cd 100644 --- a/lib/pddlparse/include/pddlparse/Exception.h +++ b/lib/pddlparse/include/pddlparse/Exception.h @@ -2,9 +2,10 @@ #define __PDDL_PARSE__EXCEPTION_H #include +#include #include -#include +#include namespace pddl { @@ -18,48 +19,45 @@ namespace pddl class Exception: public std::exception { public: - explicit Exception() + Exception() : Exception("unspecified parser error") { } - explicit Exception(const char *message) + Exception(const char *message) : Exception(static_cast(message)) { } - explicit Exception(const std::string &message) + Exception(const std::string &message) : m_message{message} { } - explicit Exception(const tokenize::Location &location) - : Exception(location, "unspecified parser error") + Exception(tokenize::Location &&location) + : Exception(std::forward(location), "unspecified parser error") { } - explicit Exception(const tokenize::Location &location, const char *message) - : Exception(location, static_cast(message)) + Exception(tokenize::Location &&location, const char *message) + : Exception(std::forward(location), static_cast(message)) { } - explicit Exception(const tokenize::Location &location, const std::string &message) - : m_location{location}, - m_message{message}, - // TODO: refactor - m_plainMessage{std::string(m_location.sectionStart) + ":" + std::to_string(m_location.rowStart) - + ":" + std::to_string(m_location.columnStart) + " " + m_message} + Exception(tokenize::Location &&location, const std::string &message) + : m_location{std::move(location)}, + m_message{message} { } ~Exception() noexcept = default; - const char *what() const throw() + const char *what() const noexcept { - return m_plainMessage.c_str(); + return m_message.c_str(); } - const tokenize::Location &location() const + const std::experimental::optional &location() const { return m_location; } @@ -70,9 +68,8 @@ class Exception: public std::exception } private: - tokenize::Location m_location; + std::experimental::optional m_location; std::string m_message; - std::string m_plainMessage; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/pddlparse/include/pddlparse/detail/parsing/Action.h b/lib/pddlparse/include/pddlparse/detail/parsing/Action.h index a26382e..19a1d2c 100644 --- a/lib/pddlparse/include/pddlparse/detail/parsing/Action.h +++ b/lib/pddlparse/include/pddlparse/detail/parsing/Action.h @@ -34,12 +34,12 @@ class ActionParser Context &m_context; ast::Domain &m_domain; - tokenize::Stream::Position m_parametersPosition; - tokenize::Stream::Position m_preconditionPosition; - tokenize::Stream::Position m_effectPosition; + tokenize::StreamPosition m_parametersPosition; + tokenize::StreamPosition m_preconditionPosition; + tokenize::StreamPosition m_effectPosition; // For compatibility with old PDDL versions - tokenize::Stream::Position m_varsPosition; + tokenize::StreamPosition m_varsPosition; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/pddlparse/include/pddlparse/detail/parsing/Description.h b/lib/pddlparse/include/pddlparse/detail/parsing/Description.h index 9ca20aa..14a71f3 100644 --- a/lib/pddlparse/include/pddlparse/detail/parsing/Description.h +++ b/lib/pddlparse/include/pddlparse/detail/parsing/Description.h @@ -27,8 +27,8 @@ class DescriptionParser void findSections(); Context &m_context; - tokenize::Stream::Position m_domainPosition; - tokenize::Stream::Position m_problemPosition; + tokenize::StreamPosition m_domainPosition; + tokenize::StreamPosition m_problemPosition; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/pddlparse/include/pddlparse/detail/parsing/Domain.h b/lib/pddlparse/include/pddlparse/detail/parsing/Domain.h index e86e62b..eadb319 100644 --- a/lib/pddlparse/include/pddlparse/detail/parsing/Domain.h +++ b/lib/pddlparse/include/pddlparse/detail/parsing/Domain.h @@ -35,11 +35,11 @@ class DomainParser Context &m_context; - tokenize::Stream::Position m_requirementsPosition; - tokenize::Stream::Position m_typesPosition; - tokenize::Stream::Position m_constantsPosition; - tokenize::Stream::Position m_predicatesPosition; - std::vector m_actionPositions; + tokenize::StreamPosition m_requirementsPosition; + tokenize::StreamPosition m_typesPosition; + tokenize::StreamPosition m_constantsPosition; + tokenize::StreamPosition m_predicatesPosition; + std::vector m_actionPositions; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/pddlparse/include/pddlparse/detail/parsing/Expressions.h b/lib/pddlparse/include/pddlparse/detail/parsing/Expressions.h index efbfb5a..4260263 100644 --- a/lib/pddlparse/include/pddlparse/detail/parsing/Expressions.h +++ b/lib/pddlparse/include/pddlparse/detail/parsing/Expressions.h @@ -64,12 +64,12 @@ std::experimental::optional> parseBinary(Context &conte auto argumentLeft = parseArgumentLeft(context, astContext, variableStack); if (!argumentLeft) - throw ParserException(tokenizer.location(), "could not parse argument of “" + std::string(Derived::Identifier) + "” expression"); + throw ParserException(tokenizer, "could not parse argument of “" + std::string(Derived::Identifier) + "” expression"); auto argumentRight = parseArgumentRight(context, astContext, variableStack); if (!argumentRight) - throw ParserException(tokenizer.location(), "could not parse argument of “" + std::string(Derived::Identifier) + "” expression"); + throw ParserException(tokenizer, "could not parse argument of “" + std::string(Derived::Identifier) + "” expression"); tokenizer.expect(")"); @@ -103,7 +103,7 @@ std::experimental::optional> parseNAry(Context &context auto argument = parseArgument(context, astContext, variableStack); if (!argument) - throw ParserException(tokenizer.location(), "could not parse argument of “" + std::string(Derived::Identifier) + "” expression"); + throw ParserException(tokenizer, "could not parse argument of “" + std::string(Derived::Identifier) + "” expression"); arguments.emplace_back(std::move(argument.value())); @@ -111,7 +111,7 @@ std::experimental::optional> parseNAry(Context &context } if (arguments.empty()) - context.warningCallback(tokenizer.location(), "“" + std::string(Derived::Identifier) + "” expressions should not be empty"); + context.warningCallback(tokenizer, "“" + std::string(Derived::Identifier) + "” expressions should not be empty"); tokenizer.expect(")"); @@ -147,7 +147,7 @@ std::experimental::optional> parseQuantified(Context &c auto argument = parseArgument(context, astContext, variableStack); if (!argument) - throw ParserException(tokenizer.location(), "could not parse argument of “" + std::string(Derived::Identifier) + "” expression"); + throw ParserException(tokenizer, "could not parse argument of “" + std::string(Derived::Identifier) + "” expression"); // Clean up variable stack variableStack.pop(); @@ -221,7 +221,7 @@ std::experimental::optional> parseNot(Context &context auto argument = parseArgument(context, astContext, variableStack); if (!argument) - throw ParserException(tokenizer.location(), "could not parse argument of “not” expression"); + throw ParserException(tokenizer, "could not parse argument of “not” expression"); tokenizer.expect(")"); diff --git a/lib/pddlparse/include/pddlparse/detail/parsing/Problem.h b/lib/pddlparse/include/pddlparse/detail/parsing/Problem.h index 110c241..7c250d9 100644 --- a/lib/pddlparse/include/pddlparse/detail/parsing/Problem.h +++ b/lib/pddlparse/include/pddlparse/detail/parsing/Problem.h @@ -36,11 +36,11 @@ class ProblemParser Context &m_context; ast::Domain &m_domain; - tokenize::Stream::Position m_domainPosition; - tokenize::Stream::Position m_requirementsPosition; - tokenize::Stream::Position m_objectsPosition; - tokenize::Stream::Position m_initialStatePosition; - tokenize::Stream::Position m_goalPosition; + tokenize::StreamPosition m_domainPosition; + tokenize::StreamPosition m_requirementsPosition; + tokenize::StreamPosition m_objectsPosition; + tokenize::StreamPosition m_initialStatePosition; + tokenize::StreamPosition m_goalPosition; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/pddlparse/src/pddlparse/detail/Requirements.cpp b/lib/pddlparse/src/pddlparse/detail/Requirements.cpp index 123a222..cfe79a8 100644 --- a/lib/pddlparse/src/pddlparse/detail/Requirements.cpp +++ b/lib/pddlparse/src/pddlparse/detail/Requirements.cpp @@ -61,9 +61,9 @@ void checkRequirement(ast::Domain &domain, ast::Requirement requirement, Context return; if (context.mode == Mode::Compatibility) - context.warningCallback(context.tokenizer.location(), "requirement “" + std::string(toString(requirement)) + "” used but never declared, silently adding requirement"); + context.warningCallback(context.tokenizer, "requirement “" + std::string(toString(requirement)) + "” used but never declared, silently adding requirement"); else - throw ParserException(context.tokenizer.location(), "requirement “" + std::string(toString(requirement)) + "” used but never declared"); + throw ParserException(context.tokenizer, "requirement “" + std::string(toString(requirement)) + "” used but never declared"); domain.requirements.push_back(requirement); } @@ -76,9 +76,9 @@ void checkRequirement(ast::Problem &problem, ast::Requirement requirement, Conte return; if (context.mode == Mode::Compatibility) - context.warningCallback(context.tokenizer.location(), "requirement “" + std::string(toString(requirement)) + "” used but never declared, silently adding requirement"); + context.warningCallback(context.tokenizer, "requirement “" + std::string(toString(requirement)) + "” used but never declared, silently adding requirement"); else - throw ParserException(context.tokenizer.location(), "requirement “" + std::string(toString(requirement)) + "” used but never declared"); + throw ParserException(context.tokenizer, "requirement “" + std::string(toString(requirement)) + "” used but never declared"); problem.requirements.push_back(requirement); } diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Action.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Action.cpp index 1dfe499..fac0b22 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Action.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Action.cpp @@ -23,10 +23,10 @@ namespace detail ActionParser::ActionParser(Context &context, ast::Domain &domain) : m_context{context}, m_domain{domain}, - m_parametersPosition{tokenize::Stream::InvalidPosition}, - m_preconditionPosition{tokenize::Stream::InvalidPosition}, - m_effectPosition{tokenize::Stream::InvalidPosition}, - m_varsPosition{tokenize::Stream::InvalidPosition} + m_parametersPosition{tokenize::InvalidStreamPosition}, + m_preconditionPosition{tokenize::InvalidStreamPosition}, + m_effectPosition{tokenize::InvalidStreamPosition}, + m_varsPosition{tokenize::InvalidStreamPosition} { } @@ -40,26 +40,26 @@ ast::ActionPointer ActionParser::parse() auto &tokenizer = m_context.tokenizer; - if (m_parametersPosition != tokenize::Stream::InvalidPosition) + if (m_parametersPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_parametersPosition); parseParameterSection(*action); } // For compatibility with old PDDL versions, vars sections are parsed in addition to parameters - if (m_varsPosition != tokenize::Stream::InvalidPosition) + if (m_varsPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_varsPosition); parseVarsSection(*action); } - if (m_preconditionPosition != tokenize::Stream::InvalidPosition) + if (m_preconditionPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_preconditionPosition); parsePreconditionSection(*action); } - if (m_effectPosition != tokenize::Stream::InvalidPosition) + if (m_effectPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_effectPosition); parseEffectSection(*action); @@ -84,10 +84,10 @@ void ActionParser::findSections(ast::Action &action) const auto setSectionPosition = [&](const std::string §ionName, auto §ionPosition, const auto value, bool unique = false) { - if (unique && sectionPosition != tokenize::Stream::InvalidPosition) + if (unique && sectionPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(value); - throw ParserException(tokenizer.location(), "only one “:" + sectionName + "” section allowed"); + throw ParserException(tokenizer, "only one “:" + sectionName + "” section allowed"); } sectionPosition = value; @@ -114,7 +114,7 @@ void ActionParser::findSections(ast::Action &action) const auto sectionIdentifier = tokenizer.getIdentifier(); tokenizer.seek(position); - throw ParserException(tokenizer.location(), "unknown action section “" + sectionIdentifier + "”"); + throw ParserException(tokenizer, "unknown action section “" + sectionIdentifier + "”"); } tokenizer.expect("("); @@ -181,7 +181,7 @@ void ActionParser::parseVarsSection(ast::Action &action) tokenizer.expect(":vars"); tokenizer.expect("("); - m_context.warningCallback(tokenizer.location(), "“vars” section is not part of the PDDL 3.1 specification, treating it like additional “parameters” section"); + m_context.warningCallback(tokenizer, "“vars” section is not part of the PDDL 3.1 specification, treating it like additional “parameters” section"); parseAndAddVariableDeclarations(m_context, m_domain, action.parameters); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Constant.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Constant.cpp index 76e30c0..df3bd16 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Constant.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Constant.cpp @@ -73,7 +73,7 @@ ast::ConstantPointer parseConstant(Context &context, ASTContext &astContext) auto constant = findConstant(constantName, astContext); if (!constant) - throw ParserException(tokenizer.location(), "undeclared constant “" + constantName + "”"); + throw ParserException(tokenizer, "undeclared constant “" + constantName + "”"); return std::move(constant.value()); } diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Description.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Description.cpp index 7da8cf1..b2929f0 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Description.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Description.cpp @@ -19,8 +19,8 @@ namespace detail DescriptionParser::DescriptionParser(Context &context) : m_context{context}, - m_domainPosition{tokenize::Stream::InvalidPosition}, - m_problemPosition{tokenize::Stream::InvalidPosition} + m_domainPosition{tokenize::InvalidStreamPosition}, + m_problemPosition{tokenize::InvalidStreamPosition} { } @@ -33,7 +33,7 @@ ast::Description DescriptionParser::parse() findSections(); - if (m_domainPosition == tokenize::Stream::InvalidPosition) + if (m_domainPosition == tokenize::InvalidStreamPosition) throw ParserException("no PDDL domain specified"); tokenizer.seek(m_domainPosition); @@ -41,7 +41,7 @@ ast::Description DescriptionParser::parse() auto domain = DomainParser(m_context).parse(); // If no problem is given, return just the domain - if (m_problemPosition == tokenize::Stream::InvalidPosition) + if (m_problemPosition == tokenize::InvalidStreamPosition) return {std::move(domain), std::experimental::nullopt}; tokenizer.seek(m_problemPosition); @@ -73,7 +73,7 @@ void DescriptionParser::findSections() if (m_context.mode == Mode::Compatibility && tokenizer.testAndReturn("in-package")) { - m_context.warningCallback(tokenizer.location(), "“in-package” section is not part of the PDDL 3.1 specification, ignoring section"); + m_context.warningCallback(tokenizer, "“in-package” section is not part of the PDDL 3.1 specification, ignoring section"); skipSection(tokenizer); tokenizer.skipWhiteSpace(); @@ -86,8 +86,8 @@ void DescriptionParser::findSections() if (tokenizer.testAndSkip("domain")) { - if (m_domainPosition != tokenize::Stream::InvalidPosition) - throw ParserException(tokenizer.location(), "PDDL description may not contain two domains"); + if (m_domainPosition != tokenize::InvalidStreamPosition) + throw ParserException(tokenizer, "PDDL description may not contain two domains"); m_domainPosition = position; skipSection(tokenizer); @@ -95,7 +95,7 @@ void DescriptionParser::findSections() } else if (m_context.tokenizer.testAndSkip("problem")) { - if (m_problemPosition != tokenize::Stream::InvalidPosition) + if (m_problemPosition != tokenize::InvalidStreamPosition) throw ParserException("PDDL description may not contain two problems currently"); m_problemPosition = position; @@ -105,7 +105,7 @@ void DescriptionParser::findSections() else { const auto sectionIdentifier = tokenizer.get(); - throw ParserException(tokenizer.location(), "unknown PDDL section “" + sectionIdentifier + "”"); + throw ParserException(tokenizer, "unknown PDDL section “" + sectionIdentifier + "”"); } tokenizer.skipWhiteSpace(); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Domain.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Domain.cpp index e2e5ca3..00bee40 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Domain.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Domain.cpp @@ -22,10 +22,10 @@ namespace detail DomainParser::DomainParser(Context &context) : m_context{context}, - m_requirementsPosition{tokenize::Stream::InvalidPosition}, - m_typesPosition{tokenize::Stream::InvalidPosition}, - m_constantsPosition{tokenize::Stream::InvalidPosition}, - m_predicatesPosition{tokenize::Stream::InvalidPosition} + m_requirementsPosition{tokenize::InvalidStreamPosition}, + m_typesPosition{tokenize::InvalidStreamPosition}, + m_constantsPosition{tokenize::InvalidStreamPosition}, + m_predicatesPosition{tokenize::InvalidStreamPosition} { } @@ -39,32 +39,32 @@ ast::DomainPointer DomainParser::parse() auto &tokenizer = m_context.tokenizer; - if (m_requirementsPosition != tokenize::Stream::InvalidPosition) + if (m_requirementsPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_requirementsPosition); parseRequirementSection(*domain); } - if (m_typesPosition != tokenize::Stream::InvalidPosition) + if (m_typesPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_typesPosition); parseTypeSection(*domain); } - if (m_constantsPosition != tokenize::Stream::InvalidPosition) + if (m_constantsPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_constantsPosition); parseConstantSection(*domain); } - if (m_predicatesPosition != tokenize::Stream::InvalidPosition) + if (m_predicatesPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_predicatesPosition); parsePredicateSection(*domain); } for (size_t i = 0; i < m_actionPositions.size(); i++) - if (m_actionPositions[i] != tokenize::Stream::InvalidPosition) + if (m_actionPositions[i] != tokenize::InvalidStreamPosition) { tokenizer.seek(m_actionPositions[i]); parseActionSection(*domain); @@ -93,10 +93,10 @@ void DomainParser::findSections(ast::Domain &domain) const auto setSectionPosition = [&](const std::string §ionName, auto §ionPosition, const auto value, bool unique = false) { - if (unique && sectionPosition != tokenize::Stream::InvalidPosition) + if (unique && sectionPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(value); - throw ParserException(tokenizer.location(), "only one “:" + sectionName + "” section allowed"); + throw ParserException(tokenizer, "only one “:" + sectionName + "” section allowed"); } sectionPosition = value; @@ -125,7 +125,7 @@ void DomainParser::findSections(ast::Domain &domain) setSectionPosition("predicates", m_predicatesPosition, position, true); else if (tokenizer.testIdentifierAndSkip("action")) { - m_actionPositions.emplace_back(tokenize::Stream::InvalidPosition); + m_actionPositions.emplace_back(tokenize::InvalidStreamPosition); setSectionPosition("action", m_actionPositions.back(), position); } else if (tokenizer.testIdentifierAndSkip("functions") @@ -137,7 +137,7 @@ void DomainParser::findSections(ast::Domain &domain) const auto sectionIdentifier = tokenizer.getIdentifier(); - m_context.warningCallback(tokenizer.location(), "section type “" + sectionIdentifier + "” currently unsupported, ignoring section"); + m_context.warningCallback(tokenizer, "section type “" + sectionIdentifier + "” currently unsupported, ignoring section"); tokenizer.seek(sectionIdentifierPosition); } @@ -146,7 +146,7 @@ void DomainParser::findSections(ast::Domain &domain) const auto sectionIdentifier = tokenizer.getIdentifier(); tokenizer.seek(position); - throw ParserException(tokenizer.location(), "unknown domain section “" + sectionIdentifier + "”"); + throw ParserException(tokenizer, "unknown domain section “" + sectionIdentifier + "”"); } // Skip section for now and parse it later @@ -246,7 +246,7 @@ void DomainParser::parseTypeSection(ast::Domain &domain) while (tokenizer.currentCharacter() != ')') { if (tokenizer.currentCharacter() == '(') - throw ParserException(tokenizer.location(), "only primitive types are allowed in type section"); + throw ParserException(tokenizer, "only primitive types are allowed in type section"); parseAndAddPrimitiveTypeDeclarations(m_context, domain); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Effect.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Effect.cpp index 6b141cd..86e210a 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Effect.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Effect.cpp @@ -83,7 +83,7 @@ std::experimental::optional parseEffectBody(Context &context, ASTCo const auto expressionIdentifier = tokenizer.getIdentifier(); tokenizer.seek(position); - throw ParserException(tokenizer.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in effect body"); + throw ParserException(tokenizer, "expression type “" + expressionIdentifier + "” unknown or not allowed in effect body"); } //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -144,7 +144,7 @@ std::experimental::optional parseConditionalEffectBody(C const auto expressionIdentifier = tokenizer.getIdentifier(); tokenizer.seek(position); - throw ParserException(tokenizer.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in conditional effect body"); + throw ParserException(tokenizer, "expression type “" + expressionIdentifier + "” unknown or not allowed in conditional effect body"); } //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/InitialState.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/InitialState.cpp index 9f3d495..030842b 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/InitialState.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/InitialState.cpp @@ -28,7 +28,7 @@ ast::InitialState parseInitialState(Context &context, ASTContext &astContext, Va auto fact = parseFact(context, astContext, variableStack); if (!fact) - throw ParserException(tokenizer.location(), "invalid initial state fact"); + throw ParserException(tokenizer, "invalid initial state fact"); initialState.facts.emplace_back(std::move(fact.value())); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Precondition.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Precondition.cpp index 53f07f0..0940b73 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Precondition.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Precondition.cpp @@ -99,7 +99,7 @@ std::experimental::optional parsePreconditionBody(Context &co const auto expressionIdentifier = tokenizer.getIdentifier(); tokenizer.seek(position); - throw ParserException(tokenizer.location(), "expression type “" + expressionIdentifier + "” unknown or not allowed in precondition body"); + throw ParserException(tokenizer, "expression type “" + expressionIdentifier + "” unknown or not allowed in precondition body"); } //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Predicate.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Predicate.cpp index 73805ef..9720ba0 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Predicate.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Predicate.cpp @@ -78,7 +78,7 @@ std::experimental::optional parsePredicate(Context &conte { // TODO: enumerate candidates and why they are incompatible tokenizer.seek(previousPosition); - throw ParserException(tokenizer.location(), "no matching declaration found for predicate “" + name + "”"); + throw ParserException(tokenizer, "no matching declaration found for predicate “" + name + "”"); } auto *declaration = matchingPredicateDeclaration->get(); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/PrimitiveType.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/PrimitiveType.cpp index 872317c..5bdc006 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/PrimitiveType.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/PrimitiveType.cpp @@ -22,7 +22,7 @@ ast::PrimitiveTypePointer parsePrimitiveType(Context &context, ast::Domain &doma auto typeName = tokenizer.getIdentifier(); if (typeName.empty()) - throw ParserException(tokenizer.location(), "could not parse primitive type, expected identifier"); + throw ParserException(tokenizer, "could not parse primitive type, expected identifier"); auto matchingType = std::find_if(types.begin(), types.end(), [&](auto &primitiveTypeDeclaration) @@ -34,9 +34,9 @@ ast::PrimitiveTypePointer parsePrimitiveType(Context &context, ast::Domain &doma if (matchingType == types.end()) { if (context.mode != Mode::Compatibility) - throw ParserException(tokenizer.location(), "primitive type “" + typeName + "” used without or before declaration"); + throw ParserException(tokenizer, "primitive type “" + typeName + "” used without or before declaration"); - context.warningCallback(tokenizer.location(), "primitive type “" + typeName + "” used without or before declaration, silently adding declaration"); + context.warningCallback(tokenizer, "primitive type “" + typeName + "” used without or before declaration, silently adding declaration"); types.emplace_back(std::make_unique(std::move(typeName))); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Problem.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Problem.cpp index 84c7c28..59a0696 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Problem.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Problem.cpp @@ -22,11 +22,11 @@ namespace detail ProblemParser::ProblemParser(Context &context, ast::Domain &domain) : m_context{context}, m_domain{domain}, - m_domainPosition{tokenize::Stream::InvalidPosition}, - m_requirementsPosition{tokenize::Stream::InvalidPosition}, - m_objectsPosition{tokenize::Stream::InvalidPosition}, - m_initialStatePosition{tokenize::Stream::InvalidPosition}, - m_goalPosition{tokenize::Stream::InvalidPosition} + m_domainPosition{tokenize::InvalidStreamPosition}, + m_requirementsPosition{tokenize::InvalidStreamPosition}, + m_objectsPosition{tokenize::InvalidStreamPosition}, + m_initialStatePosition{tokenize::InvalidStreamPosition}, + m_goalPosition{tokenize::InvalidStreamPosition} { } @@ -40,32 +40,32 @@ ast::ProblemPointer ProblemParser::parse() auto &tokenizer = m_context.tokenizer; - if (m_domainPosition == tokenize::Stream::InvalidPosition) - throw ParserException(tokenizer.location(), "problem description does not specify a corresponding domain"); + if (m_domainPosition == tokenize::InvalidStreamPosition) + throw ParserException(tokenizer, "problem description does not specify a corresponding domain"); tokenizer.seek(m_domainPosition); parseDomainSection(*problem); - if (m_requirementsPosition != tokenize::Stream::InvalidPosition) + if (m_requirementsPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_requirementsPosition); parseRequirementSection(*problem); } - if (m_objectsPosition != tokenize::Stream::InvalidPosition) + if (m_objectsPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(m_objectsPosition); parseObjectSection(*problem); } - if (m_initialStatePosition == tokenize::Stream::InvalidPosition) - throw ParserException(tokenizer.location(), "problem description does not specify an initial state"); + if (m_initialStatePosition == tokenize::InvalidStreamPosition) + throw ParserException(tokenizer, "problem description does not specify an initial state"); tokenizer.seek(m_initialStatePosition); parseInitialStateSection(*problem); - if (m_goalPosition == tokenize::Stream::InvalidPosition) - throw ParserException(tokenizer.location(), "problem description does not specify a goal"); + if (m_goalPosition == tokenize::InvalidStreamPosition) + throw ParserException(tokenizer, "problem description does not specify a goal"); tokenizer.seek(m_goalPosition); parseGoalSection(*problem); @@ -91,10 +91,10 @@ void ProblemParser::findSections(ast::Problem &problem) const auto setSectionPosition = [&](const std::string §ionName, auto §ionPosition, const auto value, bool unique = false) { - if (unique && sectionPosition != tokenize::Stream::InvalidPosition) + if (unique && sectionPosition != tokenize::InvalidStreamPosition) { tokenizer.seek(value); - throw ParserException(tokenizer.location(), "only one “:" + sectionName + "” section allowed"); + throw ParserException(tokenizer, "only one “:" + sectionName + "” section allowed"); } sectionPosition = value; @@ -129,7 +129,7 @@ void ProblemParser::findSections(ast::Problem &problem) const auto sectionIdentifier = tokenizer.getIdentifier(); - m_context.warningCallback(tokenizer.location(), "section type “" + sectionIdentifier + "” currently unsupported, ignoring section"); + m_context.warningCallback(tokenizer, "section type “" + sectionIdentifier + "” currently unsupported, ignoring section"); tokenizer.seek(sectionIdentifierPosition); } @@ -138,7 +138,7 @@ void ProblemParser::findSections(ast::Problem &problem) const auto sectionIdentifier = tokenizer.getIdentifier(); tokenizer.seek(position); - throw ParserException(tokenizer.location(), "unknown problem section “" + sectionIdentifier + "”"); + throw ParserException(tokenizer, "unknown problem section “" + sectionIdentifier + "”"); } // Skip section for now and parse it later @@ -165,7 +165,7 @@ void ProblemParser::parseDomainSection(ast::Problem &problem) const auto domainName = tokenizer.getIdentifier(); if (problem.domain->name != domainName) - throw ParserException(tokenizer.location(), "domains do not match (“" + problem.domain->name + "” and “" + domainName + "”)"); + throw ParserException(tokenizer, "domains do not match (“" + problem.domain->name + "” and “" + domainName + "”)"); tokenizer.expect(")"); } diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Requirement.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Requirement.cpp index 2f82ca8..a8e5761 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Requirement.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Requirement.cpp @@ -67,7 +67,7 @@ std::experimental::optional parseRequirement(Context &context) return matchingRequirement->second; if (context.mode == Mode::Compatibility && (requirementName == "goal-utilities" || requirementName == "domain-axioms")) - context.warningCallback(tokenizer.location(), "“" + requirementName + "” requirement is not part of the PDDL 3.1 specification, ignoring requirement"); + context.warningCallback(tokenizer, "“" + requirementName + "” requirement is not part of the PDDL 3.1 specification, ignoring requirement"); return std::experimental::nullopt; } diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Type.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Type.cpp index 16f6998..91849ad 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Type.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Type.cpp @@ -37,7 +37,7 @@ ast::Type parseType(Context &context, ast::Domain &domain) auto eitherType = parseEither(context, astContext, variableStack, parsePrimitiveTypeWrapper); if (!eitherType) - throw ParserException(tokenizer.location(), "expected primitive type or “either” expression"); + throw ParserException(tokenizer, "expected primitive type or “either” expression"); return std::move(eitherType.value()); } diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Unsupported.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Unsupported.cpp index 6d9dc6e..e60fa99 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Unsupported.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Unsupported.cpp @@ -22,7 +22,7 @@ ast::UnsupportedPointer parseUnsupported(Context &context) auto expressionType = tokenizer.getIdentifier(); - context.warningCallback(tokenizer.location(), "expression type “" + expressionType + "” currently unsupported in this context, substituting it with placeholder"); + context.warningCallback(tokenizer, "expression type “" + expressionType + "” currently unsupported in this context, substituting it with placeholder"); skipSection(tokenizer); diff --git a/lib/pddlparse/src/pddlparse/detail/parsing/Variable.cpp b/lib/pddlparse/src/pddlparse/detail/parsing/Variable.cpp index 857a507..cac3509 100644 --- a/lib/pddlparse/src/pddlparse/detail/parsing/Variable.cpp +++ b/lib/pddlparse/src/pddlparse/detail/parsing/Variable.cpp @@ -64,7 +64,7 @@ ast::VariablePointer parseVariable(Context &context, VariableStack &variableStac auto variableDeclaration = variableStack.findVariableDeclaration(variableName); if (!variableDeclaration) - throw ParserException(tokenizer.location(), "undeclared variable “" + variableName + "”"); + throw ParserException(tokenizer, "undeclared variable “" + variableName + "”"); return std::make_unique(variableDeclaration.value()); } diff --git a/lib/tokenize/include/tokenize/Location.h b/lib/tokenize/include/tokenize/Location.h index 29fecd5..8c12cb8 100644 --- a/lib/tokenize/include/tokenize/Location.h +++ b/lib/tokenize/include/tokenize/Location.h @@ -1,7 +1,7 @@ #ifndef __TOKENIZE__LOCATION_H #define __TOKENIZE__LOCATION_H -#include +#include namespace tokenize { @@ -12,16 +12,41 @@ namespace tokenize // //////////////////////////////////////////////////////////////////////////////////////////////////// -struct Location +class Stream; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +class Location { - const char *sectionStart = nullptr; - const char *sectionEnd = nullptr; + public: + Location(Stream &stream); + Location(Stream &stream, StreamPosition position); - std::size_t rowStart = -1; - std::size_t rowEnd = -1; + const char *sectionStart() const; + const char *sectionEnd() const; - std::size_t columnStart = -1; - std::size_t columnEnd = -1; + StreamPosition rowStart() const; + StreamPosition rowEnd() const; + + StreamPosition columnStart() const; + StreamPosition columnEnd() const; + + private: + void initializeLazily() const; + + Stream &m_stream; + const StreamPosition m_position; + + mutable bool m_isInitialized{false}; + + mutable const char *m_sectionStart{nullptr}; + mutable const char *m_sectionEnd{nullptr}; + + mutable StreamPosition m_rowStart{InvalidStreamPosition}; + mutable StreamPosition m_rowEnd{InvalidStreamPosition}; + + mutable StreamPosition m_columnStart{InvalidStreamPosition}; + mutable StreamPosition m_columnEnd{InvalidStreamPosition}; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/tokenize/include/tokenize/Stream.h b/lib/tokenize/include/tokenize/Stream.h index 30944f1..f8baadf 100644 --- a/lib/tokenize/include/tokenize/Stream.h +++ b/lib/tokenize/include/tokenize/Stream.h @@ -9,6 +9,7 @@ #include #include +#include #include namespace tokenize @@ -23,12 +24,9 @@ namespace tokenize class Stream { public: - using Position = size_t; - static const Position InvalidPosition; - struct Delimiter { - Position position; + StreamPosition position; std::string sectionName; }; @@ -46,15 +44,17 @@ class Stream void read(const std::experimental::filesystem::path &path); void reset(); - void seek(Position position); - Position position() const; - Location location() const; + void seek(StreamPosition position); + StreamPosition position() const; - char currentCharacter() const + const std::vector &delimiters() const { - assert(m_position < m_stream.size()); + return m_delimiters; + } - // TODO: check if this should be secured by check() + char currentCharacter() + { + check(); return m_stream[m_position]; } @@ -69,15 +69,15 @@ class Stream return m_position >= m_stream.size(); } - void check() const + void check() { if (atEnd()) - throw TokenizerException(location(), "reading past end of file"); + throw TokenizerException(*this, "reading past end of file"); } protected: std::string m_stream; - mutable Position m_position; + mutable StreamPosition m_position{0}; std::vector m_delimiters; }; diff --git a/lib/tokenize/include/tokenize/StreamPosition.h b/lib/tokenize/include/tokenize/StreamPosition.h new file mode 100644 index 0000000..a7d84f2 --- /dev/null +++ b/lib/tokenize/include/tokenize/StreamPosition.h @@ -0,0 +1,23 @@ +#ifndef __TOKENIZE__STREAM_POSITION_H +#define __TOKENIZE__STREAM_POSITION_H + +#include +#include + +namespace tokenize +{ + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// StreamPosition +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +using StreamPosition = size_t; +static const StreamPosition InvalidStreamPosition{std::numeric_limits::max()}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} + +#endif diff --git a/lib/tokenize/include/tokenize/Tokenizer.h b/lib/tokenize/include/tokenize/Tokenizer.h index b24128d..eccba87 100644 --- a/lib/tokenize/include/tokenize/Tokenizer.h +++ b/lib/tokenize/include/tokenize/Tokenizer.h @@ -189,7 +189,7 @@ void Tokenizer::expect(const Type &expectedValue) std::stringstream message; message << "unexpected value, expected “" << expectedValue << "”"; - throw TokenizerException(location(), message.str()); + throw TokenizerException(*this, message.str()); } //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -208,7 +208,7 @@ std::string Tokenizer::getIdentifier() if (!TokenizerPolicy::isIdentifierCharacter(character)) { if (value.empty()) - throw TokenizerException(location(), "could not parse identifier"); + throw TokenizerException(*this, "could not parse identifier"); return value; } @@ -406,7 +406,7 @@ uint64_t Tokenizer::getIntegerBody() check(); if (!std::isdigit(currentCharacter())) - throw TokenizerException(location(), "could not read integer value"); + throw TokenizerException(*this, "could not read integer value"); uint64_t value = 0; @@ -448,7 +448,7 @@ uint64_t Tokenizer::getImpl(Tag) skipWhiteSpace(); if (currentCharacter() == '-') - throw TokenizerException(location(), "expected unsigned integer, got signed one"); + throw TokenizerException(*this, "expected unsigned integer, got signed one"); return getIntegerBody(); } @@ -482,7 +482,7 @@ bool Tokenizer::getImpl(Tag) if (testAndSkip('1')) return true; - throw TokenizerException(location(), "could not read Boolean value"); + throw TokenizerException(*this, "could not read Boolean value"); } //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/tokenize/include/tokenize/TokenizerException.h b/lib/tokenize/include/tokenize/TokenizerException.h index cdfe9e6..f4cb8df 100644 --- a/lib/tokenize/include/tokenize/TokenizerException.h +++ b/lib/tokenize/include/tokenize/TokenizerException.h @@ -30,10 +30,7 @@ class TokenizerException: public std::exception explicit TokenizerException(const Location &location, const std::string &message) : m_location{location}, - m_message{message}, - // TODO: refactor - m_plainMessage{std::string(m_location.sectionStart) + ":" + std::to_string(m_location.rowStart) - + ":" + std::to_string(m_location.columnStart) + " " + m_message} + m_message{message} { } @@ -41,7 +38,7 @@ class TokenizerException: public std::exception const char *what() const noexcept { - return m_plainMessage.c_str(); + return m_message.c_str(); } const Location &location() const @@ -57,7 +54,6 @@ class TokenizerException: public std::exception private: Location m_location; std::string m_message; - std::string m_plainMessage; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/lib/tokenize/src/tokenize/Location.cpp b/lib/tokenize/src/tokenize/Location.cpp new file mode 100644 index 0000000..c7e2c46 --- /dev/null +++ b/lib/tokenize/src/tokenize/Location.cpp @@ -0,0 +1,150 @@ +#include + +#include + +#include + +namespace tokenize +{ + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// Location +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +Location::Location(Stream &stream) +: m_stream{stream}, + m_position{stream.position()} +{ +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +Location::Location(Stream &stream, StreamPosition position) +: m_stream{stream}, + m_position{position} +{ +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +const char *Location::sectionStart() const +{ + if (!m_isInitialized) + initializeLazily(); + + return m_sectionStart; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +const char *Location::sectionEnd() const +{ + if (!m_isInitialized) + initializeLazily(); + + return m_sectionEnd; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +StreamPosition Location::rowStart() const +{ + if (!m_isInitialized) + initializeLazily(); + + return m_rowStart; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +StreamPosition Location::rowEnd() const +{ + if (!m_isInitialized) + initializeLazily(); + + return m_rowEnd; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +StreamPosition Location::columnStart() const +{ + if (!m_isInitialized) + initializeLazily(); + + return m_columnStart; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +StreamPosition Location::columnEnd() const +{ + if (!m_isInitialized) + initializeLazily(); + + return m_columnEnd; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +void Location::initializeLazily() const +{ + const auto previousPosition = m_stream.position(); + + const auto &delimiters = m_stream.delimiters(); + + // Find current section + auto currentFile = std::find_if(delimiters.crbegin(), delimiters.crend(), + [&](const auto &fileDelimiter) + { + return m_position >= fileDelimiter.position; + }); + + // If the tokenizer is at the end of the stream, still count from the beginning of the last section + if (currentFile == delimiters.crend()) + currentFile = delimiters.crbegin(); + + // Go back to beginning of section + m_stream.seek(currentFile->position); + + StreamPosition row{1}; + StreamPosition column{1}; + + // Compute the location character by character + while (true) + { + if (m_stream.atEnd()) + break; + else if (m_stream.position() >= m_position) + break; + + const auto character = m_stream.currentCharacter(); + + if (character == '\n') + { + row++; + column = 1; + } + else if (std::isblank(character) || std::isprint(character)) + column++; + + m_stream.advance(); + } + + m_sectionStart = currentFile->sectionName.c_str(); + m_sectionEnd = currentFile->sectionName.c_str(); + m_rowStart = row; + m_rowEnd = row; + m_columnStart = column; + m_columnEnd = column; + + m_isInitialized = true; + + m_stream.seek(previousPosition); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} diff --git a/lib/tokenize/src/tokenize/Stream.cpp b/lib/tokenize/src/tokenize/Stream.cpp index 9850882..e275247 100644 --- a/lib/tokenize/src/tokenize/Stream.cpp +++ b/lib/tokenize/src/tokenize/Stream.cpp @@ -1,6 +1,5 @@ #include -#include #include namespace tokenize @@ -12,12 +11,7 @@ namespace tokenize // //////////////////////////////////////////////////////////////////////////////////////////////////// -const Stream::Position Stream::InvalidPosition{std::numeric_limits::max()}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - Stream::Stream() -: m_position{0} { std::setlocale(LC_NUMERIC, "C"); } @@ -34,18 +28,16 @@ Stream::Stream(std::string streamName, std::istream &istream) void Stream::read(std::string streamName, std::istream &istream) { // Store position of new section - const auto position = m_stream.size(); + m_delimiters.push_back({m_stream.size(), streamName}); - m_delimiters.push_back({position, streamName}); - - istream.seekg(0, std::ios::end); + /*istream.seekg(0, std::ios::end); const auto streamSize = istream.tellg(); istream.seekg(0, std::ios::beg); const auto startPosition = m_stream.size(); - m_stream.resize(m_stream.size() + streamSize); - std::copy(std::istreambuf_iterator(istream), std::istreambuf_iterator(), m_stream.begin() + startPosition); + m_stream.resize(m_stream.size() + streamSize);*/ + std::copy(std::istreambuf_iterator(istream), std::istreambuf_iterator(), std::back_inserter(m_stream)); } //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -69,65 +61,18 @@ void Stream::reset() //////////////////////////////////////////////////////////////////////////////////////////////////// -void Stream::seek(Position position) +void Stream::seek(StreamPosition position) { m_position = position; } //////////////////////////////////////////////////////////////////////////////////////////////////// -typename Stream::Position Stream::position() const +StreamPosition Stream::position() const { return m_position; } //////////////////////////////////////////////////////////////////////////////////////////////////// -Location Stream::location() const -{ - const auto currentPosition = position(); - - // Find current section - auto currentFile = std::find_if(m_delimiters.crbegin(), m_delimiters.crend(), - [&](const auto &fileDelimiter) - { - return currentPosition >= fileDelimiter.position; - }); - - // If the tokenizer is at the end of the stream, still count from the beginning of the last section - if (currentFile == m_delimiters.crend()) - currentFile = m_delimiters.crbegin(); - - // Go back to beginning of section - m_position = currentFile->position; - - size_t row = 1; - size_t column = 1; - - // Compute the location character by character - while (true) - { - if (currentPosition >= m_stream.size() && atEnd()) - break; - else if (currentPosition < m_stream.size() && position() >= currentPosition) - break; - - const auto character = currentCharacter(); - - if (character == '\n') - { - row++; - column = 1; - } - else if (std::isblank(character) || std::isprint(character)) - column++; - - m_position++; - } - - return {currentFile->sectionName.c_str(), currentFile->sectionName.c_str(), row, row, column, column}; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - } diff --git a/lib/tokenize/tests/TestTokenizer.cpp b/lib/tokenize/tests/TestTokenizer.cpp index 0318426..3b0fe95 100644 --- a/lib/tokenize/tests/TestTokenizer.cpp +++ b/lib/tokenize/tests/TestTokenizer.cpp @@ -72,7 +72,7 @@ TEST_CASE("[tokenizer] While tokenizing, the cursor position is as expected", "[ std::stringstream s(" identifier 5 \n-51\t 0 1"); tokenize::Tokenizer<> p("input", s); - tokenize::Tokenizer<>::Position pos; + tokenize::StreamPosition pos; pos = p.position(); REQUIRE(p.testAndReturn("error") == false); @@ -181,86 +181,108 @@ TEST_CASE("[tokenizer] While tokenizing, the cursor location is as expcected", " const auto startPosition = p.position(); - tokenize::Location l; - - l = p.location(); - REQUIRE(l.rowStart == 1u); - REQUIRE(l.columnStart == 1u); - REQUIRE(p.currentCharacter() == '1'); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 1u); + REQUIRE(l.columnStart() == 1u); + REQUIRE(p.currentCharacter() == '1'); + } REQUIRE_NOTHROW(p.advance()); - l = p.location(); - REQUIRE(l.rowStart == 1u); - REQUIRE(l.columnStart == 2u); - REQUIRE(p.currentCharacter() == '2'); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 1u); + REQUIRE(l.columnStart() == 2u); + REQUIRE(p.currentCharacter() == '2'); + } REQUIRE_NOTHROW(p.advance()); - l = p.location(); - REQUIRE(l.rowStart == 1u); - REQUIRE(l.columnStart == 3u); - REQUIRE(p.currentCharacter() == '3'); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 1u); + REQUIRE(l.columnStart() == 3u); + REQUIRE(p.currentCharacter() == '3'); + } REQUIRE_NOTHROW(p.advance()); - l = p.location(); - REQUIRE(l.rowStart == 1u); - REQUIRE(l.columnStart == 4u); - REQUIRE(p.currentCharacter() == ' '); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 1u); + REQUIRE(l.columnStart() == 4u); + REQUIRE(p.currentCharacter() == ' '); + } REQUIRE_NOTHROW(p.advance()); - l = p.location(); - REQUIRE(l.rowStart == 1u); - REQUIRE(l.columnStart == 5u); - REQUIRE(p.currentCharacter() == '\n'); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 1u); + REQUIRE(l.columnStart() == 5u); + REQUIRE(p.currentCharacter() == '\n'); + } REQUIRE_NOTHROW(p.advance()); - l = p.location(); - REQUIRE(l.rowStart == 2u); - REQUIRE(l.columnStart == 1u); - REQUIRE(p.currentCharacter() == '4'); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 2u); + REQUIRE(l.columnStart() == 1u); + REQUIRE(p.currentCharacter() == '4'); + } REQUIRE_NOTHROW(p.advance()); REQUIRE_NOTHROW(p.expect("test1")); - l = p.location(); - REQUIRE(l.rowStart == 3u); - REQUIRE(l.columnStart == 6u); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 3u); + REQUIRE(l.columnStart() == 6u); + } REQUIRE_NOTHROW(p.expect("test2")); - l = p.location(); - REQUIRE(l.rowStart == 4u); - REQUIRE(l.columnStart == 7u); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 4u); + REQUIRE(l.columnStart() == 7u); + } REQUIRE_NOTHROW(p.expect("test3")); - l = p.location(); - REQUIRE(l.rowStart == 5u); - REQUIRE(l.columnStart == 6u); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 5u); + REQUIRE(l.columnStart() == 6u); + } REQUIRE_NOTHROW(p.skipLine()); - l = p.location(); - REQUIRE(l.rowStart == 6u); - REQUIRE(l.columnStart == 1u); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 6u); + REQUIRE(l.columnStart() == 1u); + } REQUIRE_NOTHROW(p.skipLine()); - l = p.location(); - REQUIRE(l.rowStart == 7u); - REQUIRE(l.columnStart == 1u); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 7u); + REQUIRE(l.columnStart() == 1u); + } REQUIRE_NOTHROW(p.skipWhiteSpace()); - l = p.location(); - REQUIRE(l.rowStart == 10u); - REQUIRE(l.columnStart == 1u); - REQUIRE(p.atEnd()); + { + auto l = tokenize::Location(p); + REQUIRE(l.rowStart() == 10u); + REQUIRE(l.columnStart() == 1u); + REQUIRE(p.atEnd()); + } p.reset(); REQUIRE(p.position() == startPosition); @@ -289,19 +311,21 @@ TEST_CASE("[tokenizer] Comments are correctly removed", "[tokenizer]") p1.removeComments(";", "\n", false); - tokenize::Location l; - REQUIRE_NOTHROW(p1.expect("test1")); - l = p1.location(); - REQUIRE(l.rowStart == 2u); - REQUIRE(l.columnStart == 6u); + { + auto l = tokenize::Location(p1); + REQUIRE(l.rowStart() == 2u); + REQUIRE(l.columnStart() == 6u); + } REQUIRE_NOTHROW(p1.expect("test2")); - l = p1.location(); - REQUIRE(l.rowStart == 3u); - REQUIRE(l.columnStart == 6u); + { + auto l = tokenize::Location(p1); + REQUIRE(l.rowStart() == 3u); + REQUIRE(l.columnStart() == 6u); + } p1.skipWhiteSpace();