Refactored tokenizer to lazily evaluate file locations.

lib/tokenize/include/tokenize/Location.h

@@ -1,7 +1,7 @@
 #ifndef __TOKENIZE__LOCATION_H
 #define __TOKENIZE__LOCATION_H
 
-#include <cstdlib>
+#include <tokenize/StreamPosition.h>
 
 namespace tokenize
 {
@@ -12,16 +12,41 @@ namespace tokenize
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-struct Location
+class Stream;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class Location
 {
-	const char *sectionStart = nullptr;
-	const char *sectionEnd = nullptr;
+	public:
+		Location(Stream &stream);
+		Location(Stream &stream, StreamPosition position);
 
-	std::size_t rowStart = -1;
-	std::size_t rowEnd = -1;
+		const char *sectionStart() const;
+		const char *sectionEnd() const;
 
-	std::size_t columnStart = -1;
-	std::size_t columnEnd = -1;
+		StreamPosition rowStart() const;
+		StreamPosition rowEnd() const;
+
+		StreamPosition columnStart() const;
+		StreamPosition columnEnd() const;
+
+	private:
+		void initializeLazily() const;
+
+		Stream &m_stream;
+		const StreamPosition m_position;
+
+		mutable bool m_isInitialized{false};
+
+		mutable const char *m_sectionStart{nullptr};
+		mutable const char *m_sectionEnd{nullptr};
+
+		mutable StreamPosition m_rowStart{InvalidStreamPosition};
+		mutable StreamPosition m_rowEnd{InvalidStreamPosition};
+
+		mutable StreamPosition m_columnStart{InvalidStreamPosition};
+		mutable StreamPosition m_columnEnd{InvalidStreamPosition};
 };
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

lib/tokenize/include/tokenize/Stream.h

@@ -9,6 +9,7 @@
 #include <vector>
 
 #include <tokenize/Location.h>
+#include <tokenize/StreamPosition.h>
 #include <tokenize/TokenizerException.h>
 
 namespace tokenize
@@ -23,12 +24,9 @@ namespace tokenize
 class Stream
 {
 	public:
-		using Position = size_t;
-		static const Position InvalidPosition;
-
 		struct Delimiter
 		{
-			Position position;
+			StreamPosition position;
 			std::string sectionName;
 		};
 
@@ -46,15 +44,17 @@ class Stream
 		void read(const std::experimental::filesystem::path &path);
 
 		void reset();
-		void seek(Position position);
-		Position position() const;
-		Location location() const;
+		void seek(StreamPosition position);
+		StreamPosition position() const;
 
-		char currentCharacter() const
+		const std::vector<Delimiter> &delimiters() const
 		{
-			assert(m_position < m_stream.size());
+			return m_delimiters;
+		}
 
-			// TODO: check if this should be secured by check()
+		char currentCharacter()
+		{
+			check();
 			return m_stream[m_position];
 		}
 
@@ -69,15 +69,15 @@ class Stream
 			return m_position >= m_stream.size();
 		}
 
-		void check() const
+		void check()
 		{
 			if (atEnd())
-				throw TokenizerException(location(), "reading past end of file");
+				throw TokenizerException(*this, "reading past end of file");
 		}
 
 	protected:
 		std::string m_stream;
-		mutable Position m_position;
+		mutable StreamPosition m_position{0};
 
 		std::vector<Delimiter> m_delimiters;
 };

lib/tokenize/include/tokenize/StreamPosition.h

@@ -0,0 +1,23 @@
+#ifndef __TOKENIZE__STREAM_POSITION_H
+#define __TOKENIZE__STREAM_POSITION_H
+
+#include <cstddef>
+#include <limits>
+
+namespace tokenize
+{
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// StreamPosition
+//
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+using StreamPosition = size_t;
+static const StreamPosition InvalidStreamPosition{std::numeric_limits<StreamPosition>::max()};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+}
+
+#endif

lib/tokenize/include/tokenize/Tokenizer.h

@@ -189,7 +189,7 @@ void Tokenizer<TokenizerPolicy>::expect(const Type &expectedValue)
 	std::stringstream message;
 	message << "unexpected value, expected “" << expectedValue << "”";
 
-	throw TokenizerException(location(), message.str());
+	throw TokenizerException(*this, message.str());
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -208,7 +208,7 @@ std::string Tokenizer<TokenizerPolicy>::getIdentifier()
 		if (!TokenizerPolicy::isIdentifierCharacter(character))
 		{
 			if (value.empty())
-				throw TokenizerException(location(), "could not parse identifier");
+				throw TokenizerException(*this, "could not parse identifier");
 
 			return value;
 		}
@@ -406,7 +406,7 @@ uint64_t Tokenizer<TokenizerPolicy>::getIntegerBody()
 	check();
 
 	if (!std::isdigit(currentCharacter()))
-		throw TokenizerException(location(), "could not read integer value");
+		throw TokenizerException(*this, "could not read integer value");
 
 	uint64_t value = 0;
 
@@ -448,7 +448,7 @@ uint64_t Tokenizer<TokenizerPolicy>::getImpl(Tag<uint64_t>)
 	skipWhiteSpace();
 
 	if (currentCharacter() == '-')
-		throw TokenizerException(location(), "expected unsigned integer, got signed one");
+		throw TokenizerException(*this, "expected unsigned integer, got signed one");
 
 	return getIntegerBody();
 }
@@ -482,7 +482,7 @@ bool Tokenizer<TokenizerPolicy>::getImpl(Tag<bool>)
 	if (testAndSkip<char>('1'))
 		return true;
 
-	throw TokenizerException(location(), "could not read Boolean value");
+	throw TokenizerException(*this, "could not read Boolean value");
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

lib/tokenize/include/tokenize/TokenizerException.h

@@ -30,10 +30,7 @@ class TokenizerException: public std::exception
 
 		explicit TokenizerException(const Location &location, const std::string &message)
 		:	m_location{location},
-			m_message{message},
-			// TODO: refactor
-			m_plainMessage{std::string(m_location.sectionStart) + ":" + std::to_string(m_location.rowStart)
-				+ ":" + std::to_string(m_location.columnStart) + " " + m_message}
+			m_message{message}
 		{
 		}
 
@@ -41,7 +38,7 @@ class TokenizerException: public std::exception
 
 		const char *what() const noexcept
 		{
-			return m_plainMessage.c_str();
+			return m_message.c_str();
 		}
 
 		const Location &location() const
@@ -57,7 +54,6 @@ class TokenizerException: public std::exception
 	private:
 		Location m_location;
 		std::string m_message;
-		std::string m_plainMessage;
 };
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////

lib/tokenize/src/tokenize/Location.cpp

@@ -0,0 +1,150 @@
+#include <tokenize/Location.h>
+
+#include <algorithm>
+
+#include <tokenize/Stream.h>
+
+namespace tokenize
+{
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// Location
+//
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+Location::Location(Stream &stream)
+:	m_stream{stream},
+	m_position{stream.position()}
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+Location::Location(Stream &stream, StreamPosition position)
+:	m_stream{stream},
+	m_position{position}
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+const char *Location::sectionStart() const
+{
+	if (!m_isInitialized)
+		initializeLazily();
+
+	return m_sectionStart;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+const char *Location::sectionEnd() const
+{
+	if (!m_isInitialized)
+		initializeLazily();
+
+	return m_sectionEnd;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+StreamPosition Location::rowStart() const
+{
+	if (!m_isInitialized)
+		initializeLazily();
+
+	return m_rowStart;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+StreamPosition Location::rowEnd() const
+{
+	if (!m_isInitialized)
+		initializeLazily();
+
+	return m_rowEnd;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+StreamPosition Location::columnStart() const
+{
+	if (!m_isInitialized)
+		initializeLazily();
+
+	return m_columnStart;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+StreamPosition Location::columnEnd() const
+{
+	if (!m_isInitialized)
+		initializeLazily();
+
+	return m_columnEnd;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+void Location::initializeLazily() const
+{
+	const auto previousPosition = m_stream.position();
+
+	const auto &delimiters = m_stream.delimiters();
+
+	// Find current section
+	auto currentFile = std::find_if(delimiters.crbegin(), delimiters.crend(),
+		[&](const auto &fileDelimiter)
+		{
+			return m_position >= fileDelimiter.position;
+		});
+
+	// If the tokenizer is at the end of the stream, still count from the beginning of the last section
+	if (currentFile == delimiters.crend())
+		currentFile = delimiters.crbegin();
+
+	// Go back to beginning of section
+	m_stream.seek(currentFile->position);
+
+	StreamPosition row{1};
+	StreamPosition column{1};
+
+	// Compute the location character by character
+	while (true)
+	{
+		if (m_stream.atEnd())
+			break;
+		else if (m_stream.position() >= m_position)
+			break;
+
+		const auto character = m_stream.currentCharacter();
+
+		if (character == '\n')
+		{
+			row++;
+			column = 1;
+		}
+		else if (std::isblank(character) || std::isprint(character))
+			column++;
+
+		m_stream.advance();
+	}
+
+	m_sectionStart = currentFile->sectionName.c_str();
+	m_sectionEnd = currentFile->sectionName.c_str();
+	m_rowStart = row;
+	m_rowEnd = row;
+	m_columnStart = column;
+	m_columnEnd = column;
+
+	m_isInitialized = true;
+
+	m_stream.seek(previousPosition);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+}

lib/tokenize/src/tokenize/Stream.cpp

@@ -1,6 +1,5 @@
 #include <tokenize/Stream.h>
 
-#include <algorithm>
 #include <fstream>
 
 namespace tokenize
@@ -12,12 +11,7 @@ namespace tokenize
 //
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-const Stream::Position Stream::InvalidPosition{std::numeric_limits<Position>::max()};
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-
 Stream::Stream()
-:	m_position{0}
 {
 	std::setlocale(LC_NUMERIC, "C");
 }
@@ -34,18 +28,16 @@ Stream::Stream(std::string streamName, std::istream &istream)
 void Stream::read(std::string streamName, std::istream &istream)
 {
 	// Store position of new section
-	const auto position = m_stream.size();
+	m_delimiters.push_back({m_stream.size(), streamName});
 
-	m_delimiters.push_back({position, streamName});
-
-	istream.seekg(0, std::ios::end);
+	/*istream.seekg(0, std::ios::end);
 	const auto streamSize = istream.tellg();
 	istream.seekg(0, std::ios::beg);
 
 	const auto startPosition = m_stream.size();
 
-	m_stream.resize(m_stream.size() + streamSize);
-	std::copy(std::istreambuf_iterator<char>(istream), std::istreambuf_iterator<char>(), m_stream.begin() + startPosition);
+	m_stream.resize(m_stream.size() + streamSize);*/
+	std::copy(std::istreambuf_iterator<char>(istream), std::istreambuf_iterator<char>(), std::back_inserter(m_stream));
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -69,65 +61,18 @@ void Stream::reset()
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-void Stream::seek(Position position)
+void Stream::seek(StreamPosition position)
 {
 	m_position = position;
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-typename Stream::Position Stream::position() const
+StreamPosition Stream::position() const
 {
 	return m_position;
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
-Location Stream::location() const
-{
-	const auto currentPosition = position();
-
-	// Find current section
-	auto currentFile = std::find_if(m_delimiters.crbegin(), m_delimiters.crend(),
-		[&](const auto &fileDelimiter)
-		{
-			return currentPosition >= fileDelimiter.position;
-		});
-
-	// If the tokenizer is at the end of the stream, still count from the beginning of the last section
-	if (currentFile == m_delimiters.crend())
-		currentFile = m_delimiters.crbegin();
-
-	// Go back to beginning of section
-	m_position = currentFile->position;
-
-	size_t row = 1;
-	size_t column = 1;
-
-	// Compute the location character by character
-	while (true)
-	{
-		if (currentPosition >= m_stream.size() && atEnd())
-			break;
-		else if (currentPosition < m_stream.size() && position() >= currentPosition)
-			break;
-
-		const auto character = currentCharacter();
-
-		if (character == '\n')
-		{
-			row++;
-			column = 1;
-		}
-		else if (std::isblank(character) || std::isprint(character))
-			column++;
-
-		m_position++;
-	}
-
-	return {currentFile->sectionName.c_str(), currentFile->sectionName.c_str(), row, row, column, column};
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-
 }

lib/tokenize/tests/TestTokenizer.cpp

@@ -72,7 +72,7 @@ TEST_CASE("[tokenizer] While tokenizing, the cursor position is as expected", "[
 	std::stringstream s("  identifier  5   \n-51\t 0 1");
 	tokenize::Tokenizer<> p("input", s);
 
-	tokenize::Tokenizer<>::Position pos;
+	tokenize::StreamPosition pos;
 
 	pos = p.position();
 	REQUIRE(p.testAndReturn<std::string>("error") == false);
@@ -181,86 +181,108 @@ TEST_CASE("[tokenizer] While tokenizing, the cursor location is as expcected", "
 
 	const auto startPosition = p.position();
 
-	tokenize::Location l;
-
-	l = p.location();
-	REQUIRE(l.rowStart == 1u);
-	REQUIRE(l.columnStart == 1u);
-	REQUIRE(p.currentCharacter() == '1');
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 1u);
+		REQUIRE(l.columnStart() == 1u);
+		REQUIRE(p.currentCharacter() == '1');
+	}
 
 	REQUIRE_NOTHROW(p.advance());
 
-	l = p.location();
-	REQUIRE(l.rowStart == 1u);
-	REQUIRE(l.columnStart == 2u);
-	REQUIRE(p.currentCharacter() == '2');
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 1u);
+		REQUIRE(l.columnStart() == 2u);
+		REQUIRE(p.currentCharacter() == '2');
+	}
 
 	REQUIRE_NOTHROW(p.advance());
 
-	l = p.location();
-	REQUIRE(l.rowStart == 1u);
-	REQUIRE(l.columnStart == 3u);
-	REQUIRE(p.currentCharacter() == '3');
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 1u);
+		REQUIRE(l.columnStart() == 3u);
+		REQUIRE(p.currentCharacter() == '3');
+	}
 
 	REQUIRE_NOTHROW(p.advance());
 
-	l = p.location();
-	REQUIRE(l.rowStart == 1u);
-	REQUIRE(l.columnStart == 4u);
-	REQUIRE(p.currentCharacter() == ' ');
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 1u);
+		REQUIRE(l.columnStart() == 4u);
+		REQUIRE(p.currentCharacter() == ' ');
+	}
 
 	REQUIRE_NOTHROW(p.advance());
 
-	l = p.location();
-	REQUIRE(l.rowStart == 1u);
-	REQUIRE(l.columnStart == 5u);
-	REQUIRE(p.currentCharacter() == '\n');
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 1u);
+		REQUIRE(l.columnStart() == 5u);
+		REQUIRE(p.currentCharacter() == '\n');
+	}
 
 	REQUIRE_NOTHROW(p.advance());
 
-	l = p.location();
-	REQUIRE(l.rowStart == 2u);
-	REQUIRE(l.columnStart == 1u);
-	REQUIRE(p.currentCharacter() == '4');
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 2u);
+		REQUIRE(l.columnStart() == 1u);
+		REQUIRE(p.currentCharacter() == '4');
+	}
 
 	REQUIRE_NOTHROW(p.advance());
 
 	REQUIRE_NOTHROW(p.expect<std::string>("test1"));
 
-	l = p.location();
-	REQUIRE(l.rowStart == 3u);
-	REQUIRE(l.columnStart == 6u);
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 3u);
+		REQUIRE(l.columnStart() == 6u);
+	}
 
 	REQUIRE_NOTHROW(p.expect<std::string>("test2"));
 
-	l = p.location();
-	REQUIRE(l.rowStart == 4u);
-	REQUIRE(l.columnStart == 7u);
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 4u);
+		REQUIRE(l.columnStart() == 7u);
+	}
 
 	REQUIRE_NOTHROW(p.expect<std::string>("test3"));
 
-	l = p.location();
-	REQUIRE(l.rowStart == 5u);
-	REQUIRE(l.columnStart == 6u);
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 5u);
+		REQUIRE(l.columnStart() == 6u);
+	}
 
 	REQUIRE_NOTHROW(p.skipLine());
 
-	l = p.location();
-	REQUIRE(l.rowStart == 6u);
-	REQUIRE(l.columnStart == 1u);
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 6u);
+		REQUIRE(l.columnStart() == 1u);
+	}
 
 	REQUIRE_NOTHROW(p.skipLine());
 
-	l = p.location();
-	REQUIRE(l.rowStart == 7u);
-	REQUIRE(l.columnStart == 1u);
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 7u);
+		REQUIRE(l.columnStart() == 1u);
+	}
 
 	REQUIRE_NOTHROW(p.skipWhiteSpace());
 
-	l = p.location();
-	REQUIRE(l.rowStart == 10u);
-	REQUIRE(l.columnStart == 1u);
-	REQUIRE(p.atEnd());
+	{
+		auto l = tokenize::Location(p);
+		REQUIRE(l.rowStart() == 10u);
+		REQUIRE(l.columnStart() == 1u);
+		REQUIRE(p.atEnd());
+	}
 
 	p.reset();
 	REQUIRE(p.position() == startPosition);
@@ -289,19 +311,21 @@ TEST_CASE("[tokenizer] Comments are correctly removed", "[tokenizer]")
 
 	p1.removeComments(";", "\n", false);
 
-	tokenize::Location l;
-
 	REQUIRE_NOTHROW(p1.expect<std::string>("test1"));
 
-	l = p1.location();
-	REQUIRE(l.rowStart == 2u);
-	REQUIRE(l.columnStart == 6u);
+	{
+		auto l = tokenize::Location(p1);
+		REQUIRE(l.rowStart() == 2u);
+		REQUIRE(l.columnStart() == 6u);
+	}
 
 	REQUIRE_NOTHROW(p1.expect<std::string>("test2"));
 
-	l = p1.location();
-	REQUIRE(l.rowStart == 3u);
-	REQUIRE(l.columnStart == 6u);
+	{
+		auto l = tokenize::Location(p1);
+		REQUIRE(l.rowStart() == 3u);
+		REQUIRE(l.columnStart() == 6u);
+	}
 
 	p1.skipWhiteSpace();