Lexer.java

  1. /*
  2.  * Copyright (c) 2015-2020, Stein Eldar Johnsen
  3.  *
  4.  * Licensed to the Apache Software Foundation (ASF) under one
  5.  * or more contributor license agreements. See the NOTICE file
  6.  * distributed with this work for additional information
  7.  * regarding copyright ownership. The ASF licenses this file
  8.  * to you under the Apache License, Version 2.0 (the
  9.  * "License"); you may not use this file except in compliance
  10.  * with the License. You may obtain a copy of the License at
  11.  *
  12.  *   http://www.apache.org/licenses/LICENSE-2.0
  13.  *
  14.  * Unless required by applicable law or agreed to in writing,
  15.  * software distributed under the License is distributed on an
  16.  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17.  * KIND, either express or implied. See the License for the
  18.  * specific language governing permissions and limitations
  19.  * under the License.
  20.  */
  21. package net.morimekta.lexer;

  23. import java.io.IOException;
  24. import java.io.UncheckedIOException;
  25. import java.util.Iterator;
  26. import java.util.function.Predicate;

  28. import static net.morimekta.strings.EscapeUtil.javaEscape;

  30. /**
  31.  * Base lexer class with helper methods that does not need to be
  32.  * implemented. The base lexer should be able to continuously return
  33.  * tokens until end of the stream, or the lexer process fails.
  34.  *
  35.  * @param <TT> The token-type enum class.
  36.  * @param <T> The token implementation class.
  37.  */
  38. public class Lexer<TT extends TokenType, T extends Token<TT>> implements Iterable<T> {
  39.     /**
  40.      * Create a lexer instance using a specific tokenizer.
  41.      *
  42.      * @param tokenizer The tokenizer to be used to get tokens.
  43.      */
  44.     protected Lexer(Tokenizer<TT, T> tokenizer) {
  45.         this.tokenizer = tokenizer;
  46.     }

  48.     /**
  49.      * Make a lexing / parsing failure exception.
  50.      *
  51.      * @param line The line for the failure.
  52.      * @param lineNo The line no for the failure.
  53.      * @param linePos The line pos for the failure.
  54.      * @param message The message for the failure.
  55.      * @param args Arguments to format message.
  56.      * @return The failure exception.
  57.      */
  58.     protected LexerException eofFailure(CharSequence line, int lineNo, int linePos, String message, Object... args) {
  59.         if (args.length > 0) {
  60.             message = String.format(message, args);
  61.         }
  62.         return new LexerException(line, lineNo, linePos, 1, message);
  63.     }

  65.     /**
  66.      * Make a lexing / parsing failure exception.
  67.      *
  68.      * @param token The token causing the failure.
  69.      * @param message The message for the failure.
  70.      * @param args Arguments for formatting message.
  71.      * @return The failure exception.
  72.      */
  73.     public LexerException failure(T token, String message, Object... args) {
  74.         if (args.length > 0) {
  75.             message = String.format(message, args);
  76.         }
  77.         return new LexerException(token, message);
  78.     }

  80.     /**
  81.      * Consume and return the next token. This should not
  82.      * trigger parsing anything after this token.
  83.      *
  84.      * @return The next token, or null if it's end of the stream.
  85.      * @throws LexerException If parsing token failed.
  86.      * @throws IOException If reading failed.
  87.      */
  88.     public T next() throws LexerException, IOException {
  89.         // --- PRIVATE ---
  90.         T lastToken;
  91.         if (nextToken != null) {
  92.             lastToken = nextToken;
  93.             nextToken = null;
  94.         } else {
  95.             lastToken = tokenizer.parseNextToken();
  96.         }
  97.         return lastToken;
  98.     }

  100.     /**
  101.      * Return true if there is a 'next' token. If this method returns
  102.      * true, then 'peek' must return non-null until otherwise modified, and
  103.      * the next call to 'next' must return non-null.
  104.      *
  105.      * @return If there is a next token.
  106.      * @throws LexerException If parsing token failed.
  107.      * @throws IOException If reading failed.
  108.      */
  109.     public boolean hasNext() throws LexerException, IOException {
  110.         return peek() != null;
  111.     }

  113.     /**
  114.      * Return the token that will be returned by 'next', but do not
  115.      * 'consume' it. If this method returns a non-null value, 'next'
  116.      * must return the same value exactly once.
  117.      *
  118.      * @return The next token.
  119.      * @throws LexerException If parsing token failed.
  120.      * @throws IOException If reading failed.
  121.      */
  122.     public T peek() throws LexerException, IOException {
  123.         if (nextToken == null) {
  124.             nextToken = tokenizer.parseNextToken();
  125.         }
  126.         return nextToken;
  127.     }

  129.     /**
  130.      * Peek the next token, and fail if the token is not present.
  131.      *
  132.      * @param what The exception message on failure.
  133.      * @return The token to be the next.
  134.      * @throws LexerException On parse errors.
  135.      * @throws IOException If reading failed.
  136.      */
  137.     public T peek(String what) throws LexerException, IOException {
  138.         T token = peek();
  139.         if (token == null) {
  140.             throw eofFailure(tokenizer.currentLine(),
  141.                              tokenizer.currentLineNo(),
  142.                              tokenizer.currentLinePos(),
  143.                              "Expected %s, but got end of file",
  144.                              what);
  145.         }
  146.         return token;
  147.     }

  149.     /**
  150.      * Expect a new token, and fail there is no next token.
  151.      *
  152.      * @param what What is expected.
  153.      * @return The next token.
  154.      * @throws LexerException On parse errors or missing next token.
  155.      * @throws IOException If reading failed.
  156.      */
  157.     public T expect(String what) throws LexerException, IOException {
  158.         T next = next();
  159.         if (next == null) {
  160.             throw eofFailure(tokenizer.currentLine(),
  161.                              tokenizer.currentLineNo(),
  162.                              tokenizer.currentLinePos(),
  163.                              "Expected %s, but got end of file",
  164.                              what);
  165.         }
  166.         return next;
  167.     }

  169.     /**
  170.      * Expect a new token, and fail if the token is not of the given token type.
  171.      *
  172.      * @param what The exception message on failure.
  173.      * @param type The token type being expected.
  174.      * @return The token to be the next.
  175.      * @throws LexerException On parse errors or validation failures.
  176.      * @throws IOException If reading failed.
  177.      */
  178.     public T expect(String what, TT type) throws LexerException, IOException {
  179.         T token = expect(what);
  180.         if (!token.type().equals(type)) {
  181.             nextToken = token;
  182.             throw failure(token, "Expected %s, but got '%s'", what, javaEscape(token));
  183.         }
  184.         return token;
  185.     }

  187.     /**
  188.      * Expect a new token, and fail if the token does not validate.
  189.      *
  190.      * @param what The exception message on failure.
  191.      * @param validator Validator to check on the token.
  192.      * @return The token to be the next.
  193.      * @throws LexerException On parse errors or validation failure.
  194.      * @throws IOException If reading failed.
  195.      */
  196.     public T expect(String what, Predicate<T> validator) throws LexerException, IOException {
  197.         T token = expect(what);
  198.         if (!validator.test(token)) {
  199.             nextToken = token;
  200.             throw failure(token, "Expected %s, but got '%s'", what, javaEscape(token));
  201.         }
  202.         return token;
  203.     }

  205.     /**
  206.      * @param what The exception message on failure.
  207.      * @param symbols Symbols to be expected.
  208.      * @return The token of the symbol.
  209.      * @throws LexerException On parse errors or validation failure.
  210.      * @throws IOException If unable to parse token, or not applicable symbol.
  211.      */
  212.     public T expectSymbol(String what, char... symbols) throws LexerException, IOException {
  213.         return expect(what, t -> {
  214.             if (t.length() != 1) {
  215.                 return false;
  216.             }
  217.             for (char s : symbols) {
  218.                 if (t.isSymbol(s)) {
  219.                     return true;
  220.                 }
  221.             }
  222.             return false;
  223.         });
  224.     }

  226.     /**
  227.      * Read until termination string.
  228.      *
  229.      * @param term The termination string.
  230.      * @param type The type of token to be generated.
  231.      * @param allowEof If we allow end of file to termainate the token.
  232.      * @return The read token if it has any size.
  233.      * @throws LexerException On parse errors or validation failure.
  234.      * @throws IOException If unable to parse token.
  235.      */
  236.     public T readUntil(String term, TT type, boolean allowEof) throws LexerException, IOException {
  237.         nextToken = null;
  238.         return tokenizer.readUntil(term, type, allowEof);
  239.     }

  241.     @Override
  242.     public Iterator<T> iterator() {
  243.         return new LexerIterator();
  244.     }

  246.     // --- PRIVATE ---

  248.     private final Tokenizer<TT, T> tokenizer;
  249.     private T nextToken;

  251.     private class LexerIterator implements Iterator<T> {
  252.         @Override
  253.         public boolean hasNext() {
  254.             try {
  255.                 return Lexer.this.hasNext();
  256.             } catch (LexerException e) {
  257.                 throw new UncheckedLexerException(e);
  258.             } catch (IOException e) {
  259.                 throw new UncheckedIOException(e.getMessage(), e);
  260.             }
  261.         }

  263.         @Override
  264.         public T next() {
  265.             try {
  266.                 return Lexer.this.expect("anything");
  267.             } catch (LexerException e) {
  268.                 throw new UncheckedLexerException(e);
  269.             } catch (IOException e) {
  270.                 throw new UncheckedIOException(e.getMessage(), e);
  271.             }
  272.         }
  273.     }

  275.     @Override
  276.     public String toString() {
  277.         return getClass().getSimpleName() + "{tokenizer=" + tokenizer + ", next=" + nextToken + "}";
  278.     }
  279. }
Created with JaCoCo 0.8.8.202204050719