ThriftParser.java
/*
* Copyright 2016 Providence Authors
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package net.morimekta.providence.reflect.parser;
import net.morimekta.providence.descriptor.PEnumDescriptor;
import net.morimekta.providence.descriptor.PPrimitive;
import net.morimekta.providence.reflect.model.AnnotationDeclaration;
import net.morimekta.providence.reflect.model.ConstDeclaration;
import net.morimekta.providence.reflect.model.Declaration;
import net.morimekta.providence.reflect.model.EnumDeclaration;
import net.morimekta.providence.reflect.model.EnumValueDeclaration;
import net.morimekta.providence.reflect.model.FieldDeclaration;
import net.morimekta.providence.reflect.model.IncludeDeclaration;
import net.morimekta.providence.reflect.model.MessageDeclaration;
import net.morimekta.providence.reflect.model.MethodDeclaration;
import net.morimekta.providence.reflect.model.NamespaceDeclaration;
import net.morimekta.providence.reflect.model.ProgramDeclaration;
import net.morimekta.providence.reflect.model.ServiceDeclaration;
import net.morimekta.providence.reflect.model.TypedefDeclaration;
import net.morimekta.util.lexer.LexerException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import static net.morimekta.providence.reflect.parser.ThriftConstants.kProvidenceKeywords;
import static net.morimekta.providence.reflect.parser.ThriftConstants.kReservedWords;
import static net.morimekta.providence.reflect.parser.ThriftConstants.kThriftKeywords;
import static net.morimekta.providence.reflect.parser.ThriftToken.kKeyValueSep;
import static net.morimekta.providence.reflect.parser.ThriftToken.kLineSep1;
import static net.morimekta.providence.reflect.parser.ThriftToken.kLineSep2;
import static net.morimekta.providence.reflect.parser.ThriftToken.kListEnd;
import static net.morimekta.providence.reflect.parser.ThriftToken.kListStart;
import static net.morimekta.providence.reflect.parser.ThriftToken.kMessageEnd;
import static net.morimekta.providence.reflect.parser.ThriftToken.kMessageStart;
import static net.morimekta.providence.reflect.parser.ThriftToken.kParamsEnd;
import static net.morimekta.providence.reflect.parser.ThriftToken.kParamsStart;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kConst;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kEnum;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kException;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kExtends;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kImplements;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kInclude;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kInterface;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kNamespace;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kOf;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kOneway;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kOptional;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kRequired;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kService;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kStruct;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kThrows;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kTypedef;
import static net.morimekta.providence.reflect.parser.ThriftTokenizer.kUnion;
import static net.morimekta.providence.reflect.util.ReflectionUtils.isApacheThriftFile;
import static net.morimekta.providence.reflect.util.ReflectionUtils.isThriftBasedFileSyntax;
import static net.morimekta.providence.reflect.util.ReflectionUtils.programNameFromPath;
import static net.morimekta.util.Strings.escape;
public class ThriftParser {
private final boolean requireFieldId;
private final boolean requireEnumValue;
private final boolean allowLanguageReservedNames;
private final boolean allowProvidenceOnlyFeatures;
// @VisibleForTesting
ThriftParser() {
this(false, false, true, false);
}
public ThriftParser(boolean requireFieldId,
boolean requireEnumValue,
boolean allowLanguageReservedNames,
boolean allowProvidenceOnlyFeatures) {
this.requireFieldId = requireFieldId;
this.requireEnumValue = requireEnumValue;
this.allowLanguageReservedNames = allowLanguageReservedNames;
this.allowProvidenceOnlyFeatures = allowProvidenceOnlyFeatures;
}
public ProgramDeclaration parse(InputStream in, Path file) throws IOException {
try {
return parseInternal(in, file);
} catch (ThriftException e) {
if (e.getFile() == null) {
e.setFile(file.getFileName().toString());
}
throw e;
} catch (LexerException e) {
throw new ThriftException(e, e.getMessage())
.setFile(file.getFileName().toString());
}
}
private ProgramDeclaration parseInternal(@Nonnull InputStream in,
@Nonnull Path file) throws IOException {
String documentation = null;
List<IncludeDeclaration> includes = new ArrayList<>();
List<NamespaceDeclaration> namespaces = new ArrayList<>();
List<Declaration> declarations = new ArrayList<>();
String programName = programNameFromPath(file.getFileName());
Set<String> includedPrograms = new HashSet<>();
ThriftLexer lexer = new ThriftLexer(in);
boolean has_header = false;
boolean hasDeclaration = false;
String doc_string = null;
ThriftToken token;
while ((token = lexer.next()) != null) {
if (token.type() == ThriftTokenType.DOCUMENTATION) {
if (doc_string != null && !has_header) {
documentation = doc_string;
}
doc_string = token.parseDocumentation();
continue;
}
String keyword = token.toString();
if (!(kThriftKeywords.contains(keyword) ||
(allowProvidenceOnlyFeatures && kProvidenceKeywords.contains(keyword)))) {
throw lexer.failure(token, "Unexpected token \'%s\'", token);
}
switch (keyword) {
case kNamespace:
if (hasDeclaration) {
throw lexer.failure(token, "Unexpected token 'namespace', expected type declaration");
}
if (doc_string != null && !has_header) {
documentation = doc_string;
}
doc_string = null;
has_header = true;
parseNamespace(lexer, token, namespaces);
break;
case kInclude:
if (hasDeclaration) {
throw lexer.failure(token, "Unexpected token 'include', expected type declaration");
}
if (doc_string != null && !has_header) {
documentation = doc_string;
}
doc_string = null;
has_header = true;
includes.add(parseInclude(lexer, token, includedPrograms, file.getParent()));
break;
case kTypedef:
has_header = true;
hasDeclaration = true;
declarations.add(parseTypedef(lexer, token, doc_string));
doc_string = null;
break;
case kEnum:
has_header = true;
hasDeclaration = true;
declarations.add(parseEnum(lexer, token, doc_string));
doc_string = null;
break;
case kInterface:
if (!allowProvidenceOnlyFeatures) {
throw lexer.failure(token, "Interfaces not allowed in .thrift files");
}
case kStruct:
case kUnion:
case kException:
has_header = true;
hasDeclaration = true;
declarations.add(parseMessage(lexer, token, doc_string));
doc_string = null;
break;
case kService:
has_header = true;
hasDeclaration = true;
declarations.add(parseService(lexer, token, doc_string));
doc_string = null;
break;
case kConst:
has_header = true;
hasDeclaration = true;
declarations.add(parseConst(lexer, token, doc_string));
doc_string = null;
break;
default:
throw lexer.failure(token, "Unexpected token \'%s\'", escape(token.toString()));
}
}
return new ProgramDeclaration(
documentation,
programName,
includes,
namespaces,
declarations);
}
/**
* <pre>{@code
* service ::= 'service' {name} ('extends' {extending})? '{' {method}* '}' {annotations}?
* }</pre>
*
* @param tokenizer The tokenizer.
* @param serviceToken The 'service' token.
* @param documentation The service documentation string.
* @return The service declaration.
* @throws IOException If unable to parse the service.
*/
private ServiceDeclaration parseService(
ThriftLexer tokenizer,
ThriftToken serviceToken,
String documentation) throws IOException {
ThriftToken name = tokenizer.expect("service name", ThriftToken::isIdentifier);
if (forbiddenNameIdentifier(name.toString())) {
throw tokenizer.failure(name, "Service with reserved name: " + name);
}
ThriftToken extending = null;
ThriftToken next = tokenizer.expect("service start or extends");
if (next.toString().equals(kExtends)) {
extending = tokenizer.expect("extending type", ThriftToken::isReferenceIdentifier);
next = tokenizer.expectSymbol("service start", kMessageStart);
}
String doc_string = null;
List<MethodDeclaration> functions = new ArrayList<>();
next = tokenizer.expect("function or service end");
while (!next.isSymbol(kMessageEnd)) {
if (next.type() == ThriftTokenType.DOCUMENTATION) {
doc_string = next.parseDocumentation();
next = tokenizer.expect("function or service end");
continue;
}
functions.add(parseMethod(tokenizer, next, doc_string));
next = tokenizer.expect("function or service end");
}
List<AnnotationDeclaration> annotations = null;
if (tokenizer.hasNext() &&
tokenizer.peek("next").isSymbol(kParamsStart)) {
tokenizer.next();
annotations = parseAnnotations(tokenizer);
}
if (tokenizer.hasNext() &&
tokenizer.peek("next").isSymbol(kLineSep2)) {
tokenizer.next();
}
return new ServiceDeclaration(documentation, serviceToken, name, extending, functions, annotations);
}
/**
* <pre>{@code
* method ::= 'oneway'? {type} {name} '(' {param}* ')' ('throws' '(' {throwing}* ')')? {annotations}?
* }</pre>
*
* @param lexer The tokenizer.
* @param next The first token of the method declaration.
* @param documentation The method documentation.
* @return The method declaration.
* @throws IOException If unable to parse the method.
*/
private MethodDeclaration parseMethod(@Nonnull ThriftLexer lexer,
@Nonnull ThriftToken next,
@Nullable String documentation)
throws IOException {
ThriftToken oneway = null;
if (next.toString().equals(kOneway)) {
oneway = next;
next = lexer.expect("method return type");
}
List<ThriftToken> returnType = parseType(lexer, next);
ThriftToken name = lexer.expect("method name", ThriftToken::isIdentifier);
List<FieldDeclaration> params = new ArrayList<>();
lexer.expectSymbol("params start", kParamsStart);
next = lexer.expect("param or params end");
String doc_string = null;
AtomicInteger nextParamId = new AtomicInteger(-1);
ThriftToken requestType = null;
while (!next.isSymbol(kParamsEnd)) {
if (next.type() == ThriftTokenType.DOCUMENTATION) {
doc_string = next.parseDocumentation();
next = lexer.expect("param or params end");
continue;
}
// Special case for proto stub-like methods.
// ReturnType name(RequestType)
if (allowProvidenceOnlyFeatures &&
next.isReferenceIdentifier() &&
lexer.peek("after reference").isSymbol(kParamsEnd)) {
params = null;
requestType = next;
if (kThriftKeywords.contains(next.toString()) ||
kProvidenceKeywords.contains(next.toString())) {
if (PPrimitive.findByName(next.toString()) != null) {
throw new ThriftException(requestType, "Primitive type not allowed as request type on stubs");
}
throw new ThriftException(requestType, "Not allowed as request type, reserved word");
}
lexer.next();
break;
}
params.add(parseField(lexer, next, nextParamId, doc_string, false));
doc_string = null;
next = lexer.expect("param or params end");
}
List<FieldDeclaration> throwing = new ArrayList<>();
if (lexer.peek("throws or service end").toString().equals(kThrows)) {
lexer.next();
lexer.expectSymbol("params start", kParamsStart);
next = lexer.expect("param or params end");
nextParamId.set(-1);
doc_string = null;
while (!next.isSymbol(kParamsEnd)) {
if (next.type() == ThriftTokenType.DOCUMENTATION) {
doc_string = next.parseDocumentation();
next = lexer.expect("param or params end");
continue;
}
throwing.add(parseField(lexer, next, nextParamId, doc_string, false));
doc_string = null;
next = lexer.expect("param or params end");
}
}
List<AnnotationDeclaration> annotations = null;
if (lexer.peek("annotation or service end").isSymbol(kParamsStart)) {
lexer.next();
annotations = parseAnnotations(lexer);
}
if (lexer.peek("function or service end").isSymbol(kLineSep2)) {
lexer.next();
}
return new MethodDeclaration(documentation, oneway, returnType, name, params, requestType, throwing, annotations);
}
/**
* <pre>{@code
* namespace ::= 'namespace' {language} {namespace}
* }</pre>
*
* @param lexer The tokenizer.
* @param namespaceToken The namespace token.
* @param namespaces List of already declared namespaces.
* @throws IOException If unable to parse namespace.
*/
private void parseNamespace(@Nonnull ThriftLexer lexer,
@Nonnull ThriftToken namespaceToken,
@Nonnull List<NamespaceDeclaration> namespaces)
throws IOException {
ThriftToken language = lexer.expect("namespace language",
ThriftToken::isReferenceIdentifier);
if (namespaces.stream().anyMatch(ns -> ns.getLanguage().equals(language.toString()))) {
throw lexer.failure(language,
"Namespace for %s already defined.",
language.toString());
}
ThriftToken namespace = lexer.expect(
"namespace",
t -> VALID_NAMESPACE.matcher(t).matches());
if (lexer.hasNext() &&
lexer.peek("next").isSymbol(kLineSep2)) {
lexer.next();
}
namespaces.add(new NamespaceDeclaration(namespaceToken, language, namespace));
}
/**
* <pre>{@code
* include ::= 'include' {file}
* }</pre>
*
* @param lexer The tokenizer.
* @param includeToken The include token.
* @param includedPrograms Set of included program names.
* @return The include declaration.
* @throws IOException If unable to parse the declaration.
*/
private IncludeDeclaration parseInclude(@Nonnull ThriftLexer lexer,
@Nonnull ThriftToken includeToken,
@Nonnull Set<String> includedPrograms,
@Nonnull Path directory) throws IOException {
ThriftToken include = lexer.expect("include file", ThriftTokenType.STRING);
String filePath = include.decodeString(true);
Path path = Paths.get(filePath);
if (!isThriftBasedFileSyntax(path) ||
(!allowProvidenceOnlyFeatures && !isApacheThriftFile(path))) {
throw lexer.failure(include, "Include not valid for thrift files " + filePath);
}
if (!Files.exists(directory.resolve(filePath))) {
throw lexer.failure(include, "Included file not found " + filePath);
}
ThriftToken includedProgramToken = null;
String includedProgram;
if (allowProvidenceOnlyFeatures &&
lexer.hasNext() && lexer.peek("next").toString().equals("as")) {
lexer.next();
includedProgramToken = lexer.expect("included program name alias", ThriftToken::isIdentifier);
includedProgram = includedProgramToken.toString();
} else {
includedProgram = programNameFromPath(path);
}
if (includedPrograms.contains(includedProgram)) {
throw lexer.failure(include, "thrift program '" + includedProgram + "' already included");
}
includedPrograms.add(includedProgram);
if (lexer.hasNext() &&
lexer.peek("next").isSymbol(kLineSep2)) {
lexer.next();
}
return new IncludeDeclaration(includeToken, include, includedProgramToken);
}
/**
* <pre>{@code
* typedef ::= 'typedef' {name} {type}
* }</pre>
*
* @param tokenizer The tokenizer.
* @param documentation The documentation.
* @return The typedef declaration.
* @throws IOException If unable to parse typedef.
*/
private TypedefDeclaration parseTypedef(@Nonnull ThriftLexer tokenizer,
@Nonnull ThriftToken typedefToken,
@Nullable String documentation) throws IOException {
List<ThriftToken> type = parseType(tokenizer, tokenizer.expect("typename"));
ThriftToken name = tokenizer.expect("typedef identifier", ThriftToken::isIdentifier);
if (forbiddenNameIdentifier(name.toString())) {
throw tokenizer.failure(name, "Typedef with reserved name: " + name);
}
if (tokenizer.hasNext() &&
tokenizer.peek("next").isSymbol(kLineSep2)) {
tokenizer.next();
}
return new TypedefDeclaration(documentation, typedefToken, name, type);
}
/**
* <pre>{@code
* enum ::= 'enum' {name} '{' {enum_value}* '}'
* }</pre>
*
* @param lexer Tokenizer to parse the enum.
* @param enumToken The enum starting token.
* @param documentation The enum documentation.
* @return The enum declaration.
* @throws IOException If unable to parse the enum.
*/
private EnumDeclaration parseEnum(@Nonnull ThriftLexer lexer,
@Nonnull ThriftToken enumToken,
@Nullable String documentation) throws IOException {
ThriftToken name = lexer.expect("enum name", ThriftToken::isIdentifier);
if (forbiddenNameIdentifier(name.toString())) {
throw lexer.failure(name, "Enum with reserved name: %s", name);
}
lexer.expectSymbol("enum start", kMessageStart);
List<EnumValueDeclaration> values = new ArrayList<>();
int nextAutoId = PEnumDescriptor.DEFAULT_FIRST_VALUE;
String doc_string = null;
ThriftToken next = lexer.expect("enum value or end");
while (!next.isSymbol(kMessageEnd)) {
if (next.type() == ThriftTokenType.DOCUMENTATION) {
doc_string = next.parseDocumentation();
next = lexer.expect("enum value or end");
continue;
}
ThriftToken valueName = next;
ThriftToken valueId = null;
next = lexer.expect("enum value sep or end");
if (next.isSymbol(ThriftToken.kFieldValueSep)) {
valueId = lexer.expect("enum value id", ThriftToken::isEnumValueId);
next = lexer.expect("enum value or end");
}
if (requireEnumValue && valueId == null) {
throw lexer.failure(valueName, "Explicit enum value required, but missing");
}
List<AnnotationDeclaration> annotations = null;
if (next.isSymbol(kParamsStart)) {
annotations = parseAnnotations(lexer);
next = lexer.expect("enum value or end");
}
if (next.isSymbol(kLineSep1) ||
next.isSymbol(kLineSep2)) {
next = lexer.expect("enum value or end");
}
int id;
if (valueId != null) {
id = (int) valueId.parseInteger();
nextAutoId = Math.max(id + 1, nextAutoId);
} else {
id = nextAutoId;
++nextAutoId;
}
values.add(new EnumValueDeclaration(doc_string, valueName, valueId, id, annotations));
doc_string = null;
}
List<AnnotationDeclaration> annotations = null;
if (lexer.hasNext() &&
lexer.peek("next").isSymbol(kParamsStart)) {
lexer.next();
annotations = parseAnnotations(lexer);
}
if (lexer.hasNext() &&
lexer.peek("next").isSymbol(kLineSep2)) {
lexer.next();
}
return new EnumDeclaration(documentation, enumToken, name, values, annotations);
}
/**
* <pre>{@code
* message ::= {variant} {name} (('of' | 'implements') {implementing})? '{' {field}* '}'
* }</pre>
*
* @param lexer Tokenizer.
* @param variant The message variant.
* @param documentation The message documentation.
* @return The parsed message declaration.
* @throws IOException If unable to parse the message.
*/
private MessageDeclaration parseMessage(@Nonnull ThriftLexer lexer,
@Nonnull ThriftToken variant,
@Nullable String documentation)
throws IOException {
ThriftToken implementing = null;
List<FieldDeclaration> fields = new ArrayList<>();
boolean union = variant.toString().equals(kUnion);
boolean struct = variant.toString().equals(kStruct);
boolean iFace = variant.toString().equals(kInterface);
ThriftToken name = lexer.expect("message name identifier", ThriftToken::isIdentifier);
if (forbiddenNameIdentifier(name.toString())) {
throw lexer.failure(name, "Message with reserved name: %s", name);
}
ThriftToken next = lexer.expect("message start");
if (!next.isSymbol(kMessageStart)) {
if (!allowProvidenceOnlyFeatures) {
throw lexer.failure(next, "Expected message start, got '%s'",
escape(next));
}
if (struct && next.toString().equals(kImplements)) {
implementing = lexer.expect("implementing type", ThriftToken::isReferenceIdentifier);
} else if (union && next.toString().equals(kOf)) {
implementing = lexer.expect("union of type", ThriftToken::isReferenceIdentifier);
} else {
if (union) {
throw lexer.failure(next, "Expected message start, or union 'of', got '%s'",
escape(next.toString()));
}
if (struct) {
throw lexer.failure(next, "Expected message start, or 'implements', got '%s'",
escape(next.toString()));
}
throw lexer.failure(next, "Expected message start, got '%s'",
escape(next.toString()));
}
lexer.expectSymbol("message start", kMessageStart);
}
AtomicInteger nextAutoId = new AtomicInteger(-1);
String doc_string = null;
next = lexer.expect("field def or message end");
while (!next.isSymbol(kMessageEnd)) {
if (next.type() == ThriftTokenType.DOCUMENTATION) {
doc_string = next.parseDocumentation();
next = lexer.expect("field def or message end");
continue;
}
fields.add(parseField(lexer, next, nextAutoId, doc_string, iFace));
doc_string = null;
next = lexer.expect("field def or message end");
}
List<AnnotationDeclaration> annotations = null;
if (lexer.hasNext() &&
lexer.peek("next").isSymbol(kParamsStart)) {
lexer.next();
annotations = parseAnnotations(lexer);
}
if (lexer.hasNext() &&
lexer.peek("next").isSymbol(kLineSep2)) {
lexer.next();
}
return new MessageDeclaration(documentation, variant, name, implementing, fields, annotations);
}
/**
* <pre>{@code
* field ::= ({id} ':')? {requirement}? {type} {name} ('=' {defaultValue})? {annotations}?
* }</pre>
*
* @param lexer The tokenizer.
* @param documentation The field documentation.
* @return The field declaration.
* @throws IOException If unable to read the declaration.
*/
private FieldDeclaration parseField(ThriftLexer lexer,
ThriftToken next,
AtomicInteger nextAutoId,
String documentation,
boolean iFace) throws IOException {
int fieldId;
ThriftToken idToken = null;
if (next.isFieldId()) {
if (iFace) {
throw lexer.failure(next, "Field id in interfaces not allowed");
}
idToken = next;
fieldId = (int) idToken.parseInteger();
if (fieldId < 1) {
throw lexer.failure(next, "Non-positive ");
}
lexer.expectSymbol("field id sep", kKeyValueSep);
next = lexer.expect("field type");
} else if (requireFieldId && !iFace) {
throw lexer.failure(next, "Field id required, but missing");
} else {
fieldId = nextAutoId.getAndDecrement();
}
ThriftToken requirement = null;
if (next.toString().equals(kOptional) ||
next.toString().equals(kRequired)) {
requirement = next;
next = lexer.expect("field type");
}
List<ThriftToken> type = parseType(lexer, next);
ThriftToken name = lexer.expect("field name", ThriftToken::isIdentifier);
List<ThriftToken> defaultValue = null;
List<AnnotationDeclaration> annotations = null;
if (forbiddenNameIdentifier(name.toString())) {
throw lexer.failure(name, "Field with reserved name: " + name.toString());
}
if (lexer.peek("field value").isSymbol(ThriftToken.kFieldValueSep)) {
lexer.next();
defaultValue = parseValue(lexer);
}
if (lexer.peek("field annotation").isSymbol(kParamsStart)) {
lexer.next();
annotations = parseAnnotations(lexer);
}
if (lexer.peek("field sep").isSymbol(kLineSep1) ||
lexer.peek("field sep").isSymbol(kLineSep2)) {
lexer.next();
}
return new FieldDeclaration(documentation, idToken, fieldId, requirement, name, type, defaultValue, annotations);
}
/**
* <pre>{@code
* const type kName = value;
* }</pre>
*
* @param lexer The tokenizer.
* @return The parsed const declaration.
* @throws IOException If parsing failed.
*/
private ConstDeclaration parseConst(@Nonnull ThriftLexer lexer,
@Nonnull ThriftToken constToken,
@Nullable String documentation) throws IOException {
List<ThriftToken> type = parseType(lexer, lexer.expect("const type"));
ThriftToken name = lexer.expect("const name", ThriftToken::isIdentifier);
if (forbiddenNameIdentifier(name.toString())) {
throw lexer.failure(name, "Const with reserved name: " + name.toString());
}
lexer.expect("const value sep", t -> t.isSymbol(ThriftToken.kFieldValueSep));
List<ThriftToken> value = parseValue(lexer);
List<AnnotationDeclaration> annotations = null;
if (allowProvidenceOnlyFeatures &&
lexer.hasNext() &&
lexer.expect("next").isSymbol(kParamsStart)) {
lexer.next();
annotations = parseAnnotations(lexer);
}
if (lexer.hasNext() &&
lexer.peek("next").isSymbol(kLineSep2)) {
lexer.next();
}
return new ConstDeclaration(documentation, constToken, name, type, value, annotations);
}
/**
* Parse annotations, not including the initial '('.
*
* <pre>{@code
* annotation ::= key ('=' value)?
* annotations ::= '(' annotation (',' annotation)* ')'
* }</pre>
*
* @param tokenizer The tokenizer to get tokens from.
* @return The list of annotations.
*/
private List<AnnotationDeclaration> parseAnnotations(@Nonnull ThriftLexer tokenizer)
throws IOException {
List<AnnotationDeclaration> annotations = new ArrayList<>();
ThriftToken next = tokenizer.expect("annotation key");
while (!next.isSymbol(kParamsEnd)) {
if (!next.isReferenceIdentifier()) {
throw tokenizer.failure(next, "annotation key must be identifier-like");
}
ThriftToken key = next;
ThriftToken value = null;
next = tokenizer.expect("annotation end or sep");
if (next.isSymbol(ThriftToken.kFieldValueSep)) {
value = tokenizer.expect("annotation value", ThriftTokenType.STRING);
next = tokenizer.expect("annotation end or sep");
}
annotations.add(new AnnotationDeclaration(key, value));
if (next.isSymbol(kLineSep1)) {
next = tokenizer.expect("annotation key", t -> !t.isSymbol(kParamsEnd));
}
}
return annotations;
}
/**
* @param lexer Tokenizer to parse value.
* @return The list of tokens part of the value.
* @throws IOException If unable to parse the type.
*/
private List<ThriftToken> parseValue(@Nonnull ThriftLexer lexer) throws IOException {
return parseValue(lexer, lexer.expect("value"));
}
/**
* @param lexer Tokenizer to parse value.
* @param token First token of the value.
* @return The list of tokens part of the value.
* @throws IOException If unable to parse the type.
*/
private List<ThriftToken> parseValue(@Nonnull ThriftLexer lexer,
@Nonnull ThriftToken token) throws IOException {
// Ignore these comments.
while (token.type() == ThriftTokenType.DOCUMENTATION) {
token = lexer.expect("value");
}
List<ThriftToken> tokens = new ArrayList<>();
tokens.add(token);
if (token.isSymbol(kListStart)) {
ThriftToken next = lexer.expect("list value");
while (!next.isSymbol(kListEnd)) {
tokens.addAll(parseValue(lexer, next));
next = lexer.expect("list value, sep or end");
if (next.isSymbol(kLineSep1) || next.isSymbol(kLineSep2)) {
tokens.add(next);
next = lexer.expect("list value or end");
}
}
tokens.add(next);
} else if (token.isSymbol(kMessageStart)) {
ThriftToken next = lexer.expect("map key");
while (!next.isSymbol(kMessageEnd)) {
tokens.addAll(parseValue(lexer, next));
tokens.add(lexer.expectSymbol("key value sep", kKeyValueSep));
tokens.addAll(parseValue(lexer, lexer.expect("map value")));
next = lexer.expect("map sep or end");
if (next.isSymbol(kLineSep1) || next.isSymbol(kLineSep2)) {
tokens.add(next);
next = lexer.expect("map sep or end");
}
}
tokens.add(next);
} else if (!token.isReferenceIdentifier() &&
token.type() != ThriftTokenType.NUMBER &&
token.type() != ThriftTokenType.STRING) {
throw lexer.failure(token, "not a value type: '%s'", token.toString());
}
return tokens;
}
/**
* @param lexer The tokenizer.
* @param token The first token of the type.
* @return List of tokens part of the type string.
* @throws IOException If unable to parse the type.
*/
private List<ThriftToken> parseType(@Nonnull ThriftLexer lexer,
@Nonnull ThriftToken token) throws IOException {
if (!token.isQualifiedIdentifier() &&
!token.isIdentifier()) {
throw lexer.failure(token, "Expected type identifier but found " + token);
}
List<ThriftToken> tokens = new ArrayList<>();
tokens.add(token);
String type = token.toString();
switch (type) {
case "list":
case "set": {
tokens.add(lexer.expect(type + " generic start", t -> t.isSymbol(ThriftToken.kGenericStart)));
tokens.addAll(parseType(lexer, lexer.expect(type + " item type")));
tokens.add(lexer.expect(type + " generic end", t -> t.isSymbol(ThriftToken.kGenericEnd)));
break;
}
case "map": {
tokens.add(lexer.expect(type + " generic start", t -> t.isSymbol(ThriftToken.kGenericStart)));
tokens.addAll(parseType(lexer, lexer.expect(type + " key type")));
tokens.add(lexer.expect(type + " generic sep", t -> t.isSymbol(kLineSep1)));
tokens.addAll(parseType(lexer, lexer.expect(type + " item type")));
tokens.add(lexer.expect(type + " generic end", t -> t.isSymbol(ThriftToken.kGenericEnd)));
break;
}
}
return tokens;
}
private boolean forbiddenNameIdentifier(String name) {
if (kThriftKeywords.contains(name)) {
return true;
} else if (allowProvidenceOnlyFeatures && kProvidenceKeywords.contains(name)) {
return true;
} else return !allowLanguageReservedNames && kReservedWords.contains(name);
}
private static final Pattern VALID_IDENTIFIER = Pattern.compile(
"[_a-zA-Z][_a-zA-Z0-9]*");
private static final Pattern VALID_NAMESPACE = Pattern.compile(
"([_a-zA-Z][_a-zA-Z0-9]*[.])*[_a-zA-Z][_a-zA-Z0-9]*");
}