Skip to content

Commit

Permalink
fix column default types
Browse files Browse the repository at this point in the history
  • Loading branch information
nilswende committed Oct 23, 2024
1 parent 7fb93a0 commit acf265e
Show file tree
Hide file tree
Showing 8 changed files with 268 additions and 28 deletions.
12 changes: 10 additions & 2 deletions src/main/java/com/wn/dbml/compiler/Token.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
* An element of a DBML text.
*/
public interface Token {

/**
* The type of this Token.
*/
Expand All @@ -17,7 +18,14 @@ public interface Token {
String getValue();

/**
* Returns a copy of this Token with {@link TokenType#LITERAL}.
* Returns a copy of this Token with the given {@link TokenType}.
*/
Token withType(TokenType tokenType);

/**
* Returns a copy of this Token with the matching literal type.
*/
Token toLiteral();
default Token toLiteral() {
return withType(TokenType.LITERAL);
}
}
30 changes: 28 additions & 2 deletions src/main/java/com/wn/dbml/compiler/parser/ParserImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,25 @@
import com.wn.dbml.compiler.ParsingException;
import com.wn.dbml.compiler.Position;
import com.wn.dbml.compiler.token.TokenType;
import com.wn.dbml.model.*;
import com.wn.dbml.model.Column;
import com.wn.dbml.model.ColumnSetting;
import com.wn.dbml.model.Database;
import com.wn.dbml.model.EnumValue;
import com.wn.dbml.model.Index;
import com.wn.dbml.model.IndexSetting;
import com.wn.dbml.model.Name;
import com.wn.dbml.model.Note;
import com.wn.dbml.model.Project;
import com.wn.dbml.model.Relation;
import com.wn.dbml.model.RelationshipSetting;
import com.wn.dbml.model.Schema;
import com.wn.dbml.model.Setting;
import com.wn.dbml.model.SettingHolder;
import com.wn.dbml.model.Table;
import com.wn.dbml.model.TableSetting;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -186,7 +202,7 @@ private void parseColumnSetting(final Column column) {
case PRIMARY_KEY, PK -> addSetting(column, ColumnSetting.PRIMARY_KEY);
case UNIQUE -> addSetting(column, ColumnSetting.UNIQUE);
case INCREMENT -> addSetting(column, ColumnSetting.INCREMENT);
case DEFAULT -> addSetting(column, ColumnSetting.DEFAULT, stringTypes());
case DEFAULT -> addSetting(column, ColumnSetting.DEFAULT, stringTypesOr(EXPR, BLITERAL, NLITERAL));
case NOTE -> column.setNote(parseInlineNote());
case REF -> relationshipDefinitions.add(parseInlineRef(column));
default -> throw new IllegalStateException("Unexpected value: " + tokenType());
Expand Down Expand Up @@ -537,6 +553,16 @@ private TokenType[] stringTypes() {
return new TokenType[]{SSTRING, DSTRING, TSTRING};
}

private TokenType[] stringTypesOr(final TokenType... types) {
return types != null && types.length > 0 ? concat(stringTypes(), types) : stringTypes();
}

private static <T> T[] concat(final T[] first, final T[] second) {
var result = Arrays.copyOf(first, first.length + second.length);
System.arraycopy(second, 0, result, first.length, second.length);
return result;
}

private void next(final TokenType... types) {
tokenAccess.next(types);
}
Expand Down
87 changes: 67 additions & 20 deletions src/main/java/com/wn/dbml/compiler/parser/TokenAccess.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,24 @@
import com.wn.dbml.compiler.ParsingException;
import com.wn.dbml.compiler.Position;
import com.wn.dbml.compiler.Token;
import com.wn.dbml.compiler.token.Literals;
import com.wn.dbml.compiler.token.TokenImpl;
import com.wn.dbml.compiler.token.TokenType;

import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Objects;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;

import static com.wn.dbml.compiler.token.TokenType.*;

class TokenAccess {
private final RingBuffer<Token> lastTokens = new RingBuffer<>(5);
private final Queue<Lookahead> lookahead = new ArrayDeque<>(2);
private final Lexer lexer;
private Token token;
private Lookahead lookahead;
private boolean ignoreLinebreaks = true, ignoreSpaces = true;

TokenAccess(final Lexer lexer) {
Expand All @@ -26,23 +31,23 @@ class TokenAccess {
public void next(final TokenType... types) {
if (types != null && types.length > 0) {
token = nextToken();
if (!type().isWhitespace() && !type().isMultiKeyword()
&& Arrays.stream(types).noneMatch(this::typeIs)
&& Arrays.stream(types).anyMatch(t -> t == LITERAL)) {
token = token.toLiteral();
if (shouldParseAsLiteral(types)) {
token = nextLiteral(types);
}
lastTokens.add(token);
expecting(token, types);
}
}

private Token nextToken() {
Token token;
if (lookahead != null) {
token = lookahead.token();
lookahead = null;
return token;
if (!lookahead.isEmpty()) {
return lookahead.poll().token();
}
return nextTokenFromLexer();
}

private Token nextTokenFromLexer() {
Token token;
do {
token = lexer.nextToken();
} while (skipToken(token));
Expand All @@ -53,20 +58,62 @@ private boolean skipToken(final Token token) {
return token != null && (
// skip
token.getType() == COMMENT
// skip or collapse
|| token.getType() == LINEBREAK && (ignoreLinebreaks || typeIs(LINEBREAK))
// skip or collapse
|| token.getType() == SPACE && (ignoreSpaces || typeIs(SPACE))
// skip or collapse
|| token.getType() == LINEBREAK && (ignoreLinebreaks || typeIs(LINEBREAK))
// skip or collapse
|| token.getType() == SPACE && (ignoreSpaces || typeIs(SPACE))
);
}

private boolean shouldParseAsLiteral(TokenType[] types) {
return !type().isWhitespace() && !type().isMultiKeyword()
&& Arrays.stream(types).noneMatch(this::typeIs)
&& Arrays.stream(types).anyMatch(TokenType::isLiteral);
}

private Token nextLiteral(final TokenType... types) {
var typeSet = Set.of(types);
if (typeSet.contains(BLITERAL)) {
if (Literals.isBooleanLiteral(value())) {
return token.withType(BLITERAL);
}
}
if (typeSet.contains(NLITERAL)) {
var value = value();
var peek = doLookahead();
if (peek.getType() == DOT) {
value += peek.getValue();
peek = doLookahead();
if (peek.getType() == LITERAL) {
value += peek.getValue();
if (Literals.isNumberLiteral(value)) {
nextToken();
nextToken();
return new TokenImpl(NLITERAL, value);
}
}
} else {
if (Literals.isNumberLiteral(value)) {
return token.withType(NLITERAL);
}
}
}
return token.toLiteral();
}

public Token lookahead() {
if (lookahead == null) {
// save the current token position because nextToken() advances the lexer
var position = position();
lookahead = new Lookahead(nextToken(), position);
if (lookahead.isEmpty()) {
return doLookahead();
}
return lookahead.token();
return lookahead.peek().token();
}

private Token doLookahead() {
// save the current lexer position because nextToken() advances the lexer
var position = lexer.getPosition();
var t = nextTokenFromLexer();
lookahead.add(new Lookahead(t, position));
return t;
}

public TokenType type() {
Expand Down Expand Up @@ -110,7 +157,7 @@ public void error(final String msg) {
}

public Position position() {
return lookahead == null ? lexer.getPosition() : lookahead.position();
return lookahead.isEmpty() ? lexer.getPosition() : lookahead.peek().position();
}

public void setIgnoreLinebreaks(final boolean ignoreLinebreaks) {
Expand Down
58 changes: 58 additions & 0 deletions src/main/java/com/wn/dbml/compiler/token/Literals.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package com.wn.dbml.compiler.token;

import java.util.Set;
import java.util.regex.Pattern;

/**
* Helper for literals.
*/
public final class Literals {
private static final Set<String> BOOLEAN = Set.of("true", "false", "null");
private static final Pattern NUMBER = Pattern.compile("-?\\d+|-?\\d+\\.\\d+");

private Literals() {
throw new AssertionError();
}

/**
* Returns the string's literal subtype, if applicable.
*
* @param value a string
* @return the subtype or else null
*/
public static TokenType getSubType(final String value) {
if (isBooleanLiteral(value)) {
return TokenType.BLITERAL;
} else if (isNumberLiteral(value)) {
return TokenType.NLITERAL;
}
return null;
}

/**
* Returns true, if the string is a boolean literal.
*
* @param value a string
*/
public static boolean isBooleanLiteral(final String value) {
return value != null && BOOLEAN.contains(value);
}

/**
* Returns true, if the string is a number literal.
*
* @param value a string
*/
public static boolean isNumberLiteral(final String value) {
return value != null && NUMBER.matcher(value).matches();
}

/**
* Returns true, if the string is of any literal subtype.
*
* @param value a string
*/
public static boolean isSubType(final String value) {
return getSubType(value) != null;
}
}
4 changes: 2 additions & 2 deletions src/main/java/com/wn/dbml/compiler/token/TokenImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ public TokenImpl(final TokenType type, final String value) {
}

@Override
public Token toLiteral() {
return new TokenImpl(TokenType.LITERAL, value);
public Token withType(final TokenType tokenType) {
return new TokenImpl(tokenType, value);
}

@Override
Expand Down
12 changes: 11 additions & 1 deletion src/main/java/com/wn/dbml/compiler/token/TokenType.java
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,11 @@ public enum TokenType {
LINEBREAK,
SPACE,
_WHITESPACE_STOP,
// Literal
_LITERAL_START,
LITERAL,
BLITERAL,
NLITERAL,
_LITERAL_STOP,
SSTRING,
DSTRING,
TSTRING,
Expand Down Expand Up @@ -129,4 +132,11 @@ public boolean isMultiKeyword() {
public boolean isWhitespace() {
return TokenType._WHITESPACE_START.ordinal() < ordinal() && ordinal() < TokenType._WHITESPACE_STOP.ordinal();
}

/**
* Returns true, if this token represents a literal.
*/
public boolean isLiteral() {
return TokenType._LITERAL_START.ordinal() < ordinal() && ordinal() < TokenType._LITERAL_STOP.ordinal();
}
}
56 changes: 55 additions & 1 deletion src/test/java/com/wn/dbml/compiler/ParserTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,13 @@

import com.wn.dbml.compiler.lexer.LexerImpl;
import com.wn.dbml.compiler.parser.ParserImpl;
import com.wn.dbml.model.*;
import com.wn.dbml.model.Column;
import com.wn.dbml.model.ColumnSetting;
import com.wn.dbml.model.Database;
import com.wn.dbml.model.Name;
import com.wn.dbml.model.RelationshipSetting;
import com.wn.dbml.model.Schema;
import com.wn.dbml.model.TableSetting;
import org.junit.jupiter.api.Test;

import java.util.List;
Expand Down Expand Up @@ -793,4 +799,52 @@ void testParseNotMultiKeyword() {
var table = schema.getTable("table1");
assertEquals("integer", table.getColumn("not").getType());
}

@Test
void testParseColumnDefault() {
var dbml = """
Table Organization {
organization_id integer [pk]
name varchar [not null]
code varchar [unique, not null]
created_at timestamp [default: `CURRENT_TIMESTAMP`]
updated_at timestamp [default: `CURRENT_TIMESTAMP`]
active boolean [default: true]
percent decimal [default: 100.0]
number int [default: 0]
}""";
var database = parse(dbml);

var schema = getDefaultSchema(database);
var table = schema.getTable("Organization");
assertEquals("CURRENT_TIMESTAMP", table.getColumn("created_at").getSettings().get(ColumnSetting.DEFAULT));
assertEquals("CURRENT_TIMESTAMP", table.getColumn("updated_at").getSettings().get(ColumnSetting.DEFAULT));
assertEquals("true", table.getColumn("active").getSettings().get(ColumnSetting.DEFAULT));
assertEquals("100.0", table.getColumn("percent").getSettings().get(ColumnSetting.DEFAULT));
assertEquals("0", table.getColumn("number").getSettings().get(ColumnSetting.DEFAULT));
}

@Test
void testParseColumnDefaultInvalid() {
var dbml = """
Table Organization {
organization_id integer [pk]
number int [default: invalid]
}""";

var e = assertThrows(ParsingException.class, () -> parse(dbml));
assertTrue(e.getMessage().startsWith("[3:30] unexpected token 'LITERAL'"), e.getMessage());
}

@Test
void testParseColumnDefaultInvalidDecimal() {
var dbml = """
Table Organization {
organization_id integer [pk]
number int [default: 1.invalid]
}""";

var e = assertThrows(ParsingException.class, () -> parse(dbml));
assertTrue(e.getMessage().startsWith("[3:24] unexpected token 'LITERAL'"), e.getMessage());
}
}
Loading

0 comments on commit acf265e

Please sign in to comment.