android-lang/src/com/android/tools/idea/lang/androidSql/parser/androidSql.bnf

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* SQLite grammar adapted from http://www.sqlite.org/docsrc/doc/trunk/art/syntax/all-bnf.html
 * This should correspond directly to diagrams in the "SQL Syntax" part of SQLite documentation,
 * e.g. https://sqlite.org/lang_select.html. See also all diagrams here: http://www.sqlite.org/syntaxdiagrams.html
 *
 * Unfortunately the grammar linked above skips the most basic definitions, like string-literal,
 * table-name or digit, so we need to fill in these gaps ourselves.
 *
 * The grammar for expressions (`expr`) also needed to be reworked, see below.
 *
 * This file is used by Grammar-Kit to generate the lexer, parser, node types and PSI classes for Android SQL.
 */

{
  parserClass="com.android.tools.idea.lang.androidSql.parser.AndroidSqlParser"
  parserUtilClass="com.android.tools.idea.lang.androidSql.parser.AndroidSqlParserUtil"
  extends="com.intellij.extapi.psi.ASTWrapperPsiElement"

  psiClassPrefix="AndroidSql"
  psiImplClassSuffix="Impl"
  psiPackage="com.android.tools.idea.lang.androidSql.psi"
  psiImplPackage="com.android.tools.idea.lang.androidSql.psi.impl"
  psiImplUtilClass="com.android.tools.idea.lang.androidSql.psi.PsiImplUtil"

  elementTypeHolderClass="com.android.tools.idea.lang.androidSql.psi.AndroidSqlPsiTypes"
  elementTypeClass="com.android.tools.idea.lang.androidSql.psi.AndroidSqlAstNodeType"
  tokenTypeClass="com.android.tools.idea.lang.androidSql.psi.AndroidSqlTokenType"

  classHeader='generatedFilesHeader.txt'

  tokens=[
    // In the flex file we manually insert %caseless, in the preview only upper case keywords work.

    NUMERIC_LITERAL='regexp:(([0-9]+(\.[0-9]*)?|\.[0-9]+)(E(\+|-)?[0-9]+)?)|(0x[0-9a-f]+)'

    // Manually tweaked in the flex file to handle literals without the closing character.
    SINGLE_QUOTE_STRING_LITERAL="regexp:X?'(''|[^'])*'"
    DOUBLE_QUOTE_STRING_LITERAL="regexp:X?\"(\"\"|[^\"])*\""
    BRACKET_LITERAL="regexp:\[[^\]]*\]"
    BACKTICK_LITERAL="regexp:`(``|[^`])*`"

    // Some approximation of what's actually allowed. This doesn't seem to be defined anywhere.
    IDENTIFIER='regexp:([:letter:]|_)([:letter:]|[:digit:]|_)*'
    NAMED_PARAMETER='regexp::\w+' // Simple regexp for Live Preview, this is tweaked in the flex file.
    COMMENT='regexp:/\*.*\*/' // Simple regexp for Live Preview, this is tweaked in the flex file.
    LINE_COMMENT='regexp:--[^r\n]*'

    AMP='&'
    BAR='|'
    COMMA=','
    CONCAT='||'
    DIV='/'
    DOT='.'
    EQ='='
    EQEQ='=='
    GT='>'
    GTE='>='
    LPAREN='('
    LT='<'
    LTE='<='
    MINUS='-'
    MOD='%'
    NOT_EQ='!='
    PLUS='+'
    RPAREN=')'
    SEMICOLON=';'
    SHL='<<'
    SHR='>>'
    STAR='*'
    TILDE='~'
    UNEQ='<>'

    // More keywords from https://sqlite.org/lang_keywords.html that don't appear in the grammar below, but we want to treat them as
    // keywords (so that the IDE escapes them just in case).
    FULL='FULL'
    RIGHT='RIGHT'

    // Grammar-Kit's live preview emulates PsiBuilder's behavior of ignoring whitespace, by
    // looking for a token type that matches a space and is not used otherwise. Here's one:
    WHITE_SPACE_FOR_LIVE_PREVIEW='regexp:\s+'
  ]

  extends(".*_expression")=expression

  implements(".*_name")="com.android.tools.idea.lang.androidSql.psi.AndroidSqlNameElement"
  mixin(".*_name")="com.android.tools.idea.lang.androidSql.psi.AbstractAndroidSqlNameElement"
  generateTokenAccessors(".*_name")=false
  generate=[names='long' java='8']
}

root ::= statement ';'?

private statement ::=
  explain_prefix ?
  (
  select_statement
  | update_statement
  | insert_statement
  | delete_statement
  | with_clause_statement
  | alter_table_statement
  | analyze_statement
  | attach_statement
  | begin_statement
  | commit_statement
  | create_index_statement
  | create_table_statement
  | create_trigger_statement
  | create_view_statement
  | create_virtual_table_statement
  | detach_statement
  | drop_index_statement
  | drop_table_statement
  | drop_trigger_statement
  | drop_view_statement
  | pragma_statement
  | reindex_statement
  | release_statement
  | rollback_statement
  | savepoint_statement
  | vacuum_statement
  ) { name="statement" }

explain_prefix ::= EXPLAIN ( QUERY PLAN )?

with_clause_statement ::= with_clause (delete_statement | insert_statement | update_statement | select_statement) {
  implements="com.android.tools.idea.lang.androidSql.psi.HasWithClause"
}

with_clause_select_statement ::= with_clause select_statement {
  implements="com.android.tools.idea.lang.androidSql.psi.HasWithClause"
}

// Variant of with_clause_select_statement that is inside parens and consumes all input until the matching paren.
private subquery_greedy ::= select_statement | with_clause_select_statement {
  recoverWhile=subquery_recover
}

private subquery_recover ::= !')'

private rename_table_statement ::= RENAME TO table_definition_name
private rename_column_statement ::= RENAME COLUMN? column_name TO name
private add_column_statement ::= ADD COLUMN? column_definition
alter_table_statement ::= ALTER TABLE single_table_statement_table ( rename_table_statement | rename_column_statement | add_column_statement )

// Name of a table that is already in scope for the current query.
selected_table_name ::= name {
  methods=[getReference]
}

// Name of a table defined in the schema or using a WITH clause.
defined_table_name ::= name {
  methods=[getReference]
}

// Name of a table being defined.
table_definition_name ::= name {
  implements="com.intellij.psi.PsiNamedElement"
  methods=[getName setName]
}

database_name ::= name | TEMP

analyze_statement ::= ANALYZE ( database_name | table_or_index_name | database_name '.' table_or_index_name )?

table_or_index_name ::= name

attach_statement ::= ATTACH ( DATABASE )? expression AS database_name

begin_statement ::= BEGIN ( DEFERRED | IMMEDIATE | EXCLUSIVE )? ( TRANSACTION )?

commit_statement ::= ( COMMIT | END ) ( TRANSACTION )?

rollback_statement ::= ROLLBACK ( TRANSACTION )? ( TO ( SAVEPOINT )? savepoint_name )?

savepoint_name ::= name

savepoint_statement ::= SAVEPOINT savepoint_name

release_statement ::= RELEASE ( SAVEPOINT )? savepoint_name

create_index_statement ::=
  CREATE ( UNIQUE )? INDEX ( IF NOT EXISTS )?
  ( database_name '.' )? index_name ON defined_table_name '(' indexed_column ( ',' indexed_column )* ')'
  where_clause?

private index_name ::= name

indexed_column ::= column_name ( COLLATE collation_name )? ( ASC | DESC )?

column_name ::= name {
  methods=[getReference]
}

column_definition_name ::= name {
  implements="com.intellij.psi.PsiNamedElement"
  methods=[getName setName]
}

collation_name ::= name

create_table_statement ::=
  CREATE ( TEMP | TEMPORARY )? TABLE ( IF NOT EXISTS )?
  ( database_name '.' )? table_definition_name
  ( '(' column_definition ( ',' column_definition )* ( ',' table_constraint )* ')' ( WITHOUT "ROWID" )? | AS (select_statement | with_clause_select_statement) )

column_definition ::= column_definition_name ( type_name )? ( column_constraint )*

type_name ::= name ( '(' signed_number ')' | '(' signed_number ',' signed_number ')' )?

column_constraint ::=
  ( CONSTRAINT  name )?
  ( PRIMARY KEY ( ASC | DESC )? conflict_clause ( AUTOINCREMENT )?
  | NOT NULL conflict_clause
  | UNIQUE conflict_clause
  | CHECK '(' expression ')'
  | DEFAULT ( signed_number | literal_value | '(' expression ')' )
  | COLLATE collation_name | foreign_key_clause )

signed_number ::= ( '+' | '-' )? NUMERIC_LITERAL

table_constraint ::=
  ( CONSTRAINT  name )?
  ( ( PRIMARY KEY | UNIQUE ) '(' indexed_column ( ',' indexed_column )* ')' conflict_clause
  | CHECK '(' expression ')'
  | FOREIGN KEY '(' column_name ( ',' column_name )* ')' foreign_key_clause )

foreign_key_clause ::=
  REFERENCES foreign_table ( '(' column_name ( ',' column_name )* ')' )?
  ( ( ON ( DELETE | UPDATE ) ( SET NULL | SET DEFAULT | CASCADE | RESTRICT | NO ACTION ) | MATCH  name ) )*
  ( ( NOT )? DEFERRABLE ( INITIALLY DEFERRED | INITIALLY IMMEDIATE )? )?

foreign_table ::= name

conflict_clause ::= ( ON CONFLICT ( ROLLBACK | ABORT | FAIL | IGNORE | REPLACE ) )?

create_trigger_statement ::=
  CREATE ( TEMP | TEMPORARY )? TRIGGER ( IF NOT EXISTS )?
  ( database_name '.' )? trigger_name ( BEFORE | AFTER | INSTEAD OF )?
  ( DELETE | INSERT | UPDATE ( OF column_name ( ',' column_name )* )? ) ON defined_table_name
  ( FOR EACH ROW )? ( WHEN expression )?
  BEGIN with_clause? ( update_statement | insert_statement | delete_statement | select_statement ) ';' END

trigger_name ::= name

create_view_statement ::=
  CREATE ( TEMP | TEMPORARY )? VIEW ( IF NOT EXISTS )?
  ( database_name '.' )? view_name AS (select_statement | with_clause_select_statement)

view_name ::= name

create_virtual_table_statement ::=
  CREATE VIRTUAL TABLE ( IF NOT EXISTS )?
  ( database_name '.' )? table_definition_name
  USING module_name ( '(' module_argument ( ',' module_argument )* ')' )?

module_name ::= name
module_argument ::= name

with_clause ::= &WITH with_clause_greedy

private with_clause_greedy ::= WITH ( RECURSIVE )? with_clause_table ( ',' with_clause_table )* {
  recoverWhile=with_clause_recover
  pin=1
}

private with_clause_recover ::= !(DELETE | INSERT | REPLACE | SELECT | UPDATE | VALUES | ')')

with_clause_table ::= with_clause_table_def AS with_clause_table_def_subquery  {
  methods=[getTableDefinition]
}
private with_clause_table_def_subquery ::= '(' &(SELECT|VALUES|WITH) subquery_greedy ')' { pin=2 }

with_clause_table_def ::= table_definition_name ( '(' column_definition_name ( ',' column_definition_name )* ')' )?

delete_statement ::=
  DELETE FROM single_table_statement_table ( INDEXED BY index_name | NOT INDEXED )?
  where_clause?
  ( order_clause? LIMIT expression ( ( OFFSET | ',' ) expression )? )? {
  }

detach_statement ::= DETACH ( DATABASE )? database_name

drop_index_statement ::= DROP INDEX ( IF EXISTS )? ( database_name '.' )? index_name

drop_table_statement ::= DROP TABLE ( IF EXISTS )? ( database_name '.' )? defined_table_name

drop_trigger_statement ::= DROP TRIGGER ( IF EXISTS )? ( database_name '.' )? trigger_name

drop_view_statement ::= DROP VIEW ( IF EXISTS )? ( database_name '.' )? view_name

// Below is the original, left-recursive grammar for expressions, as pictured on the diagram: http://www.sqlite.org/syntaxdiagrams.html#expr
//
// It needed to be changed to a form that Grammar-Kit accepts (due to left recursion), see this link for details:
// https://github.com/JetBrains/Grammar-Kit/blob/master/HOWTO.md#24-compact-expression-parsing-with-priorities
//
// Operator precedence taken from http://www.sqlite.org/lang_expr.html
//
//expr ::=
//  literal_value
//  | bind_parameter
//  | ( ( database_name '.' )? table_name '.' )? column_name
//  | unary_operator expr
//  | expr binary_operator expr
//  | function_name '(' ( ( DISTINCT )? expr ( ',' expr )* | '*' )? ')'
//  | '(' expression ( ',' expression )* ')'
//  | CAST '(' expr AS type_name ')'
//  | expr COLLATE collation_name
//  | expr ( NOT )? ( LIKE | GLOB | REGEXP | MATCH ) expr ( ESCAPE expr )?
//  | expr ( ISNULL | NOTNULL | NOT NULL )
//  | expr IS ( NOT )? expr
//  | expr ( NOT )? BETWEEN expr AND expr
//  | expr ( NOT )? IN ( '(' ( select_statement | expr ( ',' expr )* )? ')' | ( database_name '.' )? table_name )
//  | ( ( NOT )? EXISTS )? '(' select_statement ')'
//  | CASE ( expr )? WHEN expr THEN expr ( ELSE expr )? END
//  | raise_function

expression ::=
  raise_function_expression
  | or_expression
  | and_expression
  | case_expression
  | exists_expression
  | in_expression
  | isnull_expression
  | like_expression
  | cast_expression
  | function_call_expression
  | equivalence_group
  | comparison_expression
  | bit_expression
  | add_expression
  | mul_expression
  | concat_expression
  | unary_expression
  | collate_expression // "The COLLATE operator has a higher precedence (binds more tightly) than any binary operator and any unary prefix operator..."
  | literal_expression
  | column_ref_expression
  | paren_expression

// "The precedence of the BETWEEN operator is the same as the precedence as operators == and != and LIKE and groups left to right."
private equivalence_group ::= equivalence_expression | between_expression

and_expression ::= expression AND expression
or_expression ::= expression OR expression
equivalence_expression ::= expression ( '==' | '=' | '!=' | '<>' | IS NOT?) expression
comparison_expression ::= expression ( '<' | '<=' | '>' | '>=' ) expression
bit_expression ::= expression ( '<<' | '>>' | '&' | '|' ) expression
add_expression ::= expression ( '+' | '-' ) expression
mul_expression ::= expression ( '*' | '/' | '%' ) expression
concat_expression ::= expression '||' expression
unary_expression ::= ('-' | '+' | '~' | NOT) expression
literal_expression ::= literal_value | bind_parameter
paren_expression ::= '(' expression ( ',' expression )* ')'
collate_expression ::= expression COLLATE collation_name
between_expression ::= expression NOT? BETWEEN expression AND expression
cast_expression ::= CAST '(' expression AS type_name ')'
case_expression ::= CASE expression? ( WHEN expression THEN expression )+ ( ELSE expression )? END
like_expression ::= expression NOT? ( LIKE | GLOB | REGEXP | MATCH ) expression ( ESCAPE expression )?
isnull_expression ::= expression ( ISNULL | NOTNULL | NOT NULL )
in_expression ::=  expression ( NOT )? IN ( '(' ( expression_subquery | expression ( ',' expression )* )? ')' | ( database_name '.' )? defined_table_name )
exists_expression ::= ( ( NOT )? EXISTS )? '(' expression_subquery ')'
function_call_expression ::= function_name '(' ( ( DISTINCT )? expression ( ',' expression )* | '*' )? ')'

private expression_subquery ::= &(WITH|SELECT|VALUES) subquery_greedy {pin=1}

column_ref_expression ::=
  database_name '.' selected_table_name '.' column_name
  | selected_table_name '.' column_name
  | column_name

// Some tokens are allowed as function names and remapped to identifiers in code
external function_name ::= parseFunctionName

bind_parameter ::= NUMBERED_PARAMETER | NAMED_PARAMETER {
  methods=[
    getParameterNameAsString
    getReference
    isColonNamedParameter
  ]
}

raise_function_expression ::= RAISE '(' ( IGNORE | ( ROLLBACK | ABORT | FAIL ) ',' error_message ) ')'

error_message ::= string_literal // TODO: check

boolean_literal ::= TRUE | FALSE

private literal_value ::=
  NUMERIC_LITERAL
  | string_literal // X marks a blob literal
  | boolean_literal
  | NULL
  | CURRENT_TIME
  | CURRENT_DATE
  | CURRENT_TIMESTAMP

insert_statement ::=
  ( INSERT ( OR ( REPLACE |  ROLLBACK |  ABORT |  FAIL |  IGNORE ))? | REPLACE ) INTO
  single_table_statement_table insert_columns?
  ( select_statement | with_clause_select_statement | DEFAULT VALUES )

insert_columns ::= '(' column_name ( ',' column_name )* ')'

pragma_statement ::= PRAGMA ( database_name '.' )? pragma_name ( '=' pragma_value | '(' pragma_value ')' )?

pragma_name ::= name // TODO: check

// SQLite has special parsing code for pragmas, we try to keep up by listing keywords that happen to be allowed values for certain pragmas.
// See https://sqlite.org/pragma.html
pragma_value ::= signed_number | name | string_literal | boolean_literal | ON | NO | FULL | DELETE | EXCLUSIVE | DEFAULT

reindex_statement ::= REINDEX ( collation_name | ( database_name '.' )? ( defined_table_name | index_name ) )?

select_statement ::= select_core (compound_operator select_core)* order_clause? limit_clause?

limit_clause ::= LIMIT expression ( ( OFFSET | ',' ) expression )?

order_clause ::= ORDER BY ordering_term ( ',' ordering_term )*

select_core ::= select_core_select | select_core_values

select_core_select ::= SELECT ( DISTINCT | ALL )? result_columns from_clause? where_clause? group_by_clause?

group_by_clause ::= GROUP BY expression ( ',' expression )* ( HAVING expression )?

where_clause ::= WHERE expression

from_clause ::= FROM table_or_subquery ( join_operator table_or_subquery join_constraint? )*

result_columns ::= result_column ( ',' result_column )* {
  implements="com.android.tools.idea.lang.androidSql.psi.AndroidSqlTableElement"
  methods=[getSqlTable]
}

select_core_values ::= VALUES '(' expression ( ',' expression )* ')' ( ',' '(' expression ( ',' expression )* ')' )*

table_or_subquery ::= from_table | select_subquery | '(' table_or_subquery ')'

from_table ::= ( database_name '.' )? defined_table_name ( ( AS )? table_alias_name )? ( INDEXED BY index_name | NOT INDEXED )? {
  implements="com.android.tools.idea.lang.androidSql.psi.AndroidSqlTableElement"
  methods=[getSqlTable]
}

select_subquery ::= '(' &(SELECT|VALUES|WITH) subquery_greedy ')' ( ( AS )? table_alias_name )? {
  implements="com.android.tools.idea.lang.androidSql.psi.AndroidSqlTableElement"
  methods=[getSqlTable]
  pin=2
}

table_alias_name ::= name {
  implements="com.intellij.psi.PsiNamedElement"
  methods=[getName setName]
}

result_column ::=
  '*'
  | selected_table_name '.' '*'
  | expression ( ( AS )? column_alias_name )?

column_alias_name ::= name {
  implements="com.intellij.psi.PsiNamedElement"
  methods=[getName setName]
}

join_operator ::= ',' | ( NATURAL )? ( LEFT ( OUTER )? | INNER | CROSS )? JOIN

join_constraint ::= ON expression | USING '(' column_name ( ',' column_name )* ')'

ordering_term ::= expression ( COLLATE collation_name )? ( ASC | DESC )?

compound_operator ::= UNION ALL? | INTERSECT | EXCEPT

update_statement ::=
  UPDATE ( OR ROLLBACK | OR ABORT | OR REPLACE | OR FAIL | OR IGNORE )? single_table_statement_table ( INDEXED BY index_name | NOT INDEXED )?
  SET column_name '=' expression ( ',' column_name '=' expression )*
  where_clause? { pin=1 }

single_table_statement_table ::= ( database_name '.' )? defined_table_name {
  implements="com.android.tools.idea.lang.androidSql.psi.AndroidSqlTableElement"
  methods=[getSqlTable]
}

vacuum_statement ::= VACUUM

private name ::= IDENTIFIER | BRACKET_LITERAL | BACKTICK_LITERAL | string_literal
private string_literal ::= SINGLE_QUOTE_STRING_LITERAL | DOUBLE_QUOTE_STRING_LITERAL