Merge branch 'main' into pr/parse_column_def

Author: nickolay

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: cargo
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+  - package-ecosystem: cargo
+    directory: "/sqlparser_bench"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10

.github/workflows/rust.yml

@@ -15,7 +15,7 @@ jobs:
           # it's an unstable feature.
           rust-version: nightly
       - uses: actions/checkout@v2
-      - run: cargo fmt -- --check --config-path <(echo 'license_template_path = "HEADER"')
+      - run: cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"')
 
   lint:
     runs-on: ubuntu-latest
@@ -49,7 +49,7 @@ jobs:
       uses: actions-rs/[email protected]
       with:
         crate: cargo-tarpaulin
-        version: 0.13.3
+        version: 0.14.2
         use-tool-cache: true
     - name: Checkout
       uses: actions/checkout@v2

Cargo.toml

@@ -23,14 +23,14 @@ path = "src/lib.rs"
 json_example = ["serde_json", "serde"]
 
 [dependencies]
-bigdecimal = { version = "0.1.0", features = ["serde"], optional = true }
-log = "0.4.5"
+bigdecimal = { version = "0.1", features = ["serde"], optional = true }
+log = "0.4"
 serde = { version = "1.0", features = ["derive"], optional = true }
 # serde_json is only used in examples/cli, but we have to put it outside
 # of dev-dependencies because of
 # https://github.com/rust-lang/cargo/issues/1596
 serde_json = { version = "1.0", optional = true }
 
 [dev-dependencies]
-simple_logger = "1.0.1"
+simple_logger = "1.6"
 matches = "0.1"

src/ast/mod.rs

@@ -873,12 +873,28 @@ impl fmt::Display for Assignment {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum FunctionArg {
+    Named { name: Ident, arg: Expr },
+    Unnamed(Expr),
+}
+
+impl fmt::Display for FunctionArg {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            FunctionArg::Named { name, arg } => write!(f, "{} => {}", name, arg),
+            FunctionArg::Unnamed(unnamed_arg) => write!(f, "{}", unnamed_arg),
+        }
+    }
+}
+
 /// A function call
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct Function {
     pub name: ObjectName,
-    pub args: Vec<Expr>,
+    pub args: Vec<FunctionArg>,
     pub over: Option<WindowSpec>,
     // aggregate functions may specify eg `COUNT(DISTINCT x)`
     pub distinct: bool,

src/ast/query.rs

@@ -226,7 +226,7 @@ pub enum TableFactor {
         /// Arguments of a table-valued function, as supported by Postgres
         /// and MSSQL. Note that deprecated MSSQL `FROM foo (NOLOCK)` syntax
         /// will also be parsed as `args`.
-        args: Vec<Expr>,
+        args: Vec<FunctionArg>,
         /// MSSQL-specific `WITH (...)` hints such as NOLOCK.
         with_hints: Vec<Expr>,
     },
@@ -235,6 +235,11 @@ pub enum TableFactor {
         subquery: Box<Query>,
         alias: Option<TableAlias>,
     },
+    /// `TABLE(<expr>)[ AS <alias> ]`
+    TableFunction {
+        expr: Expr,
+        alias: Option<TableAlias>,
+    },
     /// Represents a parenthesized table factor. The SQL spec only allows a
     /// join expression (`(foo <JOIN> bar [ <JOIN> baz ... ])`) to be nested,
     /// possibly several times, but the parser also accepts the non-standard
@@ -278,6 +283,13 @@ impl fmt::Display for TableFactor {
                 }
                 Ok(())
             }
+            TableFactor::TableFunction { expr, alias } => {
+                write!(f, "TABLE({})", expr)?;
+                if let Some(alias) = alias {
+                    write!(f, " AS {}", alias)?;
+                }
+                Ok(())
+            }
             TableFactor::NestedJoin(table_reference) => write!(f, "({})", table_reference),
         }
     }

src/dialect/mod.rs

@@ -18,6 +18,7 @@ mod mysql;
 mod postgresql;
 mod sqlite;
 
+use std::any::{Any, TypeId};
 use std::fmt::Debug;
 
 pub use self::ansi::AnsiDialect;
@@ -27,7 +28,15 @@ pub use self::mysql::MySqlDialect;
 pub use self::postgresql::PostgreSqlDialect;
 pub use self::sqlite::SQLiteDialect;
 
-pub trait Dialect: Debug {
+/// `dialect_of!(parser is SQLiteDialect |  GenericDialect)` evaluates
+/// to `true` iff `parser.dialect` is one of the `Dialect`s specified.
+macro_rules! dialect_of {
+    ( $parsed_dialect: ident is $($dialect_type: ty)|+ ) => {
+        ($($parsed_dialect.dialect.is::<$dialect_type>())||+)
+    };
+}
+
+pub trait Dialect: Debug + Any {
     /// Determine if a character starts a quoted identifier. The default
     /// implementation, accepting "double quoted" ids is both ANSI-compliant
     /// and appropriate for most dialects (with the notable exception of
@@ -41,3 +50,51 @@ pub trait Dialect: Debug {
     /// Determine if a character is a valid unquoted identifier character
     fn is_identifier_part(&self, ch: char) -> bool;
 }
+
+impl dyn Dialect {
+    #[inline]
+    pub fn is<T: Dialect>(&self) -> bool {
+        // borrowed from `Any` implementation
+        TypeId::of::<T>() == self.type_id()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::ansi::AnsiDialect;
+    use super::generic::GenericDialect;
+    use super::*;
+
+    struct DialectHolder<'a> {
+        dialect: &'a dyn Dialect,
+    }
+
+    #[test]
+    fn test_is_dialect() {
+        let generic_dialect: &dyn Dialect = &GenericDialect {};
+        let ansi_dialect: &dyn Dialect = &AnsiDialect {};
+
+        let generic_holder = DialectHolder {
+            dialect: generic_dialect,
+        };
+        let ansi_holder = DialectHolder {
+            dialect: ansi_dialect,
+        };
+
+        assert_eq!(
+            dialect_of!(generic_holder is GenericDialect |  AnsiDialect),
+            true
+        );
+        assert_eq!(dialect_of!(generic_holder is  AnsiDialect), false);
+
+        assert_eq!(dialect_of!(ansi_holder is  AnsiDialect), true);
+        assert_eq!(
+            dialect_of!(ansi_holder is  GenericDialect | AnsiDialect),
+            true
+        );
+        assert_eq!(
+            dialect_of!(ansi_holder is  GenericDialect | MsSqlDialect),
+            false
+        );
+    }
+}

src/lib.rs

@@ -35,6 +35,7 @@
 #![warn(clippy::all)]
 
 pub mod ast;
+#[macro_use]
 pub mod dialect;
 pub mod parser;
 pub mod tokenizer;

src/parser.rs

@@ -15,9 +15,8 @@
 use log::debug;
 
 use super::ast::*;
-use super::dialect::keywords;
 use super::dialect::keywords::Keyword;
-use super::dialect::Dialect;
+use super::dialect::*;
 use super::tokenizer::*;
 use std::error::Error;
 use std::fmt;
@@ -82,24 +81,28 @@ impl fmt::Display for ParserError {
 
 impl Error for ParserError {}
 
-/// SQL Parser
-pub struct Parser {
+pub struct Parser<'a> {
     tokens: Vec<Token>,
     /// The index of the first unprocessed token in `self.tokens`
     index: usize,
+    dialect: &'a dyn Dialect,
 }
 
-impl Parser {
+impl<'a> Parser<'a> {
     /// Parse the specified tokens
-    pub fn new(tokens: Vec<Token>) -> Self {
-        Parser { tokens, index: 0 }
+    pub fn new(tokens: Vec<Token>, dialect: &'a dyn Dialect) -> Self {
+        Parser {
+            tokens,
+            index: 0,
+            dialect,
+        }
     }
 
     /// Parse a SQL statement and produce an Abstract Syntax Tree (AST)
     pub fn parse_sql(dialect: &dyn Dialect, sql: &str) -> Result<Vec<Statement>, ParserError> {
         let mut tokenizer = Tokenizer::new(dialect, &sql);
         let tokens = tokenizer.tokenize()?;
-        let mut parser = Parser::new(tokens);
+        let mut parser = Parser::new(tokens, dialect);
         let mut stmts = Vec::new();
         let mut expecting_statement_delimiter = false;
         debug!("Parsing sql '{}'...", sql);
@@ -950,7 +953,7 @@ impl Parser {
     /// Parse a comma-separated list of 1+ items accepted by `F`
     pub fn parse_comma_separated<T, F>(&mut self, mut f: F) -> Result<Vec<T>, ParserError>
     where
-        F: FnMut(&mut Parser) -> Result<T, ParserError>,
+        F: FnMut(&mut Parser<'a>) -> Result<T, ParserError>,
     {
         let mut values = vec![];
         loop {
@@ -1289,12 +1292,14 @@ impl Parser {
             let expr = self.parse_expr()?;
             self.expect_token(&Token::RParen)?;
             Ok(Some(ColumnOption::Check(expr)))
-        } else if self.parse_keyword(Keyword::AUTO_INCREMENT) {
+        } else if self.parse_keyword(Keyword::AUTO_INCREMENT)
+            && dialect_of!(self is MySqlDialect |  GenericDialect)
+        {
             // Support AUTO_INCREMENT for MySQL
-            Ok(Some(ColumnOption::DialectSpecific(vec![
-                Token::make_keyword("AUTO_INCREMENT"),
-            ])))
-        } else if self.parse_keyword(Keyword::AUTOINCREMENT) {
+            Ok(Some(ColumnOption::DialectSpecific(vec![Token::make_keyword("AUTO_INCREMENT")])))
+        } else if self.parse_keyword(Keyword::AUTOINCREMENT)
+            && dialect_of!(self is SQLiteDialect |  GenericDialect)
+        {
             // Support AUTOINCREMENT for SQLite
             Ok(Some(ColumnOption::DialectSpecific(vec![
                 Token::make_keyword("AUTOINCREMENT"),
@@ -2072,10 +2077,15 @@ impl Parser {
             if !self.consume_token(&Token::LParen) {
                 self.expected("subquery after LATERAL", self.peek_token())?;
             }
-            return self.parse_derived_table_factor(Lateral);
-        }
-
-        if self.consume_token(&Token::LParen) {
+            self.parse_derived_table_factor(Lateral)
+        } else if self.parse_keyword(Keyword::TABLE) {
+            // parse table function (SELECT * FROM TABLE (<expr>) [ AS <alias> ])
+            self.expect_token(&Token::LParen)?;
+            let expr = self.parse_expr()?;
+            self.expect_token(&Token::RParen)?;
+            let alias = self.parse_optional_table_alias(keywords::RESERVED_FOR_TABLE_ALIAS)?;
+            Ok(TableFactor::TableFunction { expr, alias })
+        } else if self.consume_token(&Token::LParen) {
             // A left paren introduces either a derived table (i.e., a subquery)
             // or a nested join. It's nearly impossible to determine ahead of
             // time which it is... so we just try to parse both.
@@ -2209,11 +2219,24 @@ impl Parser {
         Ok(Assignment { id, value })
     }
 
-    pub fn parse_optional_args(&mut self) -> Result<Vec<Expr>, ParserError> {
+    fn parse_function_args(&mut self) -> Result<FunctionArg, ParserError> {
+        if self.peek_nth_token(1) == Token::RArrow {
+            let name = self.parse_identifier()?;
+
+            self.expect_token(&Token::RArrow)?;
+            let arg = self.parse_expr()?;
+
+            Ok(FunctionArg::Named { name, arg })
+        } else {
+            Ok(FunctionArg::Unnamed(self.parse_expr()?))
+        }
+    }
+
+    pub fn parse_optional_args(&mut self) -> Result<Vec<FunctionArg>, ParserError> {
         if self.consume_token(&Token::RParen) {
             Ok(vec![])
         } else {
-            let args = self.parse_comma_separated(Parser::parse_expr)?;
+            let args = self.parse_comma_separated(Parser::parse_function_args)?;
             self.expect_token(&Token::RParen)?;
             Ok(args)
         }

src/test_utils.rs

@@ -53,7 +53,7 @@ impl TestedDialects {
         self.one_of_identical_results(|dialect| {
             let mut tokenizer = Tokenizer::new(dialect, sql);
             let tokens = tokenizer.tokenize().unwrap();
-            f(&mut Parser::new(tokens))
+            f(&mut Parser::new(tokens, dialect))
         })
     }
 
@@ -104,7 +104,9 @@ impl TestedDialects {
     /// Ensures that `sql` parses as an expression, and is not modified
     /// after a serialization round-trip.
     pub fn verified_expr(&self, sql: &str) -> Expr {
-        let ast = self.run_parser_method(sql, Parser::parse_expr).unwrap();
+        let ast = self
+            .run_parser_method(sql, |parser| parser.parse_expr())
+            .unwrap();
         assert_eq!(sql, &ast.to_string(), "round-tripping without changes");
         ast
     }

src/tokenizer.rs

@@ -99,6 +99,8 @@ pub enum Token {
     LBrace,
     /// Right brace `}`
     RBrace,
+    /// Right Arrow `=>`
+    RArrow,
 }
 
 impl fmt::Display for Token {
@@ -139,6 +141,7 @@ impl fmt::Display for Token {
             Token::Pipe => f.write_str("|"),
             Token::LBrace => f.write_str("{"),
             Token::RBrace => f.write_str("}"),
+            Token::RArrow => f.write_str("=>"),
         }
     }
 }
@@ -400,7 +403,13 @@ impl<'a> Tokenizer<'a> {
                         _ => Ok(Some(Token::Pipe)),
                     }
                 }
-                '=' => self.consume_and_return(chars, Token::Eq),
+                '=' => {
+                    chars.next(); // consume
+                    match chars.peek() {
+                        Some('>') => self.consume_and_return(chars, Token::RArrow),
+                        _ => Ok(Some(Token::Eq)),
+                    }
+                }
                 '.' => self.consume_and_return(chars, Token::Period),
                 '!' => {
                     chars.next(); // consume
@@ -766,6 +775,23 @@ mod tests {
         compare(expected, tokens);
     }
 
+    #[test]
+    fn tokenize_right_arrow() {
+        let sql = String::from("FUNCTION(key=>value)");
+        let dialect = GenericDialect {};
+        let mut tokenizer = Tokenizer::new(&dialect, &sql);
+        let tokens = tokenizer.tokenize().unwrap();
+        let expected = vec![
+            Token::make_word("FUNCTION", None),
+            Token::LParen,
+            Token::make_word("key", None),
+            Token::RArrow,
+            Token::make_word("value", None),
+            Token::RParen,
+        ];
+        compare(expected, tokens);
+    }
+
     #[test]
     fn tokenize_is_null() {
         let sql = String::from("a IS NULL");