From 82bcb446d492fd12eb6d63630dffc2af3c6c22ad Mon Sep 17 00:00:00 2001 From: Alex Sladkov Date: Mon, 14 Feb 2022 18:47:11 +0300 Subject: [PATCH 1/4] UUID, Array(T) and DateTime64 types support --- src/lib.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++++ tests/lib.rs | 9 +++++++-- tests/tables2.sql | 12 +++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4ef8d42..9ea5a2c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -185,11 +185,14 @@ pub enum SqlType { Enum(Option, Vec<(String, i16)>), Date, DateTime(Option), + DateTime64(u8, Option), Float32, Float64, FixedString(usize), IPv4, IPv6, + UUID, + Array(Box), } impl fmt::Display for SqlType { @@ -209,11 +212,15 @@ impl fmt::Display for SqlType { SqlType::Date => write!(f, "Date"), SqlType::DateTime(None) => write!(f, "DateTime"), SqlType::DateTime(Some(timezone)) => write!(f, "DateTime({})", timezone), + SqlType::DateTime64(precision, None) => write!(f, "DateTime64({})", precision), + SqlType::DateTime64(precision, Some(timezone)) => write!(f, "DateTime64({}, {})", precision, timezone), SqlType::Float32 => write!(f, "Float32"), SqlType::Float64 => write!(f, "Float64"), SqlType::FixedString(size) => write!(f, "FixedString({})", size), SqlType::IPv4 => write!(f, "IPv4"), SqlType::IPv6 => write!(f, "IPv6"), + SqlType::UUID => write!(f, "UUID"), + SqlType::Array(t) => write!(f, "Array({})", t), } } } @@ -434,6 +441,27 @@ fn type_identifier(i: &[u8]) -> IResult<&[u8], SqlType> { map(tag_no_case("string"), |_| SqlType::String), map(tag_no_case("float32"), |_| SqlType::Float32), map(tag_no_case("float64"), |_| SqlType::Float64), + map( + tuple(( + tag_no_case("datetime64"), + multispace0, + tag("("), + multispace0, + one_of("0123456789"), + multispace0, + opt(map( + tuple(( + tag(","), + multispace0, + delimited(tag("'"), take_until("'"), tag("'")), + )), + |(_, _, timezone)| str::from_utf8(timezone).unwrap().to_string() + )), + multispace0, + tag(")"), + )), + |(_, _, _, _, precision, _, timezone, _, _)| SqlType::DateTime64(precision.to_digit(10).unwrap() as u8, timezone) + ), map( tuple(( tag_no_case("datetime"), @@ -461,6 +489,16 @@ fn type_identifier(i: &[u8]) -> IResult<&[u8], SqlType> { ), map(tag_no_case("ipv4"), |_| SqlType::IPv4), map(tag_no_case("ipv6"), |_| SqlType::IPv6), + map(tag_no_case("uuid"), |_| SqlType::UUID), + map( + tuple(( + tag_no_case("array"), + tag("("), + type_identifier, + tag(")"), + )), + |(_,_,t,_)| SqlType::Array(Box::new(t)) + ), ))(i) } @@ -531,11 +569,24 @@ mod test { ( "Float32", SqlType::Float32 ), ( "Float64", SqlType::Float64 ), + ( "DateTime64(9)", SqlType::DateTime64(9, None) ), + ( "DateTime64( 3 ,'Etc/UTC' )", SqlType::DateTime64(3, Some("Etc/UTC".into())) ), + ( "DateTime", SqlType::DateTime(None) ), ( "DateTime('Cont/City')", SqlType::DateTime(Some("Cont/City".into())) ), ( "DateTime ( 'Cont/City')", SqlType::DateTime(Some("Cont/City".into())) ), ( "FixedString(3)", SqlType::FixedString(3) ), + + ( "UUID", SqlType::UUID ), + ( "Array(FixedString(2))", SqlType::Array(Box::new(SqlType::FixedString(2))) ), + ( "Array(Array(Array(Int64)))", SqlType::Array(Box::new( + SqlType::Array(Box::new( + SqlType::Array(Box::new( + SqlType::Int(TypeSize::B64), + )), + )), + )) ), ]; parse_set_for_test(type_identifier, patterns); } diff --git a/tests/lib.rs b/tests/lib.rs index ca5041e..673dc24 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -126,9 +126,14 @@ fn parse_file(path: &str) -> (i32, i32) { #[test] fn tables() { let (_ok, fail) = parse_file("tests/tables.sql"); - let (_ok, fail) = parse_file("tests/tables2.sql"); - assert_eq!(0, fail); + assert_eq!(0, fail); } +#[test] +fn tables2() { + let (_ok, fail) = parse_file("tests/tables2.sql"); + + assert_eq!(0, fail); +} diff --git a/tests/tables2.sql b/tests/tables2.sql index 3a9e39d..58f5ece 100644 --- a/tests/tables2.sql +++ b/tests/tables2.sql @@ -209,3 +209,15 @@ CREATE TABLE default.api3_http_request ( `osVersion` LowCardinality(Nullable(String)), `deviceId` Nullable(String) ) ENGINE = Distributed('nginx_cluster', '', 'api3_http_request', assumeNotNull(if(length(deviceId) > 1, murmurHash3_64(deviceId), rand()))); + +CREATE TABLE default.EventServerLoggerRequest +( + `uuid` UUID, + `searchId` Nullable(UUID), + `eventId` String, + `ip` UInt32, + `drivingLicence` Array(String), + `created` DateTime64(3) CODEC(DoubleDelta), + `loaded` DateTime64(6, 'Europe/Moscow') CODEC(DoubleDelta) +) +ENGINE = Distributed('nginx_cluster', '', 'EventServerRtbLoggerRequest', rand()); From 87e086af3c5de68359cb7ba6a271aafead184c7f Mon Sep 17 00:00:00 2001 From: Alex Sladkov Date: Tue, 15 Feb 2022 17:28:06 +0300 Subject: [PATCH 2/4] Allow `Array(Nullable(T))` and `Array(LowCardinality(T))` --- src/lib.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9ea5a2c..dd8009f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,6 +34,7 @@ use column::Column; use create::{ CreateTableStatement, creation, + field_specification_opts, }; fn eof>(input: I) -> IResult { @@ -192,7 +193,7 @@ pub enum SqlType { IPv4, IPv6, UUID, - Array(Box), + Array(Box), } impl fmt::Display for SqlType { @@ -494,7 +495,7 @@ fn type_identifier(i: &[u8]) -> IResult<&[u8], SqlType> { tuple(( tag_no_case("array"), tag("("), - type_identifier, + field_specification_opts, tag(")"), )), |(_,_,t,_)| SqlType::Array(Box::new(t)) @@ -558,6 +559,13 @@ mod test { #[test] fn t_type_identifier() { + fn t(nullable: bool, lowcardinality: bool, t: SqlType) -> Box { + Box::new(SqlTypeOpts { + ftype: t, + nullable: nullable, + lowcardinality: lowcardinality, + }) + } let patterns = vec![ ( "Int32", SqlType::Int(TypeSize::B32)), ( "UInt32", SqlType::UnsignedInt(TypeSize::B32)), @@ -579,10 +587,12 @@ mod test { ( "FixedString(3)", SqlType::FixedString(3) ), ( "UUID", SqlType::UUID ), - ( "Array(FixedString(2))", SqlType::Array(Box::new(SqlType::FixedString(2))) ), - ( "Array(Array(Array(Int64)))", SqlType::Array(Box::new( - SqlType::Array(Box::new( - SqlType::Array(Box::new( + ( "Array(FixedString(2))", SqlType::Array(t(false, false, SqlType::FixedString(2))) ), + ( "Array(Nullable(Int32))", SqlType::Array(t(true, false, SqlType::Int(TypeSize::B32))) ), + ( "Array(LowCardinality(String))", SqlType::Array(t(false, true, SqlType::String)) ), + ( "Array(Array(Array(Int64)))", SqlType::Array(t(false, false, + SqlType::Array(t(false, false, + SqlType::Array(t(false, false, SqlType::Int(TypeSize::B64), )), )), From 23b8803942f6ed7b6ae2f24d97044fcfd67a58f7 Mon Sep 17 00:00:00 2001 From: Alex Sladkov Date: Tue, 15 Feb 2022 18:02:45 +0300 Subject: [PATCH 3/4] Add helper methods for convenient constructing SqlType and SqlTypeOpts values --- src/lib.rs | 60 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index dd8009f..a5dd0ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -226,6 +226,16 @@ impl fmt::Display for SqlType { } } +impl SqlType { + pub fn array_from_sql_type(t: SqlType) -> SqlType { + SqlType::array_from_sql_type_opts(SqlTypeOpts::from_sql_type(t)) + } + + pub fn array_from_sql_type_opts(t: SqlTypeOpts) -> SqlType { + SqlType::Array(Box::new(t)) + } +} + #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct SqlTypeOpts { pub ftype: SqlType, @@ -244,6 +254,27 @@ impl fmt::Display for SqlTypeOpts{ } } +impl SqlTypeOpts { + pub fn from_sql_type(t: SqlType) -> SqlTypeOpts { + SqlTypeOpts { + ftype: t, + nullable: false, + lowcardinality: false, + } + } + + pub fn nullable(&self) -> SqlTypeOpts { + let mut r = self.clone(); + r.nullable = true; + r + } + + pub fn low_carinality(&self) -> SqlTypeOpts { + let mut r = self.clone(); + r.lowcardinality = true; + r + } +} fn ttl_expression(i: &[u8]) -> IResult<&[u8], &[u8]> { //date + INTERVAL 1 DAY @@ -559,13 +590,6 @@ mod test { #[test] fn t_type_identifier() { - fn t(nullable: bool, lowcardinality: bool, t: SqlType) -> Box { - Box::new(SqlTypeOpts { - ftype: t, - nullable: nullable, - lowcardinality: lowcardinality, - }) - } let patterns = vec![ ( "Int32", SqlType::Int(TypeSize::B32)), ( "UInt32", SqlType::UnsignedInt(TypeSize::B32)), @@ -587,16 +611,20 @@ mod test { ( "FixedString(3)", SqlType::FixedString(3) ), ( "UUID", SqlType::UUID ), - ( "Array(FixedString(2))", SqlType::Array(t(false, false, SqlType::FixedString(2))) ), - ( "Array(Nullable(Int32))", SqlType::Array(t(true, false, SqlType::Int(TypeSize::B32))) ), - ( "Array(LowCardinality(String))", SqlType::Array(t(false, true, SqlType::String)) ), - ( "Array(Array(Array(Int64)))", SqlType::Array(t(false, false, - SqlType::Array(t(false, false, - SqlType::Array(t(false, false, + ( "Array(FixedString(2))", SqlType::array_from_sql_type(SqlType::FixedString(2)) ), + ( "Array(Nullable(Int32))", SqlType::array_from_sql_type_opts( + SqlTypeOpts::from_sql_type(SqlType::Int(TypeSize::B32)).nullable() + ) ), + ( "Array(LowCardinality(String))", SqlType::array_from_sql_type_opts( + SqlTypeOpts::from_sql_type(SqlType::String).low_carinality() + ) ), + ( "Array(Array(Array(Int64)))", SqlType::array_from_sql_type( + SqlType::array_from_sql_type( + SqlType::array_from_sql_type( SqlType::Int(TypeSize::B64), - )), - )), - )) ), + ), + ), + ) ), ]; parse_set_for_test(type_identifier, patterns); } From bca990a2f1cd5aed482ac4faf572c36ff94402f2 Mon Sep 17 00:00:00 2001 From: Alex Sladkov Date: Wed, 16 Feb 2022 05:53:38 +0300 Subject: [PATCH 4/4] Support for Array(t) default values --- src/lib.rs | 17 +++++++++++++++++ tests/tables2.sql | 1 + 2 files changed, 18 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index a5dd0ce..535cff9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -323,6 +323,7 @@ fn sql_expression(i: &[u8]) -> IResult<&[u8], &[u8]> { sql_simple_expression, ))), sql_simple_expression, + sql_array, ))(i) } fn sql_simple_expression(i: &[u8]) -> IResult<&[u8], &[u8]> { @@ -351,6 +352,14 @@ fn sql_tuple(i: &[u8]) -> IResult<&[u8], &[u8]> { )))(i) } +fn sql_array(i: &[u8]) -> IResult<&[u8], &[u8]> { + recognize(tuple(( + tag("["), + separated_list(ws_sep_comma, sql_expression), + tag("]"), + )))(i) +} + fn sql_cast_function(i: &[u8]) -> IResult<&[u8], &[u8]> { recognize(tuple(( tag_no_case("CAST"), @@ -652,6 +661,14 @@ mod test { "assumeNotNull(if(length(deviceId) > 1, murmurHash3_64(deviceId), rand()))", "assumeNotNull(if(length(deviceId) > 1, murmurHash3_64(deviceId), rand()))".to_string() ), + ( + "[]", + "[]".to_string() + ), + ( + "[1, 2, 3]", + "[1, 2, 3]".to_string() + ), ]; parse_set_for_test(|i| sql_expression(i) .map(|(_, o)| ("".as_bytes(), str::from_utf8(o).unwrap().to_string())), diff --git a/tests/tables2.sql b/tests/tables2.sql index 58f5ece..53e291a 100644 --- a/tests/tables2.sql +++ b/tests/tables2.sql @@ -217,6 +217,7 @@ CREATE TABLE default.EventServerLoggerRequest `eventId` String, `ip` UInt32, `drivingLicence` Array(String), + `keywordIds` Array(UInt32) DEFAULT [] CODEC(T64), `created` DateTime64(3) CODEC(DoubleDelta), `loaded` DateTime64(6, 'Europe/Moscow') CODEC(DoubleDelta) )