expression: pushdown trim to tikv | tidb-test=pr/2728

pkg/expression/expr_to_pb_test.go

@@ -362,6 +362,40 @@ func TestDateFunc2Pb(t *testing.T) {
 	require.Equal(t, "{\"tp\":10000,\"children\":[{\"tp\":201,\"val\":\"gAAAAAAAAAE=\",\"sig\":0,\"field_type\":{\"tp\":12,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":-63,\"charset\":\"binary\",\"array\":false},\"has_distinct\":false},{\"tp\":201,\"val\":\"gAAAAAAAAAI=\",\"sig\":0,\"field_type\":{\"tp\":254,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":-46,\"charset\":\"utf8mb4\",\"array\":false},\"has_distinct\":false}],\"sig\":6001,\"field_type\":{\"tp\":253,\"flag\":0,\"flen\":0,\"decimal\":-1,\"collate\":-46,\"charset\":\"utf8mb4\",\"array\":false},\"has_distinct\":false}", string(js))
 }
 
+func TestTrimFunc2Pb(t *testing.T) {
+	ctx := mock.NewContext()
+	client := new(mock.Client)
+	stringColumn := genColumn(mysql.TypeString, 1)
+	remstrColumn := genColumn(mysql.TypeString, 2)
+	direction := &Constant{
+		Value:   types.NewIntDatum(int64(ast.TrimLeading)),
+		RetType: types.NewFieldType(mysql.TypeLonglong),
+	}
+
+	funcs := make([]Expression, 0, 3)
+	function, err := NewFunction(ctx, ast.Trim, types.NewFieldType(mysql.TypeString), stringColumn)
+	require.NoError(t, err)
+	require.Equal(t, tipb.ScalarFuncSig_Trim1Arg, function.(*ScalarFunction).Function.PbCode())
+	funcs = append(funcs, function)
+
+	function, err = NewFunction(ctx, ast.Trim, types.NewFieldType(mysql.TypeString), stringColumn, remstrColumn)
+	require.NoError(t, err)
+	require.Equal(t, tipb.ScalarFuncSig_Trim2Args, function.(*ScalarFunction).Function.PbCode())
+	funcs = append(funcs, function)
+
+	function, err = NewFunction(ctx, ast.Trim, types.NewFieldType(mysql.TypeString), stringColumn, remstrColumn, direction)
+	require.NoError(t, err)
+	require.Equal(t, tipb.ScalarFuncSig_Trim3Args, function.(*ScalarFunction).Function.PbCode())
+	funcs = append(funcs, function)
+
+	pbExprs, err := ExpressionsToPBList(ctx, funcs, client)
+	require.NoError(t, err)
+	require.Len(t, pbExprs, 3)
+	require.Equal(t, tipb.ScalarFuncSig_Trim1Arg, pbExprs[0].Sig)
+	require.Equal(t, tipb.ScalarFuncSig_Trim2Args, pbExprs[1].Sig)
+	require.Equal(t, tipb.ScalarFuncSig_Trim3Args, pbExprs[2].Sig)
+}
+
 func TestLogicalFunc2Pb(t *testing.T) {
 	var logicalFuncs = make([]Expression, 0)
 	ctx := mock.NewContext()

pkg/expression/infer_pushdown.go

@@ -210,9 +210,10 @@ func scalarExprSupportedByTiKV(ctx EvalContext, sf *ScalarFunction) bool {
 
 		// string functions.
 		// ast.Bin, ast.Unhex, ast.Locate, ast.Ord, ast.Lpad, ast.Rpad,
-		// ast.Trim, ast.FromBase64, ast.ToBase64, ast.InsertFunc,
+		// ast.FromBase64, ast.ToBase64, ast.InsertFunc,
 		// ast.MakeSet, ast.SubstringIndex, ast.Instr, ast.Quote, ast.Oct,
 		// ast.FindInSet, ast.Repeat,
+		ast.Trim,
 		ast.Upper, ast.Lower,
 		ast.Length, ast.BitLength, ast.Concat, ast.ConcatWS, ast.Replace, ast.ASCII, ast.Hex,
 		ast.Reverse, ast.LTrim, ast.RTrim, ast.Strcmp, ast.Space, ast.Elt, ast.Field,

tests/integrationtest/r/expression/builtin.result

@@ -1931,6 +1931,84 @@ hex(trim(''))	hex(trim('x' from ''))
 select hex(trim(null from 'bar')), hex(trim('x' from null)), hex(trim(null)), hex(trim(leading null from 'bar'));
 hex(trim(null from 'bar'))	hex(trim('x' from null))	hex(trim(null))	hex(trim(leading null from 'bar'))
 NULL	NULL	NULL	NULL
+drop table if exists t_trim_pushdown;
+create table t_trim_pushdown(
+id int primary key,
+c varchar(64) charset utf8mb4 collate utf8mb4_general_ci
+);
+insert into t_trim_pushdown values
+(1, 'aaaaa'),
+(2, 'ppp'),
+(3, 'xyxyx');
+explain format = 'brief' select /*+ read_from_storage(tikv[t_trim_pushdown]) */ id from t_trim_pushdown where trim('aaa' from c) = 'aa' order by id;
+id	estRows	task	access object	operator info
+TableReader	8000.00	root		data:Projection
+└─Projection	8000.00	cop[tikv]		expression__builtin.t_trim_pushdown.id
+  └─Selection	8000.00	cop[tikv]		eq(trim(expression__builtin.t_trim_pushdown.c, "aaa"), "aa")
+    └─TableFullScan	10000.00	cop[tikv]	table:t_trim_pushdown	keep order:true, stats:pseudo
+explain format = 'brief' select /*+ read_from_storage(tikv[t_trim_pushdown]) */ id from t_trim_pushdown where trim(both 'pp' from c) = 'p' order by id;
+id	estRows	task	access object	operator info
+TableReader	8000.00	root		data:Projection
+└─Projection	8000.00	cop[tikv]		expression__builtin.t_trim_pushdown.id
+  └─Selection	8000.00	cop[tikv]		eq(trim(expression__builtin.t_trim_pushdown.c, "pp", 1), "p")
+    └─TableFullScan	10000.00	cop[tikv]	table:t_trim_pushdown	keep order:true, stats:pseudo
+explain format = 'brief' select /*+ read_from_storage(tikv[t_trim_pushdown]) */ id from t_trim_pushdown where trim(both 'xyx' from c) = 'yx' order by id;
+id	estRows	task	access object	operator info
+TableReader	8000.00	root		data:Projection
+└─Projection	8000.00	cop[tikv]		expression__builtin.t_trim_pushdown.id
+  └─Selection	8000.00	cop[tikv]		eq(trim(expression__builtin.t_trim_pushdown.c, "xyx", 1), "yx")
+    └─TableFullScan	10000.00	cop[tikv]	table:t_trim_pushdown	keep order:true, stats:pseudo
+delete from mysql.expr_pushdown_blacklist where name = 'trim' and store_type = 'tikv,tiflash,tidb' and reason = 'for trim pushdown test';
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_on(id int primary key);
+insert into trim_push_on select id from t_trim_pushdown where trim('aaa' from c) = 'aa' order by id;
+insert into mysql.expr_pushdown_blacklist values('trim', 'tikv,tiflash,tidb', 'for trim pushdown test');
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_off(id int primary key);
+insert into trim_push_off select id from t_trim_pushdown where trim('aaa' from c) = 'aa' order by id;
+select ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') as push_on,
+ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as push_off,
+ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') =
+ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as same;
+push_on	push_off	same
+1	1	1
+drop temporary table trim_push_on;
+drop temporary table trim_push_off;
+delete from mysql.expr_pushdown_blacklist where name = 'trim' and store_type = 'tikv,tiflash,tidb' and reason = 'for trim pushdown test';
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_on(id int primary key);
+insert into trim_push_on select id from t_trim_pushdown where trim(both 'pp' from c) = 'p' order by id;
+insert into mysql.expr_pushdown_blacklist values('trim', 'tikv,tiflash,tidb', 'for trim pushdown test');
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_off(id int primary key);
+insert into trim_push_off select id from t_trim_pushdown where trim(both 'pp' from c) = 'p' order by id;
+select ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') as push_on,
+ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as push_off,
+ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') =
+ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as same;
+push_on	push_off	same
+2	2	1
+drop temporary table trim_push_on;
+drop temporary table trim_push_off;
+delete from mysql.expr_pushdown_blacklist where name = 'trim' and store_type = 'tikv,tiflash,tidb' and reason = 'for trim pushdown test';
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_on(id int primary key);
+insert into trim_push_on select id from t_trim_pushdown where trim(both 'xyx' from c) = 'yx' order by id;
+insert into mysql.expr_pushdown_blacklist values('trim', 'tikv,tiflash,tidb', 'for trim pushdown test');
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_off(id int primary key);
+insert into trim_push_off select id from t_trim_pushdown where trim(both 'xyx' from c) = 'yx' order by id;
+select ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') as push_on,
+ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as push_off,
+ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') =
+ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as same;
+push_on	push_off	same
+3	3	1
+drop temporary table trim_push_on;
+drop temporary table trim_push_off;
+delete from mysql.expr_pushdown_blacklist where name = 'trim' and store_type = 'tikv,tiflash,tidb' and reason = 'for trim pushdown test';
+admin reload expr_pushdown_blacklist;
+drop table if exists t_trim_pushdown;
 drop table if exists t;
 create table t(a char(20), b int, c double, d datetime, e time, f binary(5));
 insert into t values('www.pingcap.com', 12345, 123.45, "2017-01-01 12:01:01", "12:01:01", "HelLo");

tests/integrationtest/t/expression/builtin.test

@@ -870,6 +870,63 @@ select hex(trim('\t   bar\n   ')), hex(trim('   \rbar   \t'));
 select hex(trim(leading from '   bar')), hex(trim('x' from 'xxxbarxxx')), hex(trim('x' from 'bar')), hex(trim('' from '   bar   '));
 select hex(trim('')), hex(trim('x' from ''));
 select hex(trim(null from 'bar')), hex(trim('x' from null)), hex(trim(null)), hex(trim(leading null from 'bar'));
+drop table if exists t_trim_pushdown;
+create table t_trim_pushdown(
+  id int primary key,
+  c varchar(64) charset utf8mb4 collate utf8mb4_general_ci
+);
+insert into t_trim_pushdown values
+  (1, 'aaaaa'),
+  (2, 'ppp'),
+  (3, 'xyxyx');
+explain format = 'brief' select /*+ read_from_storage(tikv[t_trim_pushdown]) */ id from t_trim_pushdown where trim('aaa' from c) = 'aa' order by id;
+explain format = 'brief' select /*+ read_from_storage(tikv[t_trim_pushdown]) */ id from t_trim_pushdown where trim(both 'pp' from c) = 'p' order by id;
+explain format = 'brief' select /*+ read_from_storage(tikv[t_trim_pushdown]) */ id from t_trim_pushdown where trim(both 'xyx' from c) = 'yx' order by id;
+delete from mysql.expr_pushdown_blacklist where name = 'trim' and store_type = 'tikv,tiflash,tidb' and reason = 'for trim pushdown test';
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_on(id int primary key);
+insert into trim_push_on select id from t_trim_pushdown where trim('aaa' from c) = 'aa' order by id;
+insert into mysql.expr_pushdown_blacklist values('trim', 'tikv,tiflash,tidb', 'for trim pushdown test');
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_off(id int primary key);
+insert into trim_push_off select id from t_trim_pushdown where trim('aaa' from c) = 'aa' order by id;
+select ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') as push_on,
+       ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as push_off,
+       ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') =
+       ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as same;
+drop temporary table trim_push_on;
+drop temporary table trim_push_off;
+delete from mysql.expr_pushdown_blacklist where name = 'trim' and store_type = 'tikv,tiflash,tidb' and reason = 'for trim pushdown test';
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_on(id int primary key);
+insert into trim_push_on select id from t_trim_pushdown where trim(both 'pp' from c) = 'p' order by id;
+insert into mysql.expr_pushdown_blacklist values('trim', 'tikv,tiflash,tidb', 'for trim pushdown test');
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_off(id int primary key);
+insert into trim_push_off select id from t_trim_pushdown where trim(both 'pp' from c) = 'p' order by id;
+select ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') as push_on,
+       ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as push_off,
+       ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') =
+       ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as same;
+drop temporary table trim_push_on;
+drop temporary table trim_push_off;
+delete from mysql.expr_pushdown_blacklist where name = 'trim' and store_type = 'tikv,tiflash,tidb' and reason = 'for trim pushdown test';
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_on(id int primary key);
+insert into trim_push_on select id from t_trim_pushdown where trim(both 'xyx' from c) = 'yx' order by id;
+insert into mysql.expr_pushdown_blacklist values('trim', 'tikv,tiflash,tidb', 'for trim pushdown test');
+admin reload expr_pushdown_blacklist;
+create temporary table trim_push_off(id int primary key);
+insert into trim_push_off select id from t_trim_pushdown where trim(both 'xyx' from c) = 'yx' order by id;
+select ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') as push_on,
+       ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as push_off,
+       ifnull((select group_concat(id order by id) from trim_push_on), '<empty>') =
+       ifnull((select group_concat(id order by id) from trim_push_off), '<empty>') as same;
+drop temporary table trim_push_on;
+drop temporary table trim_push_off;
+delete from mysql.expr_pushdown_blacklist where name = 'trim' and store_type = 'tikv,tiflash,tidb' and reason = 'for trim pushdown test';
+admin reload expr_pushdown_blacklist;
+drop table if exists t_trim_pushdown;
 drop table if exists t;
 create table t(a char(20), b int, c double, d datetime, e time, f binary(5));
 insert into t values('www.pingcap.com', 12345, 123.45, "2017-01-01 12:01:01", "12:01:01", "HelLo");
@@ -1672,4 +1729,4 @@ set sql_mode = default;
 
 # Issue 59420
 SELECT GET_FORMAT(TIME, 'usa'), GET_FORMAT(TIME, 'USA'), GET_FORMAT(TIME, 'UsA'), GET_FORMAT(time, 'UsA'), GET_FORMAT(TIme, 'UsA');
-SELECT GET_FORMAT(DATE, 'jis'),GET_FORMAT(DATETIME, 'iso'),GET_FORMAT(TIMESTAMP, 'eur'),GET_FORMAT(TIME, 'internal');
+SELECT GET_FORMAT(DATE, 'jis'),GET_FORMAT(DATETIME, 'iso'),GET_FORMAT(TIMESTAMP, 'eur'),GET_FORMAT(TIME, 'internal');