MDEV-37157 Fix rownum in having clause and subquery

mysql-test/main/rownum.result

@@ -1015,5 +1015,45 @@ ERROR HY000: The target table v of the UPDATE is not updatable
 DROP VIEW v;
 DROP TABLE t;
 #
+# rownum in having without group by
+#
+create table t1 (a int primary key, b int);
+insert into t1 values (1, 1), (2, 1), (3, 1), (4, 2), (5, 2), (6, 2);
+select * from (select * from t1 order by a, b) dt having rownum() = 4;
+a	b
+4	2
+drop table t1;
+#
+# rownum in subquery
+#
+create table t1 (a int primary key, b int);
+insert into t1 values (1, 1), (2, 1), (3, 1), (4, 2), (5, 2), (6, 2);
+explain select * from t1 where a in (select rownum() from t1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<subquery2>	ALL	distinct_key	NULL	NULL	NULL	6	
+1	PRIMARY	t1	eq_ref	PRIMARY	PRIMARY	4	<subquery2>.rownum()	1	Using index condition
+2	MATERIALIZED	t1	index	NULL	PRIMARY	4	NULL	6	Using index
+select * from t1 where a in (select rownum() from t1);
+a	b
+1	1
+2	1
+3	1
+4	2
+5	2
+6	2
+set optimizer_switch='materialization=off';
+explain select * from t1 where a + 1 in (select rownum() from t1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	6	Using where
+2	DEPENDENT SUBQUERY	t1	index	NULL	PRIMARY	4	NULL	6	Using index
+select * from t1 where a + 1 in (select rownum() from t1);
+a	b
+1	1
+2	1
+3	1
+4	2
+5	2
+drop table t1;
+#
 # End of 10.6 tests
 #

mysql-test/main/rownum.test

@@ -617,6 +617,34 @@ UPDATE v SET f = 10 WHERE e > 42 LIMIT 1;
 DROP VIEW v;
 DROP TABLE t;
 
+--echo #
+--echo # rownum in having without group by
+--echo #
+
+create table t1 (a int primary key, b int);
+insert into t1 values (1, 1), (2, 1), (3, 1), (4, 2), (5, 2), (6, 2);
+select * from (select * from t1 order by a, b) dt having rownum() = 4;
+
+drop table t1;
+
+--echo #
+--echo # rownum in subquery
+--echo #
+
+create table t1 (a int primary key, b int);
+insert into t1 values (1, 1), (2, 1), (3, 1), (4, 2), (5, 2), (6, 2);
+
+# When rownum used in subquery, the subquery shouldn't be transformed to semijoin
+explain select * from t1 where a in (select rownum() from t1);
+select * from t1 where a in (select rownum() from t1);
+
+set optimizer_switch='materialization=off';
+# When rownum used in IN-subquery and the subquery is transformed to EXISTS, the new condition should be added to HAVING clause.
+explain select * from t1 where a + 1 in (select rownum() from t1);
+select * from t1 where a + 1 in (select rownum() from t1);
+
+drop table t1;
+
 --echo #
 --echo # End of 10.6 tests
 --echo #

sql/item_subselect.cc

@@ -2369,7 +2369,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join,
   *having_item= NULL;
 
   if (join_having || select_lex->with_sum_func ||
-      select_lex->group_list.elements)
+      select_lex->group_list.elements || select_lex->with_rownum)
   {
     LEX_CSTRING field_name= this->full_name_cstring();
     Item *item= func->create(thd, expr,
@@ -2616,9 +2616,10 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join,
     during JOIN::optimize: this->tmp_having= this->having; this->having= 0;
   */
   Item* join_having= join->having ? join->having : join->tmp_having;
-  bool is_having_used= (join_having || select_lex->with_sum_func ||
-                        select_lex->group_list.first ||
-                        !select_lex->table_list.elements);
+  bool is_having_used=
+      (join_having || select_lex->with_sum_func ||
+       select_lex->group_list.first || !select_lex->table_list.elements ||
+       select_lex->with_rownum);
   LEX_CSTRING list_ref= { STRING_WITH_LEN("<list ref>")};
   DBUG_ENTER("Item_in_subselect::create_row_in_to_exists_cond");
   DBUG_ASSERT(thd == join->thd);

sql/opt_subselect.cc

@@ -737,25 +737,26 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         11. It is first optimisation (the subquery could be moved from ON
             clause during first optimisation and then be considered for SJ
             on the second when it is too late)
+        12. Subquery does not have ROWNUM
 
       There are also other requirements which cannot be checked at this phase,
       yet. They are checked later in convert_join_subqueries_to_semijoins(),
       look for calls to block_conversion_to_sj().
     */
-    if (select_lex->semijoin_enabled(thd) &&
-        in_subs &&                                                    // 1
-        !select_lex->is_part_of_union() &&                            // 2
-        !select_lex->group_list.elements && !join->order &&           // 3
-        !join->having && !select_lex->with_sum_func &&                // 4
-        in_subs->emb_on_expr_nest &&                                  // 5
-        !select_lex->is_sj_conversion_prohibited(thd) &&              // 6
-        parent_unit->first_select()->leaf_tables.elements &&          // 7
-        !in_subs->has_strategy() &&                                   // 8
-        select_lex->outer_select()->table_list.first &&               // 9
-        !((join->select_options |                                     // 10
-           select_lex->outer_select()->join->select_options)          // 10
-          & SELECT_STRAIGHT_JOIN) &&                                  // 10
-        select_lex->first_cond_optimization)                          // 11
+    if (select_lex->semijoin_enabled(thd) && in_subs &&      // 1
+        !select_lex->is_part_of_union() &&                   // 2
+        !select_lex->group_list.elements && !join->order &&  // 3
+        !join->having && !select_lex->with_sum_func &&       // 4
+        in_subs->emb_on_expr_nest &&                         // 5
+        !select_lex->is_sj_conversion_prohibited(thd) &&     // 6
+        parent_unit->first_select()->leaf_tables.elements && // 7
+        !in_subs->has_strategy() &&                          // 8
+        select_lex->outer_select()->table_list.first &&      // 9
+        !((join->select_options |                            // 10
+           select_lex->outer_select()->join->select_options) // 10
+          & SELECT_STRAIGHT_JOIN) &&                         // 10
+        select_lex->first_cond_optimization &&               // 11
+        !select_lex->with_rownum)                            // 12
     {
       DBUG_PRINT("info", ("Subquery is semi-join conversion candidate"));
 

sql/sql_select.cc

@@ -25768,7 +25768,14 @@ end_send(JOIN *join, JOIN_TAB *join_tab, bool end_of_records)
     copy_fields(&join->tmp_table_param);
   }
   if (join->having && join->having->val_bool() == 0)
+  {
+    /*
+      If we have HAVING clause and it is not satisfied, we don't send
+      the row to the client, but rownum should be incremented.
+    */
+    join->accepted_rows++;
     DBUG_RETURN(NESTED_LOOP_OK);               // Didn't match having
+  }
   if (join->procedure)
   {
     if (join->procedure->send_row(join->procedure_fields_list))