Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:EDV_Lotse:1C
postgresql10
00009-opt_group_by_and_cost_sort.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 00009-opt_group_by_and_cost_sort.patch of Package postgresql10
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index f6bb2fc472..2641f0d6f6 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -566,21 +566,23 @@ SELECT t1."C 1", t2.c1, t3.c1 FROM "S 1"."T 1" t1 left join ft1 t2 full join ft2 -- Test similar to above with all full outer joins EXPLAIN (VERBOSE, COSTS OFF) SELECT t1."C 1", t2.c1, t3.c1 FROM "S 1"."T 1" t1 full join ft1 t2 full join ft2 t3 on (t2.c1 = t3.c1) on (t3.c1 = t1."C 1") OFFSET 100 LIMIT 10; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------- Limit Output: t1."C 1", t2.c1, t3.c1 -> Merge Full Join Output: t1."C 1", t2.c1, t3.c1 - Inner Unique: true - Merge Cond: (t3.c1 = t1."C 1") - -> Foreign Scan - Output: t2.c1, t3.c1 - Relations: (public.ft1 t2) FULL JOIN (public.ft2 t3) - Remote SQL: SELECT r2."C 1", r3."C 1" FROM ("S 1"."T 1" r2 FULL JOIN "S 1"."T 1" r3 ON (((r2."C 1" = r3."C 1")))) ORDER BY r3."C 1" ASC NULLS LAST + Merge Cond: (t1."C 1" = t3.c1) -> Index Only Scan using t1_pkey on "S 1"."T 1" t1 Output: t1."C 1" -(12 rows) + -> Sort + Output: t2.c1, t3.c1 + Sort Key: t3.c1 + -> Foreign Scan + Output: t2.c1, t3.c1 + Relations: (public.ft1 t2) FULL JOIN (public.ft2 t3) + Remote SQL: SELECT r2."C 1", r3."C 1" FROM ("S 1"."T 1" r2 FULL JOIN "S 1"."T 1" r3 ON (((r2."C 1" = r3."C 1")))) +(14 rows) SELECT t1."C 1", t2.c1, t3.c1 FROM "S 1"."T 1" t1 full join ft1 t2 full join ft2 t3 on (t2.c1 = t3.c1) on (t3.c1 = t1."C 1") OFFSET 100 LIMIT 10; C 1 | c1 | c1 @@ -1046,18 +1048,15 @@ SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t -- join three tables EXPLAIN (VERBOSE, COSTS OFF) SELECT t1.c1, t2.c2, t3.c3 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) JOIN ft4 t3 ON (t3.c1 = t1.c1) ORDER BY t1.c3, t1.c1 OFFSET 10 LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit Output: t1.c1, t2.c2, t3.c3, t1.c3 - -> Sort + -> Foreign Scan Output: t1.c1, t2.c2, t3.c3, t1.c3 - Sort Key: t1.c3, t1.c1 - -> Foreign Scan - Output: t1.c1, t2.c2, t3.c3, t1.c3 - Relations: ((public.ft1 t1) INNER JOIN (public.ft2 t2)) INNER JOIN (public.ft4 t3) - Remote SQL: SELECT r1."C 1", r1.c3, r2.c2, r4.c3 FROM (("S 1"."T 1" r1 INNER JOIN "S 1"."T 1" r2 ON (((r1."C 1" = r2."C 1")))) INNER JOIN "S 1"."T 3" r4 ON (((r1."C 1" = r4.c1)))) -(9 rows) + Relations: ((public.ft1 t1) INNER JOIN (public.ft2 t2)) INNER JOIN (public.ft4 t3) + Remote SQL: SELECT r1."C 1", r1.c3, r2.c2, r4.c3 FROM (("S 1"."T 1" r1 INNER JOIN "S 1"."T 1" r2 ON (((r1."C 1" = r2."C 1")))) INNER JOIN "S 1"."T 3" r4 ON (((r1."C 1" = r4.c1)))) ORDER BY r1.c3 ASC NULLS LAST, r1."C 1" ASC NULLS LAST +(6 rows) SELECT t1.c1, t2.c2, t3.c3 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) JOIN ft4 t3 ON (t3.c1 = t1.c1) ORDER BY t1.c3, t1.c1 OFFSET 10 LIMIT 10; c1 | c2 | c3 @@ -1707,22 +1706,18 @@ SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t -> Sort Output: t1.c1, t1.c3, t1.*, t2.c1, t2.* Sort Key: t1.c3, t1.c1 - -> Merge Join + -> Hash Join Output: t1.c1, t1.c3, t1.*, t2.c1, t2.* - Merge Cond: (t1.c1 = t2.c1) - -> Sort + Hash Cond: (t1.c1 = t2.c1) + -> Foreign Scan on public.ft1 t1 Output: t1.c1, t1.c3, t1.* - Sort Key: t1.c1 - -> Foreign Scan on public.ft1 t1 - Output: t1.c1, t1.c3, t1.* - Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR UPDATE - -> Sort + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR UPDATE + -> Hash Output: t2.c1, t2.* - Sort Key: t2.c1 -> Foreign Scan on public.ft2 t2 Output: t2.c1, t2.* Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" -(26 rows) +(22 rows) SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t1.c1 OFFSET 100 LIMIT 10 FOR UPDATE OF t1; c1 | c1 @@ -1754,22 +1749,18 @@ SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t -> Sort Output: t1.c1, t1.c3, t1.*, t2.c1, t2.* Sort Key: t1.c3, t1.c1 - -> Merge Join + -> Hash Join Output: t1.c1, t1.c3, t1.*, t2.c1, t2.* - Merge Cond: (t1.c1 = t2.c1) - -> Sort + Hash Cond: (t1.c1 = t2.c1) + -> Foreign Scan on public.ft1 t1 Output: t1.c1, t1.c3, t1.* - Sort Key: t1.c1 - -> Foreign Scan on public.ft1 t1 - Output: t1.c1, t1.c3, t1.* - Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR UPDATE - -> Sort + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR UPDATE + -> Hash Output: t2.c1, t2.* - Sort Key: t2.c1 -> Foreign Scan on public.ft2 t2 Output: t2.c1, t2.* Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR UPDATE -(26 rows) +(22 rows) SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t1.c1 OFFSET 100 LIMIT 10 FOR UPDATE; c1 | c1 @@ -1802,22 +1793,18 @@ SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t -> Sort Output: t1.c1, t1.c3, t1.*, t2.c1, t2.* Sort Key: t1.c3, t1.c1 - -> Merge Join + -> Hash Join Output: t1.c1, t1.c3, t1.*, t2.c1, t2.* - Merge Cond: (t1.c1 = t2.c1) - -> Sort + Hash Cond: (t1.c1 = t2.c1) + -> Foreign Scan on public.ft1 t1 Output: t1.c1, t1.c3, t1.* - Sort Key: t1.c1 - -> Foreign Scan on public.ft1 t1 - Output: t1.c1, t1.c3, t1.* - Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR SHARE - -> Sort + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR SHARE + -> Hash Output: t2.c1, t2.* - Sort Key: t2.c1 -> Foreign Scan on public.ft2 t2 Output: t2.c1, t2.* Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" -(26 rows) +(22 rows) SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t1.c1 OFFSET 100 LIMIT 10 FOR SHARE OF t1; c1 | c1 @@ -1849,22 +1836,18 @@ SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t -> Sort Output: t1.c1, t1.c3, t1.*, t2.c1, t2.* Sort Key: t1.c3, t1.c1 - -> Merge Join + -> Hash Join Output: t1.c1, t1.c3, t1.*, t2.c1, t2.* - Merge Cond: (t1.c1 = t2.c1) - -> Sort + Hash Cond: (t1.c1 = t2.c1) + -> Foreign Scan on public.ft1 t1 Output: t1.c1, t1.c3, t1.* - Sort Key: t1.c1 - -> Foreign Scan on public.ft1 t1 - Output: t1.c1, t1.c3, t1.* - Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR SHARE - -> Sort + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR SHARE + -> Hash Output: t2.c1, t2.* - Sort Key: t2.c1 -> Foreign Scan on public.ft2 t2 Output: t2.c1, t2.* Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8 FROM "S 1"."T 1" FOR SHARE -(26 rows) +(22 rows) SELECT t1.c1, t2.c1 FROM ft1 t1 JOIN ft2 t2 ON (t1.c1 = t2.c1) ORDER BY t1.c3, t1.c1 OFFSET 100 LIMIT 10 FOR SHARE; c1 | c1 diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 5bc8bca45f..62d933ad4c 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -2727,7 +2727,8 @@ estimate_path_cost_size(PlannerInfo *root, numGroups = estimate_num_groups(root, get_sortgrouplist_exprs(root->parse->groupClause, fpinfo->grouped_tlist), - input_rows, NULL); + input_rows, NULL, + NULL, 0); /* * Number of rows expected from foreign server will be same as diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 7f867530d4..476d7171de 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -1596,6 +1596,283 @@ cost_recursive_union(Path *runion, Path *nrterm, Path *rterm) rterm->pathtarget->width); } +/* + * is_fake_var + * Workaround for generate_append_tlist() which generates fake Vars with + * varno == 0, that will cause a fail of estimate_num_group() call + */ +static bool +is_fake_var(Expr *expr) +{ + if (IsA(expr, RelabelType)) + expr = (Expr *) ((RelabelType *) expr)->arg; + + return (IsA(expr, Var) && ((Var *) expr)->varno == 0); +} + +/* + * get_width_cost_multiplier + * Returns relative complexity of comparing two valyes based on it's width. + * The idea behind - long values have more expensive comparison. Return value is + * in cpu_operator_cost unit. + */ +static double +get_width_cost_multiplier(PlannerInfo *root, Expr *expr) +{ + double width = -1.0; /* fake value */ + + if (IsA(expr, RelabelType)) + expr = (Expr *) ((RelabelType *) expr)->arg; + + /* Try to find actual stat in corresonding relation */ + if (IsA(expr, Var)) + { + Var *var = (Var *) expr; + + if (var->varno > 0 && var->varno < root->simple_rel_array_size) + { + RelOptInfo *rel = root->simple_rel_array[var->varno]; + + if (rel != NULL && + var->varattno >= rel->min_attr && + var->varattno <= rel->max_attr) + { + int ndx = var->varattno - rel->min_attr; + + if (rel->attr_widths[ndx] > 0) + width = rel->attr_widths[ndx]; + } + } + } + + /* Didn't find any actual stats, use estimation by type */ + if (width < 0.0) + { + Node *node = (Node*) expr; + + width = get_typavgwidth(exprType(node), exprTypmod(node)); + } + + /* + * Any value in pgsql is passed by Datum type, so any operation with value + * could not be cheaper than operation with Datum type + */ + if (width <= sizeof(Datum)) + return 1.0; + + /* + * Seems, cost of comparision is not directly proportional to args width, + * because comparing args could be differ width (we known only average over + * column) and difference often could be defined only by looking on first + * bytes. So, use log16(width) as estimation. + */ + return 1.0 + 0.125 * LOG2(width / sizeof(Datum)); +} + +/* + * compute_cpu_sort_cost + * compute CPU cost of sort (i.e. in-memory) + * + * NOTE: some callers currently pass NIL for pathkeys because they + * can't conveniently supply the sort keys. In this case, it will fallback to + * simple comparison cost estimate. + * + * Estimation algorithm is based on ideas from course Algorithms, + * Robert Sedgewick, Kevin Wayne, https://algs4.cs.princeton.edu/home/ and paper + * "Quicksort Is Optimal For Many Equal Keys", Sebastian Wild, + * arXiv:1608.04906v4 [cs.DS] 1 Nov 2017. + * + * In term of that papers, let N - number of tuples, Xi - number of tuples with + * key Ki, then estimation is: + * log(N! / (X1! * X2! * ..)) ~ sum(Xi * log(N/Xi)) + * In our case all Xi are the same because noew we don't have an estimation of + * group sizes, we have only estimation of number of groups. In this case, + * formula becomes: N * log(NumberOfGroups). Next, to support correct estimation + * of multicolumn sort we need separately compute each column, so, let k is a + * column number, Gk - number of groups defined by k columns: + * N * sum( Fk * log(Gk) ) + * Fk is a function costs (includeing width) for k columns. + */ + +static Cost +compute_cpu_sort_cost(PlannerInfo *root, List *pathkeys, int nPresortedKeys, + Cost comparison_cost, double tuples, double output_tuples, + bool heapSort) +{ + Cost per_tuple_cost = 0.0; + ListCell *lc; + List *pathkeyExprs = NIL; + double tuplesPerPrevGroup = tuples; + double totalFuncCost = 1.0; + bool has_fake_var = false; + int i = 0; + Oid prev_datatype = InvalidOid; + Cost funcCost; + List *cache_varinfos = NIL; + + /* fallback if pathkeys is unknown */ + if (list_length(pathkeys) == 0) + { + /* + * If we'll use a bounded heap-sort keeping just K tuples in memory, for + * a total number of tuple comparisons of N log2 K; but the constant + * factor is a bit higher than for quicksort. Tweak it so that the + * cost curve is continuous at the crossover point. + */ + output_tuples = (heapSort) ? 2.0 * output_tuples : tuples; + per_tuple_cost += 2.0 * cpu_operator_cost * LOG2(output_tuples); + + /* add cost provided by caller */ + per_tuple_cost += comparison_cost; + + return per_tuple_cost * tuples; + } + + /* + * Computing total cost of sorting takes into account: + * - per column comparison function cost + * - we try to compute needed number of comparison per column + */ + + foreach(lc, pathkeys) + { + PathKey *pathkey = (PathKey*)lfirst(lc); + EquivalenceMember *em; + double nGroups, + correctedNGroups; + + /* + * We believe than equivalence members aren't very different, so, to + * estimate cost we take just first member + */ + em = (EquivalenceMember *) linitial(pathkey->pk_eclass->ec_members); + + if (em->em_datatype != InvalidOid) + { + /* do not lookup funcCost if data type is the same as previous */ + if (prev_datatype != em->em_datatype) + { + Oid sortop; + + sortop = get_opfamily_member(pathkey->pk_opfamily, + em->em_datatype, em->em_datatype, + pathkey->pk_strategy); + + funcCost = get_func_cost(get_opcode(sortop)); + prev_datatype = em->em_datatype; + } + } + else + funcCost = 1.0; /* fallback */ + + /* Try to take into account actual width fee */ + funcCost *= get_width_cost_multiplier(root, em->em_expr); + + totalFuncCost += funcCost; + + /* remeber if we have a fake var in pathkeys */ + has_fake_var |= is_fake_var(em->em_expr); + pathkeyExprs = lappend(pathkeyExprs, em->em_expr); + + /* + * Prevent call estimate_num_groups() with fake Var. Note, + * pathkeyExprs contains only previous columns + */ + if (has_fake_var == false) + /* + * Recursively compute number of group in group from previous step + */ + nGroups = estimate_num_groups(root, pathkeyExprs, + tuplesPerPrevGroup, NULL, + &cache_varinfos, + list_length(pathkeyExprs) - 1); + else if (tuples > 4.0) + /* + * Use geometric mean as estimation if there is no any stats. + * Don't use DEFAULT_NUM_DISTINCT because it used for only one + * column while here we try to estimate number of groups over + * set of columns. + */ + nGroups = ceil(2.0 + sqrt(tuples) * + list_length(pathkeyExprs) / list_length(pathkeys)); + else + nGroups = tuples; + + /* + * Presorted keys aren't participated in comparison but still checked + * by qsort comparator. + */ + if (i >= nPresortedKeys) + { + if (heapSort) + { + if (tuplesPerPrevGroup < output_tuples) + /* comparing only inside output_tuples */ + correctedNGroups = + ceil(2.0 * output_tuples / (tuplesPerPrevGroup / nGroups)); + else + /* two groups - in output and out */ + correctedNGroups = 2.0; + } + else + correctedNGroups = nGroups; + + if (correctedNGroups <= 1.0) + correctedNGroups = 2.0; + else + correctedNGroups = ceil(correctedNGroups); + per_tuple_cost += totalFuncCost * LOG2(correctedNGroups); + } + + i++; + + /* + * Real-world distribution isn't uniform but now we don't have a way to + * determine that, so, add multiplier to get closer to worst case. + */ + tuplesPerPrevGroup = ceil(1.5 * tuplesPerPrevGroup / nGroups); + + /* + * We could skip all followed columns for cost estimation, because we + * believe that tuples are unique by set ot previous columns + */ + if (tuplesPerPrevGroup <= 1.0) + break; + } + + list_free(pathkeyExprs); + + /* per_tuple_cost is in cpu_operator_cost units */ + per_tuple_cost *= cpu_operator_cost; + + /* + * Accordingly to "Introduction to algorithms", Thomas H. Cormen, Charles E. + * Leiserson, Ronald L. Rivest, ISBN 0-07-013143-0, quicksort estimation + * formula has additional term proportional to number of tuples (See Chapter + * 8.2 and Theorem 4.1). It has meaning with low number of tuples, + * approximately less that 1e4. Of course, it could be inmplemented as + * additional multiplier under logarithm, but use more complicated formula + * which takes into account number of unique tuples and it isn't clear how + * to combine multiplier with groups. Estimate it as 10 in cpu_operator_cost + * unit. + */ + per_tuple_cost += 10 * cpu_operator_cost; + + per_tuple_cost += comparison_cost; + + return tuples * per_tuple_cost; +} + +/* + * simple wrapper just to estimate best sort path + */ +Cost +cost_sort_estimate(PlannerInfo *root, List *pathkeys, int nPresortedKeys, + double tuples) +{ + return compute_cpu_sort_cost(root, pathkeys, nPresortedKeys, + 0, tuples, tuples, false); +} /* * cost_sort * Determines and returns the cost of sorting a relation, including @@ -1612,7 +1889,7 @@ cost_recursive_union(Path *runion, Path *nrterm, Path *rterm) * number of initial runs formed and M is the merge order used by tuplesort.c. * Since the average initial run should be about sort_mem, we have * disk traffic = 2 * relsize * ceil(logM(p / sort_mem)) - * cpu = comparison_cost * t * log2(t) + * and cpu cost computed by compute_cpu_sort_cost(). * * If the sort is bounded (i.e., only the first k result tuples are needed) * and k tuples can fit into sort_mem, we use a heap method that keeps only @@ -1633,13 +1910,6 @@ cost_recursive_union(Path *runion, Path *nrterm, Path *rterm) * 'comparison_cost' is the extra cost per comparison, if any * 'sort_mem' is the number of kilobytes of work memory allowed for the sort * 'limit_tuples' is the bound on the number of output tuples; -1 if no bound - * - * NOTE: some callers currently pass NIL for pathkeys because they - * can't conveniently supply the sort keys. Since this routine doesn't - * currently do anything with pathkeys anyway, that doesn't matter... - * but if it ever does, it should react gracefully to lack of key data. - * (Actually, the thing we'd most likely be interested in is just the number - * of sort keys, which all callers *could* supply.) */ void cost_sort(Path *path, PlannerInfo *root, @@ -1666,9 +1936,6 @@ cost_sort(Path *path, PlannerInfo *root, if (tuples < 2.0) tuples = 2.0; - /* Include the default cost-per-comparison */ - comparison_cost += 2.0 * cpu_operator_cost; - /* Do we have a useful LIMIT? */ if (limit_tuples > 0 && limit_tuples < tuples) { @@ -1697,7 +1964,9 @@ cost_sort(Path *path, PlannerInfo *root, * * Assume about N log2 N comparisons */ - startup_cost += comparison_cost * tuples * LOG2(tuples); + startup_cost += compute_cpu_sort_cost(root, pathkeys, 0, + comparison_cost, tuples, + tuples, false); /* Disk costs */ @@ -1713,18 +1982,17 @@ cost_sort(Path *path, PlannerInfo *root, } else if (tuples > 2 * output_tuples || input_bytes > sort_mem_bytes) { - /* - * We'll use a bounded heap-sort keeping just K tuples in memory, for - * a total number of tuple comparisons of N log2 K; but the constant - * factor is a bit higher than for quicksort. Tweak it so that the - * cost curve is continuous at the crossover point. - */ - startup_cost += comparison_cost * tuples * LOG2(2.0 * output_tuples); + /* We'll use a bounded heap-sort keeping just K tuples in memory. */ + startup_cost += compute_cpu_sort_cost(root, pathkeys, 0, + comparison_cost, tuples, + output_tuples, true); } else { /* We'll use plain quicksort on all the input tuples */ - startup_cost += comparison_cost * tuples * LOG2(tuples); + startup_cost += compute_cpu_sort_cost(root, pathkeys, 0, + comparison_cost, tuples, + tuples, false); } /* diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index cc7f247347..7828f57e26 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -648,7 +648,18 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (opcintype == cur_em->em_datatype && equal(expr, cur_em->em_expr)) - return cur_ec; /* Match! */ + { + /* + * Match! + * + * Copy sortref if it wasn't set yet, it's possible if ec was + * constructed from WHERE clause, ie it doesn't have target + * reference at all + */ + if (cur_ec->ec_sortref == 0 && sortref > 0) + cur_ec->ec_sortref = sortref; + return cur_ec; + } } } diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index c104e910e5..77e6c58c51 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -2055,7 +2055,7 @@ adjust_rowcount_for_semijoins(PlannerInfo *root, nunique = estimate_num_groups(root, sjinfo->semi_rhs_exprs, nraw, - NULL); + NULL, NULL, 0); if (rowcount > nunique) rowcount = nunique; } diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 176689835b..395c3f72c4 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -17,15 +17,18 @@ */ #include "postgres.h" +#include "miscadmin.h" #include "access/stratnum.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/plannodes.h" #include "optimizer/clauses.h" +#include "optimizer/cost.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/tlist.h" #include "utils/lsyscache.h" +#include "utils/selfuncs.h" static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); @@ -327,6 +330,290 @@ pathkeys_contained_in(List *keys1, List *keys2) return false; } +/* + * Reorder GROUP BY pathkeys and clauses to match order of pathkeys. Function + * returns new lists, original GROUP BY lists stay untouched. + */ +int +group_keys_reorder_by_pathkeys(List *pathkeys, List **group_pathkeys, + List **group_clauses) +{ + List *new_group_pathkeys= NIL, + *new_group_clauses = NIL; + ListCell *key; + int n; + + if (pathkeys == NIL || *group_pathkeys == NIL) + return 0; + + /* + * For each pathkey it tries to find corresponding GROUP BY pathkey and + * clause. + */ + foreach(key, pathkeys) + { + PathKey *pathkey = (PathKey *) lfirst(key); + SortGroupClause *sgc; + + /* + * Pathkey should use the same allocated struct, so, equiality of + * pointers is enough + */ + if (!list_member_ptr(*group_pathkeys, pathkey)) + break; + + new_group_pathkeys = lappend(new_group_pathkeys, pathkey); + + sgc = get_sortgroupref_clause(pathkey->pk_eclass->ec_sortref, + *group_clauses); + new_group_clauses = lappend(new_group_clauses, sgc); + } + + n = list_length(new_group_pathkeys); + + /* + * Just append the rest of pathkeys and clauses + */ + *group_pathkeys = list_concat_unique_ptr(new_group_pathkeys, + *group_pathkeys); + *group_clauses = list_concat_unique_ptr(new_group_clauses, + *group_clauses); + + return n; +} + +typedef struct MutatorState { + List *elemsList; + ListCell **elemCells; + void **elems; + int *positions; + int mutatorNColumns; + int count; +} MutatorState; + +static void +initMutator(MutatorState *state, List *elems, int start, int end) +{ + int i; + int n = end - start; + ListCell *lc; + + memset(state, 0, sizeof(*state)); + + state->mutatorNColumns = n; + + state->elemsList = list_copy(elems); + + state->elems = palloc(sizeof(void*) * n); + state->elemCells = palloc(sizeof(ListCell*) * n); + state->positions = palloc(sizeof(int) * n); + + i = 0; + for_each_cell(lc, list_nth_cell(state->elemsList, start)) + { + state->elemCells[i] = lc; + state->elems[i] = lfirst(lc); + state->positions[i] = i + 1; + i++; + if (i >= n) + break; + } +} + +static void +swap(int *a, int i, int j) +{ + int s = a[i]; + + a[i] = a[j]; + a[j] = s; +} + +static bool +getNextSet(int *a, int n) +{ + int j, k, l, r; + + j = n - 2; + while (j >= 0 && a[j] >= a[j + 1]) + j--; + if (j < 0) + return false; + + k = n - 1; + while (k >= 0 && a[j] >= a[k]) + k--; + swap(a, j, k); + + l = j + 1; + r = n - 1; + while (l < r) + swap(a, l++, r--); + + + return true; +} + +static List* +doMutator(MutatorState *state) +{ + int i; + + state->count++; + + /* first set is original set */ + if (state->count == 1) + return state->elemsList; + + if (getNextSet(state->positions, state->mutatorNColumns) == false) + { + pfree(state->elems); + pfree(state->elemCells); + pfree(state->positions); + list_free(state->elemsList); + + return NIL; + } + + for(i=0; i<state->mutatorNColumns; i++) + lfirst(state->elemCells[i]) = + (void*) state->elems[ state->positions[i] - 1 ]; + + return state->elemsList; +} + +typedef struct { + Cost cost; + PathKey *pathkey; +} PathkeySortCost; + +static int +pathkey_sort_cost_comparator(const void *_a, const void *_b) +{ + const PathkeySortCost *a = (PathkeySortCost *) _a; + const PathkeySortCost *b = (PathkeySortCost *) _b; + + if (a->cost < b->cost) + return -1; + else if (a->cost == b->cost) + return 0; + return 1; +} +/* + * Order tail of list of group pathkeys by uniqueness descendetly. It allows to + * speedup sorting. Returns newly allocated lists, old ones stay untouched. + * n_preordered defines a head of list which order should be prevented. + */ +void +get_cheapest_group_keys_order(PlannerInfo *root, double nrows, + List **group_pathkeys, List **group_clauses, + int n_preordered) +{ + List *new_group_pathkeys = NIL, + *new_group_clauses = NIL, + *var_group_pathkeys; + + ListCell *cell; + MutatorState mstate; + double cheapest_sort_cost = -1.0; + + int nFreeKeys; + int nToPermute; + int i; + + if (list_length(*group_pathkeys) - n_preordered < 2) + return; /* nothing to do */ + + /* + * Will try to match ORDER BY pathkeys in hope that one sort is cheaper than + * two + */ + if (n_preordered == 0 && root->sort_pathkeys) + { + n_preordered = group_keys_reorder_by_pathkeys(root->sort_pathkeys, + group_pathkeys, + group_clauses); + + if (list_length(*group_pathkeys) - n_preordered < 2) + return; /* nothing to do */ + } + + /* + * Try all permutations of at most 4 cheapeast pathkeys. + */ + nFreeKeys = list_length(*group_pathkeys) - n_preordered; + nToPermute = 4; + if (nFreeKeys > nToPermute) + { + /* + * Sort the remaining pathkeys cheapest first. + */ + PathkeySortCost *costs = palloc(sizeof(*costs) * nFreeKeys); + cell = list_nth_cell(*group_pathkeys, n_preordered); + for (i = 0; cell != NULL; i++, (cell = lnext(cell))) + { + List *to_cost = list_make1(lfirst(cell)); + + Assert(i < nFreeKeys); + + costs[i].pathkey = lfirst(cell); + costs[i].cost = cost_sort_estimate(root, to_cost, 0, nrows); + + pfree(to_cost); + } + qsort(costs, nFreeKeys, sizeof(*costs), pathkey_sort_cost_comparator); + + /* Construct the sorted list. First, the preordered pathkeys. */ + new_group_pathkeys = list_truncate(list_copy(*group_pathkeys), n_preordered); + + /* The rest, ordered by increasing cost */ + for (i = 0; i < nFreeKeys; i++) + new_group_pathkeys = lappend(new_group_pathkeys, costs[i].pathkey); + + pfree(costs); + } + else + { + new_group_pathkeys = *group_pathkeys; + nToPermute = nFreeKeys; + } + + initMutator(&mstate, new_group_pathkeys, n_preordered, n_preordered + nToPermute); + + while((var_group_pathkeys = doMutator(&mstate)) != NIL) + { + Cost cost; + + cost = cost_sort_estimate(root, var_group_pathkeys, n_preordered, nrows); + + if (cost < cheapest_sort_cost || cheapest_sort_cost < 0) + { + list_free(new_group_pathkeys); + new_group_pathkeys = list_copy(var_group_pathkeys); + cheapest_sort_cost = cost; + } + } + + /* + * repeat order of pathkeys for clauses + */ + foreach(cell, new_group_pathkeys) + { + PathKey *pathkey = (PathKey *) lfirst(cell); + + new_group_clauses = lappend(new_group_clauses, + get_sortgroupref_clause(pathkey->pk_eclass->ec_sortref, + *group_clauses)); + } + + /* Just append the rest GROUP BY clauses */ + new_group_clauses = list_concat_unique_ptr(new_group_clauses, + *group_clauses); + + *group_pathkeys = new_group_pathkeys; + *group_clauses = new_group_clauses; +} + /* * get_cheapest_path_for_pathkeys * Find the cheapest path (according to the specified criterion) that @@ -1602,6 +1889,39 @@ pathkeys_useful_for_ordering(PlannerInfo *root, List *pathkeys) return 0; /* path ordering not useful */ } +/* + * pathkeys_useful_for_grouping + * Count the number of pathkeys that are useful for grouping (instead of + * explicit sort) + * + * Group pathkeys could be reordered, so we don't bother about actual order in + * pathkeys + */ +static int +pathkeys_useful_for_grouping(PlannerInfo *root, List *pathkeys) +{ + ListCell *key; + int n = 0; + + if (root->group_pathkeys == NIL) + return 0; /* no special ordering requested */ + + if (pathkeys == NIL) + return 0; /* unordered path */ + + foreach(key, pathkeys) + { + PathKey *pathkey = (PathKey *) lfirst(key); + + if (!list_member_ptr(root->group_pathkeys, pathkey)) + break; + + n++; + } + + return n; +} + /* * truncate_useless_pathkeys * Shorten the given pathkey list to just the useful pathkeys. @@ -1616,6 +1936,9 @@ truncate_useless_pathkeys(PlannerInfo *root, nuseful = pathkeys_useful_for_merging(root, rel, pathkeys); nuseful2 = pathkeys_useful_for_ordering(root, pathkeys); + if (nuseful2 > nuseful) + nuseful = nuseful2; + nuseful2 = pathkeys_useful_for_grouping(root, pathkeys); if (nuseful2 > nuseful) nuseful = nuseful2; @@ -1651,6 +1974,8 @@ has_useful_pathkeys(PlannerInfo *root, RelOptInfo *rel) { if (rel->joininfo != NIL || rel->has_eclass_joins) return true; /* might be able to use pathkeys for merging */ + if (root->group_pathkeys != NIL) + return true; /* might be able to use pathkeys for grouping */ if (root->query_pathkeys != NIL) return true; /* might be able to use them for ordering */ return false; /* definitely useless */ diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 24db16f68d..e3c934549b 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3406,7 +3406,8 @@ get_number_of_groups(PlannerInfo *root, double numGroups = estimate_num_groups(root, groupExprs, path_rows, - &gset); + &gset, + NULL, 0); gs->numGroups = numGroups; rollup->numGroups += numGroups; @@ -3431,7 +3432,8 @@ get_number_of_groups(PlannerInfo *root, double numGroups = estimate_num_groups(root, groupExprs, path_rows, - &gset); + &gset, + NULL, 0); gs->numGroups = numGroups; gd->dNumHashGroups += numGroups; @@ -3447,7 +3449,8 @@ get_number_of_groups(PlannerInfo *root, parse->targetList); dNumGroups = estimate_num_groups(root, groupExprs, path_rows, - NULL); + NULL, + NULL, 0); } } else if (parse->groupingSets) @@ -3778,19 +3781,36 @@ create_grouping_paths(PlannerInfo *root, foreach(lc, input_rel->partial_pathlist) { Path *path = (Path *) lfirst(lc); + List *group_pathkeys = root->group_pathkeys, + *group_clauses = parse->groupClause; bool is_sorted; + int n_preordered_groups; + + + n_preordered_groups = + group_keys_reorder_by_pathkeys(path->pathkeys, + &group_pathkeys, + &group_clauses); + + is_sorted = (n_preordered_groups == + list_length(group_pathkeys)); - is_sorted = pathkeys_contained_in(root->group_pathkeys, - path->pathkeys); if (path == cheapest_partial_path || is_sorted) { /* Sort the cheapest partial path, if it isn't already */ if (!is_sorted) + { + get_cheapest_group_keys_order(root, + path->rows, + &group_pathkeys, + &group_clauses, + n_preordered_groups); path = (Path *) create_sort_path(root, grouped_rel, path, - root->group_pathkeys, + group_pathkeys, -1.0); + } if (parse->hasAggs) add_partial_path(grouped_rel, (Path *) @@ -3798,9 +3818,9 @@ create_grouping_paths(PlannerInfo *root, grouped_rel, path, partial_grouping_target, - parse->groupClause ? AGG_SORTED : AGG_PLAIN, + group_clauses ? AGG_SORTED : AGG_PLAIN, AGGSPLIT_INITIAL_SERIAL, - parse->groupClause, + group_clauses, NIL, &agg_partial_costs, dNumPartialGroups)); @@ -3810,7 +3830,7 @@ create_grouping_paths(PlannerInfo *root, grouped_rel, path, partial_grouping_target, - parse->groupClause, + group_clauses, NIL, dNumPartialGroups)); } @@ -3859,18 +3879,45 @@ create_grouping_paths(PlannerInfo *root, { Path *path = (Path *) lfirst(lc); bool is_sorted; + List *group_pathkeys = root->group_pathkeys, + *group_clauses = parse->groupClause; + int n_preordered_groups = 0; + + if (parse->groupingSets) + { + /* + * prevent group pathkey rreordering, just check the same + * order paths pathkeys and group pathkeys + */ + is_sorted = pathkeys_contained_in(group_pathkeys, + path->pathkeys); + } + else + { + n_preordered_groups = + group_keys_reorder_by_pathkeys(path->pathkeys, + &group_pathkeys, + &group_clauses); + is_sorted = (n_preordered_groups == list_length(group_pathkeys)); + } - is_sorted = pathkeys_contained_in(root->group_pathkeys, - path->pathkeys); if (path == cheapest_path || is_sorted) { /* Sort the cheapest-total path if it isn't already sorted */ if (!is_sorted) + { + if (!parse->groupingSets) + get_cheapest_group_keys_order(root, + path->rows, + &group_pathkeys, + &group_clauses, + n_preordered_groups); path = (Path *) create_sort_path(root, grouped_rel, path, - root->group_pathkeys, + group_pathkeys, -1.0); + } /* Now decide what to stick atop it */ if (parse->groupingSets) @@ -3890,9 +3937,9 @@ create_grouping_paths(PlannerInfo *root, grouped_rel, path, target, - parse->groupClause ? AGG_SORTED : AGG_PLAIN, + group_clauses ? AGG_SORTED : AGG_PLAIN, AGGSPLIT_SIMPLE, - parse->groupClause, + group_clauses, (List *) parse->havingQual, agg_costs, dNumGroups)); @@ -3908,7 +3955,7 @@ create_grouping_paths(PlannerInfo *root, grouped_rel, path, target, - parse->groupClause, + group_clauses, (List *) parse->havingQual, dNumGroups)); } @@ -4763,7 +4810,8 @@ create_distinct_paths(PlannerInfo *root, parse->targetList); numDistinctRows = estimate_num_groups(root, distinctExprs, cheapest_input_path->rows, - NULL); + NULL, + NULL, 0); } /* diff --git a/src/backend/optimizer/prep/prepunion.c b/src/backend/optimizer/prep/prepunion.c index ed81fea7b8..79ef365603 100644 --- a/src/backend/optimizer/prep/prepunion.c +++ b/src/backend/optimizer/prep/prepunion.c @@ -361,7 +361,7 @@ recurse_set_operations(Node *setOp, PlannerInfo *root, *pNumGroups = estimate_num_groups(subroot, get_tlist_exprs(subquery->targetList, false), subpath->rows, - NULL); + NULL, NULL, 0); } return (Path *) path; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 3cd7e37e13..4ca05159f7 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1546,7 +1546,7 @@ create_unique_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, pathnode->path.rows = estimate_num_groups(root, sjinfo->semi_rhs_exprs, rel->rows, - NULL); + NULL, NULL, 0); numCols = list_length(sjinfo->semi_rhs_exprs); if (sjinfo->semi_can_btree) diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 3d633de8ff..ebe8507660 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -3269,13 +3269,13 @@ add_unique_group_var(PlannerInfo *root, List *varinfos, */ double estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, - List **pgset) + List **pgset, List **cache_varinfos, int prevNExprs) { - List *varinfos = NIL; + List *varinfos = (cache_varinfos) ? *cache_varinfos : NIL; double srf_multiplier = 1.0; double numdistinct; ListCell *l; - int i; + int i, j; /* * We don't ever want to return an estimate of zero groups, as that tends @@ -3302,7 +3302,7 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, */ numdistinct = 1.0; - i = 0; + i = j = 0; foreach(l, groupExprs) { Node *groupexpr = (Node *) lfirst(l); @@ -3311,6 +3311,14 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, List *varshere; ListCell *l2; + /* was done on previous call */ + if (cache_varinfos && j++ < prevNExprs) + { + if (pgset) + i++; /* to keep in sync with lines below */ + continue; + } + /* is expression in this grouping set? */ if (pgset && !list_member_int(*pgset, i++)) continue; @@ -3372,7 +3380,11 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, if (varshere == NIL) { if (contain_volatile_functions(groupexpr)) + { + if (cache_varinfos) + *cache_varinfos = varinfos; return input_rows; + } continue; } @@ -3389,6 +3401,9 @@ estimate_num_groups(PlannerInfo *root, List *groupExprs, double input_rows, } } + if (cache_varinfos) + *cache_varinfos = varinfos; + /* * If now no Vars, we must have an all-constant or all-boolean GROUP BY * list. diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index ba9ad291b8..fb737b8c6a 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1665,7 +1665,6 @@ static struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, - /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 7925e4cc24..80bf9f0cd0 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -105,6 +105,8 @@ extern void cost_sort(Path *path, PlannerInfo *root, List *pathkeys, Cost input_cost, double tuples, int width, Cost comparison_cost, int sort_mem, double limit_tuples); +extern Cost cost_sort_estimate(PlannerInfo *root, List *pathkeys, + int nPresortedKeys, double tuples); extern void cost_merge_append(Path *path, PlannerInfo *root, List *pathkeys, int n_streams, Cost input_startup_cost, Cost input_total_cost, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 569d3f9b6b..a7155f5e74 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -184,6 +184,14 @@ typedef enum extern PathKeysComparison compare_pathkeys(List *keys1, List *keys2); extern bool pathkeys_contained_in(List *keys1, List *keys2); +extern int group_keys_reorder_by_pathkeys(List *pathkeys, + List **group_pathkeys, + List **group_clauses); +extern void get_cheapest_group_keys_order(PlannerInfo *root, + double nrows, + List **group_pathkeys, + List **group_clauses, + int n_preordered); extern Path *get_cheapest_path_for_pathkeys(List *paths, List *pathkeys, Relids required_outer, CostSelector cost_criterion, diff --git a/src/include/utils/selfuncs.h b/src/include/utils/selfuncs.h index c7fdd540e8..4f13589507 100644 --- a/src/include/utils/selfuncs.h +++ b/src/include/utils/selfuncs.h @@ -204,7 +204,8 @@ extern void mergejoinscansel(PlannerInfo *root, Node *clause, Selectivity *rightstart, Selectivity *rightend); extern double estimate_num_groups(PlannerInfo *root, List *groupExprs, - double input_rows, List **pgset); + double input_rows, List **pgset, + List **cache_varinfos, int prevNExprs); extern Selectivity estimate_hash_bucketsize(PlannerInfo *root, Node *hashkey, double nbuckets); diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index 56d5fbb4fb..ad2054ccd4 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -906,7 +906,8 @@ explain (costs off) select distinct min(f1), max(f1) from minmaxtest; QUERY PLAN ---------------------------------------------------------------------------------------------- - Unique + HashAggregate + Group Key: $0, $1 InitPlan 1 (returns $0) -> Limit -> Merge Append @@ -929,10 +930,8 @@ explain (costs off) -> Index Only Scan using minmaxtest2i on minmaxtest2 minmaxtest2_1 Index Cond: (f1 IS NOT NULL) -> Index Only Scan Backward using minmaxtest3i on minmaxtest3 minmaxtest3_1 - -> Sort - Sort Key: ($0), ($1) - -> Result -(26 rows) + -> Result +(25 rows) select distinct min(f1), max(f1) from minmaxtest; min | max @@ -2054,6 +2053,236 @@ SELECT balk(hundred) FROM tenk1; (1 row) ROLLBACK; +-- GROUP BY optimization by reorder columns +SELECT + i AS id, + i/2 AS p, + format('%60s', i%2) AS v, + i/4 AS c, + i/8 AS d, + (random() * (10000/8))::int as e --the same as d but no correlation with p + INTO btg +FROM + generate_series(1, 10000) i; +VACUUM btg; +ANALYZE btg; +-- GROUP BY optimization by reorder columns by frequency +SET enable_hashagg=off; +SET max_parallel_workers= 0; +SET max_parallel_workers_per_gather = 0; +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, v + -> Sort + Sort Key: p, v + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, v + -> Sort + Sort Key: p, v + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, c, v + -> Sort + Sort Key: p, c, v + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c ORDER BY v, p, c; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: v, p, c + -> Sort + Sort Key: v, p, c + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c; + QUERY PLAN +------------------------------ + GroupAggregate + Group Key: p, d, c, v + -> Sort + Sort Key: p, d, c, v + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c ORDER BY v, p, d ,c; + QUERY PLAN +------------------------------ + GroupAggregate + Group Key: v, p, d, c + -> Sort + Sort Key: v, p, d, c + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c ORDER BY p, v, d ,c; + QUERY PLAN +------------------------------ + GroupAggregate + Group Key: p, v, d, c + -> Sort + Sort Key: p, v, d, c + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, d, e; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, d, e + -> Sort + Sort Key: p, d, e + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, e, d; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, e, d + -> Sort + Sort Key: p, e, d + -> Seq Scan on btg +(5 rows) + +CREATE STATISTICS btg_dep ON d, e, p FROM btg; +ANALYZE btg; +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, d, e; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, d, e + -> Sort + Sort Key: p, d, e + -> Seq Scan on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, e, d; + QUERY PLAN +----------------------------- + GroupAggregate + Group Key: p, e, d + -> Sort + Sort Key: p, e, d + -> Seq Scan on btg +(5 rows) + +-- GROUP BY optimization by reorder columns by index scan +CREATE INDEX ON btg(p, v); +SET enable_seqscan=off; +SET enable_bitmapscan=off; +VACUUM btg; +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v; + QUERY PLAN +------------------------------------------------ + GroupAggregate + Group Key: p, v + -> Index Only Scan using btg_p_v_idx on btg +(3 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v ORDER BY p, v; + QUERY PLAN +------------------------------------------------ + GroupAggregate + Group Key: p, v + -> Index Only Scan using btg_p_v_idx on btg +(3 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p; + QUERY PLAN +------------------------------------------------ + GroupAggregate + Group Key: p, v + -> Index Only Scan using btg_p_v_idx on btg +(3 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p ORDER BY p, v; + QUERY PLAN +------------------------------------------------ + GroupAggregate + Group Key: p, v + -> Index Only Scan using btg_p_v_idx on btg +(3 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c; + QUERY PLAN +------------------------------------------------- + GroupAggregate + Group Key: p, v, c + -> Sort + Sort Key: p, v, c + -> Index Scan using btg_p_v_idx on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c ORDER BY p, v; + QUERY PLAN +------------------------------------------------- + GroupAggregate + Group Key: p, v, c + -> Sort + Sort Key: p, v, c + -> Index Scan using btg_p_v_idx on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, c, p, d; + QUERY PLAN +------------------------------------------------- + GroupAggregate + Group Key: p, v, c, d + -> Sort + Sort Key: p, v, c, d + -> Index Scan using btg_p_v_idx on btg +(5 rows) + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, c, p, d ORDER BY p, v; + QUERY PLAN +------------------------------------------------- + GroupAggregate + Group Key: p, v, c, d + -> Sort + Sort Key: p, v, c, d + -> Index Scan using btg_p_v_idx on btg +(5 rows) + +RESET enable_hashagg; +RESET max_parallel_workers; +RESET max_parallel_workers_per_gather; +RESET enable_seqscan; +RESET enable_bitmapscan; -- test coverage for aggregate combine/serial/deserial functions BEGIN ISOLATION LEVEL REPEATABLE READ; SET parallel_setup_cost = 0; diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out index 9097c4a97c..7fab6fcc3d 100644 --- a/src/test/regress/expected/join.out +++ b/src/test/regress/expected/join.out @@ -1906,8 +1906,8 @@ USING (name); ------+----+---- bb | 12 | 13 cc | 22 | 23 - dd | | 33 ee | 42 | + dd | | 33 (4 rows) -- Cases with non-nullable expressions in subquery results; @@ -1941,8 +1941,8 @@ NATURAL FULL JOIN ------+------+------+------+------ bb | 12 | 2 | 13 | 3 cc | 22 | 2 | 23 | 3 - dd | | | 33 | 3 ee | 42 | 2 | | + dd | | | 33 | 3 (4 rows) SELECT * FROM @@ -4165,18 +4165,20 @@ select d.* from d left join (select * from b group by b.id, b.c_id) s explain (costs off) select d.* from d left join (select distinct * from b) s on d.a = s.id; - QUERY PLAN --------------------------------------- - Merge Right Join - Merge Cond: (b.id = d.a) - -> Unique - -> Sort - Sort Key: b.id, b.c_id - -> Seq Scan on b + QUERY PLAN +--------------------------------------------- + Merge Left Join + Merge Cond: (d.a = s.id) -> Sort Sort Key: d.a -> Seq Scan on d -(9 rows) + -> Sort + Sort Key: s.id + -> Subquery Scan on s + -> HashAggregate + Group Key: b.id, b.c_id + -> Seq Scan on b +(11 rows) -- check join removal works when uniqueness of the join condition is enforced -- by a UNION @@ -5667,44 +5669,39 @@ select * from j1 natural join j2; explain (verbose, costs off) select * from j1 inner join (select distinct id from j3) j3 on j1.id = j3.id; - QUERY PLAN ------------------------------------------ + QUERY PLAN +----------------------------------- Nested Loop Output: j1.id, j3.id Inner Unique: true Join Filter: (j1.id = j3.id) - -> Unique + -> HashAggregate Output: j3.id - -> Sort + Group Key: j3.id + -> Seq Scan on public.j3 Output: j3.id - Sort Key: j3.id - -> Seq Scan on public.j3 - Output: j3.id -> Seq Scan on public.j1 Output: j1.id -(13 rows) +(11 rows) -- ensure group by clause allows the inner to become unique explain (verbose, costs off) select * from j1 inner join (select id from j3 group by id) j3 on j1.id = j3.id; - QUERY PLAN ------------------------------------------ + QUERY PLAN +----------------------------------- Nested Loop Output: j1.id, j3.id Inner Unique: true Join Filter: (j1.id = j3.id) - -> Group + -> HashAggregate Output: j3.id Group Key: j3.id - -> Sort + -> Seq Scan on public.j3 Output: j3.id - Sort Key: j3.id - -> Seq Scan on public.j3 - Output: j3.id -> Seq Scan on public.j1 Output: j1.id -(14 rows) +(11 rows) drop table j1; drop table j2; diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 054a381dad..7597bb27b8 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -281,9 +281,9 @@ EXPLAIN (COSTS off) QUERY PLAN ----------------------------------- GroupAggregate - Group Key: a, b + Group Key: b, a -> Sort - Sort Key: a, b + Sort Key: b, a -> Seq Scan on ndistinct (5 rows) @@ -292,9 +292,9 @@ EXPLAIN (COSTS off) QUERY PLAN ----------------------------------- GroupAggregate - Group Key: a, b, c + Group Key: b, a, c -> Sort - Sort Key: a, b, c + Sort Key: b, a, c -> Seq Scan on ndistinct (5 rows) @@ -303,9 +303,9 @@ EXPLAIN (COSTS off) QUERY PLAN ----------------------------------- GroupAggregate - Group Key: a, b, c, d + Group Key: d, a, b, c -> Sort - Sort Key: a, b, c, d + Sort Key: d, a, b, c -> Seq Scan on ndistinct (5 rows) diff --git a/src/test/regress/sql/aggregates.sql b/src/test/regress/sql/aggregates.sql index 651ca02536..fda02e7a00 100644 --- a/src/test/regress/sql/aggregates.sql +++ b/src/test/regress/sql/aggregates.sql @@ -901,6 +901,102 @@ SELECT balk(hundred) FROM tenk1; ROLLBACK; +-- GROUP BY optimization by reorder columns + +SELECT + i AS id, + i/2 AS p, + format('%60s', i%2) AS v, + i/4 AS c, + i/8 AS d, + (random() * (10000/8))::int as e --the same as d but no correlation with p + INTO btg +FROM + generate_series(1, 10000) i; + +VACUUM btg; +ANALYZE btg; + +-- GROUP BY optimization by reorder columns by frequency + +SET enable_hashagg=off; +SET max_parallel_workers= 0; +SET max_parallel_workers_per_gather = 0; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c ORDER BY v, p, c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c ORDER BY v, p, d ,c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, d, c ORDER BY p, v, d ,c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, d, e; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, e, d; + +CREATE STATISTICS btg_dep ON d, e, p FROM btg; +ANALYZE btg; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, d, e; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, e, d; + + +-- GROUP BY optimization by reorder columns by index scan + +CREATE INDEX ON btg(p, v); +SET enable_seqscan=off; +SET enable_bitmapscan=off; +VACUUM btg; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY p, v ORDER BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p ORDER BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, p, c ORDER BY p, v; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, c, p, d; + +EXPLAIN (COSTS off) +SELECT count(*) FROM btg GROUP BY v, c, p, d ORDER BY p, v; + +RESET enable_hashagg; +RESET max_parallel_workers; +RESET max_parallel_workers_per_gather; +RESET enable_seqscan; +RESET enable_bitmapscan; + -- test coverage for aggregate combine/serial/deserial functions BEGIN ISOLATION LEVEL REPEATABLE READ;
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
