Performance regression of reduction operations

[YarShev]

Describe the problem

After this commit we got performance regression for several operations. An example for mean is below:

Source code / logs

import numpy as np
NCOLS = 2 ** 10
NROWS = 2 ** 10
RAND_LOW = 0
RAND_HIGH = 100
random_state = np.random.RandomState(seed=42)
from timeit import default_timer as timer
def measure(df):
     start = timer()
     df.mean(axis=0)
     end = timer()
     print(end - start)
float_nan_data = {
      "col{}".format(int((i - NCOLS / 2) % NCOLS + 1)): [
          x if (j % 4 == 0 and i > NCOLS // 2) or (j != i and i <= NCOLS // 2) else np.NaN
          for j, x in enumerate(random_state.uniform(RAND_LOW, RAND_HIGH, size=(NROWS)))
      ]
       for i in range(NCOLS)
  }
import modin.pandas as pd
df = pd.DataFrame(float_nan_data)
measure(df)
before the commit
0.6 s
after the commit
1.4 s

[dchigarev]

Regression:

Method name	Before 55e3459	After 55e3459	Degradation xTIMES
mean	0.55s	1.12s	2.03x
std	0.23s	0.77s	3.3x
kurt	0.23s	0.76s	3.3x
max	0.51s	1.07s	2.09x
median	0.23s	0.76s	3.3x
min	0.52s	1.06s	2.03x
prod	0.51s	0.97s	1.9x
sem	0.23s	0.76s	3.3x
skew	0.23s	0.76s	3.3x
sum	0.50s	1.06s	2.12x
var	0.22s	0.77s	3.3x

script

import pandas
import modin.pandas as pd

from timeit import default_timer as timer
import numpy as np

NCOLS = 2 ** 10
NROWS = 2 ** 10
RAND_LOW = 0
RAND_HIGH = 100
random_state = np.random.RandomState(seed=42)


def measure(df, op):
    start = timer()
    getattr(df, op)(axis=0)
    end = timer()
    return end - start


float_nan_data = {
    f"col{int((i - NCOLS /2)% NCOLS + 1)}": [
        x if (j % 4 == 0 and i > NCOLS // 2) or (j != i and i <= NCOLS // 2) else np.NaN
        for j, x in enumerate(random_state.uniform(RAND_LOW, RAND_HIGH, size=(NROWS)))
    ]
    for i in range(NCOLS)
}

operations = ["mean", "std", "kurt", "max", "median", "min", "prod", "sem", "skew", "sum", "var"]
NTRIES = 20

df = pd.DataFrame(float_nan_data)
for op in operations:
    result = np.min([measure(df, op) for _ in range(NTRIES)])
    print(f"Min time of {NTRIES} tries for {op}: {result}")