Merge remote-tracking branch 'upstream/master' into factorize

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -153,3 +153,12 @@ jobs:
       run: |
         source activate pandas-dev
         pytest pandas/tests/frame/methods --array-manager
+        pytest pandas/tests/arithmetic/ --array-manager
+
+        # indexing subset (temporary since other tests don't pass yet)
+        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean --array-manager
+        pytest pandas/tests/frame/indexing/test_where.py --array-manager
+        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_multi_index --array-manager
+        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns --array-manager
+        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups --array-manager
+        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column --array-manager

.github/workflows/database.yml

@@ -170,3 +170,11 @@ jobs:
 
     - name: Print skipped tests
       run: python ci/print_skipped.py
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        files: /tmp/test_coverage.xml
+        flags: unittests
+        name: codecov-pandas
+        fail_ci_if_error: true

.pre-commit-config.yaml

@@ -180,6 +180,12 @@ repos:
         language: pygrep
         types: [python]
         files: ^pandas/tests/
+    -   id: title-capitalization
+        name: Validate correct capitalization among titles in documentation
+        entry: python scripts/validate_rst_title_capitalization.py
+        language: python
+        types: [rst]
+        files: ^doc/source/(development|reference)/
 -   repo: https://github.com/asottile/yesqa
     rev: v1.2.2
     hooks:

MANIFEST.in

@@ -1,9 +1,4 @@
-include MANIFEST.in
-include LICENSE
 include RELEASE.md
-include README.md
-include setup.py
-include pyproject.toml
 
 graft doc
 prune doc/build
@@ -16,10 +11,12 @@ global-exclude *.bz2
 global-exclude *.csv
 global-exclude *.dta
 global-exclude *.feather
+global-exclude *.tar
 global-exclude *.gz
 global-exclude *.h5
 global-exclude *.html
 global-exclude *.json
+global-exclude *.jsonl
 global-exclude *.pickle
 global-exclude *.png
 global-exclude *.pyc
@@ -40,6 +37,11 @@ global-exclude .DS_Store
 global-exclude .git*
 global-exclude \#*
 
+# GH 39321
+# csv_dir_path fixture checks the existence of the directory
+# exclude the whole directory to avoid running related tests in sdist
+prune pandas/tests/io/parser/data
+
 include versioneer.py
 include pandas/_version.py
 include pandas/io/formats/templates/*.tpl

asv_bench/benchmarks/arithmetic.py

@@ -4,7 +4,13 @@
 import numpy as np
 
 import pandas as pd
-from pandas import DataFrame, Series, Timestamp, date_range, to_timedelta
+from pandas import (
+    DataFrame,
+    Series,
+    Timestamp,
+    date_range,
+    to_timedelta,
+)
 import pandas._testing as tm
 from pandas.core.algorithms import checked_add_with_arr
 
@@ -110,16 +116,26 @@ class FrameWithFrameWide:
             operator.add,
             operator.floordiv,
             operator.gt,
-        ]
+        ],
+        [
+            # (n_rows, n_columns)
+            (1_000_000, 10),
+            (100_000, 100),
+            (10_000, 1000),
+            (1000, 10_000),
+        ],
     ]
-    param_names = ["op"]
+    param_names = ["op", "shape"]
 
-    def setup(self, op):
+    def setup(self, op, shape):
         # we choose dtypes so as to make the blocks
         #  a) not perfectly match between right and left
         #  b) appreciably bigger than single columns
-        n_cols = 2000
-        n_rows = 500
+        n_rows, n_cols = shape
+
+        if op is operator.floordiv:
+            # floordiv is much slower than the other operations -> use less data
+            n_rows = n_rows // 10
 
         # construct dataframe with 2 blocks
         arr1 = np.random.randn(n_rows, n_cols // 2).astype("f8")
@@ -131,7 +147,7 @@ def setup(self, op):
         df._consolidate_inplace()
 
         # TODO: GH#33198 the setting here shoudlnt need two steps
-        arr1 = np.random.randn(n_rows, n_cols // 4).astype("f8")
+        arr1 = np.random.randn(n_rows, max(n_cols // 4, 3)).astype("f8")
         arr2 = np.random.randn(n_rows, n_cols // 2).astype("i8")
         arr3 = np.random.randn(n_rows, n_cols // 4).astype("f8")
         df2 = pd.concat(
@@ -145,11 +161,11 @@ def setup(self, op):
         self.left = df
         self.right = df2
 
-    def time_op_different_blocks(self, op):
+    def time_op_different_blocks(self, op, shape):
         # blocks (and dtypes) are not aligned
         op(self.left, self.right)
 
-    def time_op_same_blocks(self, op):
+    def time_op_same_blocks(self, op, shape):
         # blocks (and dtypes) are aligned
         op(self.left, self.left)
 

asv_bench/benchmarks/ctors.py

@@ -1,6 +1,12 @@
 import numpy as np
 
-from pandas import DatetimeIndex, Index, MultiIndex, Series, Timestamp
+from pandas import (
+    DatetimeIndex,
+    Index,
+    MultiIndex,
+    Series,
+    Timestamp,
+)
 
 from .pandas_vb_common import tm
 

asv_bench/benchmarks/dtypes.py

@@ -5,7 +5,10 @@
 import pandas as pd
 from pandas import DataFrame
 import pandas._testing as tm
-from pandas.api.types import is_extension_array_dtype, pandas_dtype
+from pandas.api.types import (
+    is_extension_array_dtype,
+    pandas_dtype,
+)
 
 from .pandas_vb_common import (
     datetime_dtypes,

asv_bench/benchmarks/frame_ctor.py

@@ -1,12 +1,21 @@
 import numpy as np
 
 import pandas as pd
-from pandas import DataFrame, MultiIndex, Series, Timestamp, date_range
+from pandas import (
+    DataFrame,
+    MultiIndex,
+    Series,
+    Timestamp,
+    date_range,
+)
 
 from .pandas_vb_common import tm
 
 try:
-    from pandas.tseries.offsets import Hour, Nano
+    from pandas.tseries.offsets import (
+        Hour,
+        Nano,
+    )
 except ImportError:
     # For compatibility with older versions
     from pandas.core.datetools import *  # noqa

asv_bench/benchmarks/frame_methods.py

@@ -3,7 +3,15 @@
 
 import numpy as np
 
-from pandas import DataFrame, MultiIndex, NaT, Series, date_range, isnull, period_range
+from pandas import (
+    DataFrame,
+    MultiIndex,
+    NaT,
+    Series,
+    date_range,
+    isnull,
+    period_range,
+)
 
 from .pandas_vb_common import tm
 

asv_bench/benchmarks/gil.py

@@ -1,6 +1,12 @@
 import numpy as np
 
-from pandas import DataFrame, Series, date_range, factorize, read_csv
+from pandas import (
+    DataFrame,
+    Series,
+    date_range,
+    factorize,
+    read_csv,
+)
 from pandas.core.algorithms import take_nd
 
 from .pandas_vb_common import tm