Merge branch 'master' into period_immutable

Author: jbrockmendel

appveyor.yml

@@ -72,7 +72,7 @@ install:
   - cmd: conda info -a
 
   # create our env
-  - cmd: conda create -n pandas python=%PYTHON_VERSION% cython pytest pytest-xdist
+  - cmd: conda create -n pandas python=%PYTHON_VERSION% cython pytest>=3.1.0 pytest-xdist
   - cmd: activate pandas
   - SET REQ=ci\requirements-%PYTHON_VERSION%_WIN.run
   - cmd: echo "installing requirements from %REQ%"

asv_bench/benchmarks/timeseries.py

@@ -510,3 +510,17 @@ def time_begin_incr_rng(self):
 
     def time_begin_decr_rng(self):
         self.rng - self.semi_month_begin
+
+
+class DatetimeAccessor(object):
+    def setup(self):
+        self.N = 100000
+        self.series = pd.Series(
+            pd.date_range(start='1/1/2000', periods=self.N, freq='T')
+        )
+
+    def time_dt_accessor(self):
+        self.series.dt
+
+    def time_dt_accessor_normalize(self):
+        self.series.dt.normalize()

ci/install_circle.sh

@@ -64,7 +64,7 @@ fi
 # create envbuild deps
 echo "[create env: ${REQ_BUILD}]"
 time conda create -n pandas -q --file=${REQ_BUILD} || exit 1
-time conda install -n pandas pytest || exit 1
+time conda install -n pandas pytest>=3.1.0 || exit 1
 
 source activate pandas
 

ci/install_travis.sh

@@ -103,7 +103,7 @@ if [ -e ${REQ} ]; then
     time bash $REQ || exit 1
 fi
 
-time conda install -n pandas pytest
+time conda install -n pandas pytest>=3.1.0
 time pip install pytest-xdist
 
 if [ "$LINT" ]; then

ci/requirements_all.txt

@@ -1,4 +1,4 @@
-pytest
+pytest>=3.1.0
 pytest-cov
 pytest-xdist
 flake8

ci/requirements_dev.txt

@@ -2,6 +2,6 @@ python-dateutil
 pytz
 numpy
 cython
-pytest
+pytest>=3.1.0
 pytest-cov
 flake8

doc/source/contributing.rst

@@ -598,6 +598,10 @@ Like many packages, *pandas* uses `pytest
 extensions in `numpy.testing
 <http://docs.scipy.org/doc/numpy/reference/routines.testing.html>`_.
 
+.. note::
+
+   The earliest supported pytest version is 3.1.0.
+
 Writing tests
 ~~~~~~~~~~~~~
 
@@ -654,7 +658,9 @@ Using ``pytest``
 Here is an example of a self-contained set of tests that illustrate multiple features that we like to use.
 
 - functional style: tests are like ``test_*`` and *only* take arguments that are either fixtures or parameters
+- ``pytest.mark`` can be used to set metadata on test functions, e.g. ``skip`` or ``xfail``.
 - using ``parametrize``: allow testing of multiple cases
+- to set a mark on a parameter, ``pytest.param(..., marks=...)`` syntax should be used
 - ``fixture``, code for object construction, on a per-test basis
 - using bare ``assert`` for scalars and truth-testing
 - ``tm.assert_series_equal`` (and its counter part ``tm.assert_frame_equal``), for pandas object comparisons.
@@ -673,6 +679,13 @@ We would name this file ``test_cool_feature.py`` and put in an appropriate place
    def test_dtypes(dtype):
        assert str(np.dtype(dtype)) == dtype
 
+   @pytest.mark.parametrize('dtype', ['float32',
+       pytest.param('int16', marks=pytest.mark.skip),
+       pytest.param('int32',
+                    marks=pytest.mark.xfail(reason='to show how it works'))])
+   def test_mark(dtype):
+       assert str(np.dtype(dtype)) == 'float32'
+
    @pytest.fixture
    def series():
        return pd.Series([1, 2, 3])
@@ -695,13 +708,16 @@ A test run of this yields
 
    ((pandas) bash-3.2$ pytest  test_cool_feature.py  -v
    =========================== test session starts ===========================
-   platform darwin -- Python 3.5.2, pytest-3.0.5, py-1.4.31, pluggy-0.4.0
-   collected 8 items
+   platform darwin -- Python 3.6.2, pytest-3.2.1, py-1.4.31, pluggy-0.4.0
+   collected 11 items
 
    tester.py::test_dtypes[int8] PASSED
    tester.py::test_dtypes[int16] PASSED
    tester.py::test_dtypes[int32] PASSED
    tester.py::test_dtypes[int64] PASSED
+   tester.py::test_mark[float32] PASSED
+   tester.py::test_mark[int16] SKIPPED
+   tester.py::test_mark[int32] xfail
    tester.py::test_series[int8] PASSED
    tester.py::test_series[int16] PASSED
    tester.py::test_series[int32] PASSED
@@ -714,8 +730,8 @@ Tests that we have ``parametrized`` are now accessible via the test name, for ex
 
    ((pandas) bash-3.2$ pytest  test_cool_feature.py  -v -k int8
    =========================== test session starts ===========================
-   platform darwin -- Python 3.5.2, pytest-3.0.5, py-1.4.31, pluggy-0.4.0
-   collected 8 items
+   platform darwin -- Python 3.6.2, pytest-3.2.1, py-1.4.31, pluggy-0.4.0
+   collected 11 items
 
    test_cool_feature.py::test_dtypes[int8] PASSED
    test_cool_feature.py::test_series[int8] PASSED

doc/source/io.rst

@@ -2020,6 +2020,13 @@ into a flat table.
 .. ipython:: python
 
    from pandas.io.json import json_normalize
+   data = [{'id': 1, 'name': {'first': 'Coleen', 'last': 'Volk'}},
+           {'name': {'given': 'Mose', 'family': 'Regner'}},
+           {'id': 2, 'name': 'Faye Raker'}]
+   json_normalize(data)
+
+.. ipython:: python
+
    data = [{'state': 'Florida',
              'shortname': 'FL',
              'info': {

doc/source/whatsnew/v0.21.0.txt

@@ -126,6 +126,8 @@ Other Enhancements
 - :func:`date_range` now accepts 'Y' in addition to 'A' as an alias for end of year (:issue:`9313`)
 - Integration with `Apache Parquet <https://parquet.apache.org/>`__, including a new top-level :func:`read_parquet` and :func:`DataFrame.to_parquet` method, see :ref:`here <io.parquet>`.
 - :func:`DataFrame.add_prefix` and :func:`DataFrame.add_suffix` now accept strings containing the '%' character. (:issue:`17151`)
+- `read_*` methods can now infer compression from non-string paths, such as ``pathlib.Path`` objects (:issue:`17206`).
+- :func:`pd.read_sas()` now recognizes much more of the most frequently used date (datetime) formats in SAS7BDAT files (:issue:`15871`).
 
 .. _whatsnew_0210.api_breaking:
 
@@ -275,7 +277,7 @@ Other API Changes
 - Removed the ``@slow`` decorator from ``pandas.util.testing``, which caused issues for some downstream packages' test suites. Use ``@pytest.mark.slow`` instead, which achieves the same thing (:issue:`16850`)
 - Moved definition of ``MergeError`` to the ``pandas.errors`` module.
 - The signature of :func:`Series.set_axis` and :func:`DataFrame.set_axis` has been changed from ``set_axis(axis, labels)`` to ``set_axis(labels, axis=0)``, for consistency with the rest of the API. The old signature is deprecated and will show a ``FutureWarning`` (:issue:`14636`)
-
+- :func:`Series.argmin` and :func:`Series.argmax` will now raise a ``TypeError`` when used with ``object`` dtypes, instead of a ``ValueError`` (:issue:`13595`)
 
 .. _whatsnew_0210.deprecations:
 
@@ -306,6 +308,7 @@ Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Improved performance of instantiating :class:`SparseDataFrame` (:issue:`16773`)
+- :attr:`Series.dt` no longer performs frequency inference, yielding a large speedup when accessing the attribute (:issue:`17210`)
 
 
 .. _whatsnew_0210.bug_fixes:
@@ -317,8 +320,11 @@ Bug Fixes
 Conversion
 ^^^^^^^^^^
 
-- Bug in assignment against datetime-like data with ``int`` may incorrectly converte to datetime-like (:issue:`14145`)
+- Bug in assignment against datetime-like data with ``int`` may incorrectly convert to datetime-like (:issue:`14145`)
 - Bug in assignment against ``int64`` data with ``np.ndarray`` with ``float64`` dtype may keep ``int64`` dtype (:issue:`14001`)
+- Fix :func:`DataFrame.memory_usage` to support PyPy. Objects on PyPy do not have a fixed size, so an approximation is used instead (:issue:`17228`)
+- Fixed the return type of ``IntervalIndex.is_non_overlapping_monotonic`` to be a Python ``bool`` for consistency with similar attributes/methods.  Previously returned a ``numpy.bool_``. (:issue:`17237`)
+- Bug in ``IntervalIndex.is_non_overlapping_monotonic`` when intervals are closed on both sides and overlap at a point (:issue:`16560`)
 
 
 Indexing
@@ -379,6 +385,7 @@ Reshaping
 - Fixes regression from 0.20, :func:`Series.aggregate` and :func:`DataFrame.aggregate` allow dictionaries as return values again (:issue:`16741`)
 - Fixes dtype of result with integer dtype input, from :func:`pivot_table` when called with ``margins=True`` (:issue:`17013`)
 - Bug in :func:`crosstab` where passing two ``Series`` with the same name raised a ``KeyError`` (:issue:`13279`)
+- :func:`Series.argmin`, :func:`Series.argmax`, and their counterparts on ``DataFrame`` and groupby objects work correctly with floating point data that contains infinite values (:issue:`13595`).
 
 Numeric
 ^^^^^^^
@@ -388,6 +395,9 @@ Numeric
 Categorical
 ^^^^^^^^^^^
 - Bug in :func:`Series.isin` when called with a categorical (:issue`16639`)
+- Bug in the categorical constructor with empty values and categories causing
+  the ``.categories`` to be an empty ``Float64Index`` rather than an empty
+  ``Index`` with object dtype (:issue:`17248`)
 
 
 Other

pandas/_libs/hashtable.pyx

@@ -2,7 +2,27 @@
 
 from cpython cimport PyObject, Py_INCREF, PyList_Check, PyTuple_Check
 
-from khash cimport *
+from khash cimport (
+    khiter_t,
+
+    kh_str_t, kh_init_str, kh_put_str, kh_exist_str,
+    kh_get_str, kh_destroy_str, kh_resize_str,
+
+    kh_put_strbox, kh_get_strbox, kh_init_strbox,
+
+    kh_int64_t, kh_init_int64, kh_resize_int64, kh_destroy_int64,
+    kh_get_int64, kh_exist_int64, kh_put_int64,
+
+    kh_float64_t, kh_exist_float64, kh_put_float64, kh_init_float64,
+    kh_get_float64, kh_destroy_float64, kh_resize_float64,
+
+    kh_resize_uint64, kh_exist_uint64, kh_destroy_uint64, kh_put_uint64,
+    kh_get_uint64, kh_init_uint64,
+
+    kh_destroy_pymap, kh_exist_pymap, kh_init_pymap, kh_get_pymap,
+    kh_put_pymap, kh_resize_pymap)
+
+
 from numpy cimport *
 
 from libc.stdlib cimport malloc, free