Merge remote-tracking branch 'upstream/master'

Author: Matt Roeschke

ci/azure/posix.yml

@@ -33,6 +33,12 @@ jobs:
           PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "it_IT.UTF-8"
 
+        py36_32bit:
+          ENV_FILE: ci/deps/azure-36-32bit.yaml
+          CONDA_PY: "36"
+          PATTERN: "not slow and not network"
+          BITS32: "yes"
+
         py37_locale:
           ENV_FILE: ci/deps/azure-37-locale.yaml
           CONDA_PY: "37"

ci/deps/azure-36-32bit.yaml

@@ -0,0 +1,20 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - gcc_linux-32
+  - gcc_linux-32
+  - gxx_linux-32
+  - cython=0.28.2
+  - numpy=1.14.*
+  - python-dateutil
+  - python=3.6.*
+  - pytz=2017.2
+  # universal
+  - pytest>=4.0.2,<5.0.0
+  - pytest-xdist
+  - pytest-mock
+  - pytest-azurepipelines
+  - hypothesis>=3.58.0
+  - pip

ci/deps/azure-37-numpydev.yaml

@@ -17,4 +17,5 @@ dependencies:
     - "--pre"
     - "numpy"
     - "scipy"
-    - pytest-azurepipelines
+    # https://github.com/pandas-dev/pandas/issues/27421
+    - pytest-azurepipelines<1.0.0

ci/deps/azure-macos-35.yaml

@@ -29,4 +29,6 @@ dependencies:
     - pytest-xdist
     - pytest-mock
     - hypothesis>=3.58.0
-    - pytest-azurepipelines
+    # https://github.com/pandas-dev/pandas/issues/27421
+    - pytest-azurepipelines<1.0.0
+

ci/setup_env.sh

@@ -94,6 +94,12 @@ echo
 echo "conda env create -q --file=${ENV_FILE}"
 time conda env create -q --file="${ENV_FILE}"
 
+
+if [[ "$BITS32" == "yes" ]]; then
+    # activate 32-bit compiler
+    export CONDA_BUILD=1
+fi
+
 echo "activate pandas-dev"
 source activate pandas-dev
 

doc/source/development/contributing.rst

@@ -288,7 +288,7 @@ complex changes to the documentation as well.
 Some other important things to know about the docs:
 
 * The *pandas* documentation consists of two parts: the docstrings in the code
-  itself and the docs in this folder ``pandas/doc/``.
+  itself and the docs in this folder ``doc/``.
 
   The docstrings provide a clear explanation of the usage of the individual
   functions, while the documentation in this folder consists of tutorial-like
@@ -404,11 +404,11 @@ Building the documentation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 So how do you build the docs? Navigate to your local
-``pandas/doc/`` directory in the console and run::
+``doc/`` directory in the console and run::
 
     python make.py html
 
-Then you can find the HTML output in the folder ``pandas/doc/build/html/``.
+Then you can find the HTML output in the folder ``doc/build/html/``.
 
 The first time you build the docs, it will take quite a while because it has to run
 all the code examples and build all the generated docstring pages. In subsequent
@@ -448,7 +448,7 @@ You can also specify to use multiple cores to speed up the documentation build::
 Open the following file in a web browser to see the full documentation you
 just built::
 
-    pandas/docs/build/html/index.html
+    doc/build/html/index.html
 
 And you'll have the satisfaction of seeing your new and improved documentation!
 

doc/source/reference/extensions.rst

@@ -18,10 +18,44 @@ objects.
    api.extensions.register_series_accessor
    api.extensions.register_index_accessor
    api.extensions.ExtensionDtype
-   api.extensions.ExtensionArray
 
 .. autosummary::
    :toctree: api/
    :template: autosummary/class_without_autosummary.rst
 
+   api.extensions.ExtensionArray
    arrays.PandasArray
+
+.. We need this autosummary so that methods and attributes are generated.
+.. Separate block, since they aren't classes.
+
+   .. autosummary::
+      :toctree: api/
+
+      api.extensions.ExtensionArray._concat_same_type
+      api.extensions.ExtensionArray._formatter
+      api.extensions.ExtensionArray._formatting_values
+      api.extensions.ExtensionArray._from_factorized
+      api.extensions.ExtensionArray._from_sequence
+      api.extensions.ExtensionArray._from_sequence_of_strings
+      api.extensions.ExtensionArray._ndarray_values
+      api.extensions.ExtensionArray._reduce
+      api.extensions.ExtensionArray._values_for_argsort
+      api.extensions.ExtensionArray._values_for_factorize
+      api.extensions.ExtensionArray.argsort
+      api.extensions.ExtensionArray.astype
+      api.extensions.ExtensionArray.copy
+      api.extensions.ExtensionArray.dropna
+      api.extensions.ExtensionArray.factorize
+      api.extensions.ExtensionArray.fillna
+      api.extensions.ExtensionArray.isna
+      api.extensions.ExtensionArray.ravel
+      api.extensions.ExtensionArray.repeat
+      api.extensions.ExtensionArray.searchsorted
+      api.extensions.ExtensionArray.shift
+      api.extensions.ExtensionArray.take
+      api.extensions.ExtensionArray.unique
+      api.extensions.ExtensionArray.dtype
+      api.extensions.ExtensionArray.nbytes
+      api.extensions.ExtensionArray.ndim
+      api.extensions.ExtensionArray.shape

doc/source/whatsnew/v0.25.0.rst

@@ -400,7 +400,7 @@ of ``object`` dtype. :attr:`Series.str` will now infer the dtype data *within* t
 .. _whatsnew_0250.api_breaking.groupby_categorical:
 
 Categorical dtypes are preserved during groupby
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Previously, columns that were categorical, but not the groupby key(s) would be converted to ``object`` dtype during groupby operations. Pandas now will preserve these dtypes. (:issue:`18502`)
 
@@ -740,6 +740,47 @@ consistent with NumPy and the rest of pandas (:issue:`21801`).
    cat.argsort()
    cat[cat.argsort()]
 
+.. _whatsnew_0250.api_breaking.list_of_dict:
+
+Column order is preserved when passing a list of dicts to DataFrame
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Starting with Python 3.7 the key-order of ``dict`` is `guaranteed <https://mail.python.org/pipermail/python-dev/2017-December/151283.html>`_. In practice, this has been true since
+Python 3.6. The :class:`DataFrame` constructor now treats a list of dicts in the same way as
+it does a list of ``OrderedDict``, i.e. preserving the order of the dicts.
+This change applies only when pandas is running on Python>=3.6 (:issue:`27309`).
+
+.. ipython:: python
+
+   data = [
+       {'name': 'Joe', 'state': 'NY', 'age': 18},
+       {'name': 'Jane', 'state': 'KY', 'age': 19, 'hobby': 'Minecraft'},
+       {'name': 'Jean', 'state': 'OK', 'age': 20, 'finances': 'good'}
+   ]
+
+*Previous Behavior*:
+
+The columns were lexicographically sorted previously,
+
+.. code-block:: python
+
+   In [1]: pd.DataFrame(data)
+   Out[1]:
+      age finances      hobby  name state
+   0   18      NaN        NaN   Joe    NY
+   1   19      NaN  Minecraft  Jane    KY
+   2   20     good        NaN  Jean    OK
+
+*New Behavior*:
+
+The column order now matches the insertion-order of the keys in the ``dict``,
+considering all the records from top to bottom. As a consequence, the column
+order of the resulting DataFrame has changed compared to previous pandas verisons.
+
+.. ipython:: python
+
+   pd.DataFrame(data)
+
 .. _whatsnew_0250.api_breaking.deps:
 
 Increased minimum versions for dependencies
@@ -939,6 +980,7 @@ Performance improvements
 - Improved performance by removing the need for a garbage collect when checking for ``SettingWithCopyWarning`` (:issue:`27031`)
 - For :meth:`to_datetime` changed default value of cache parameter to ``True`` (:issue:`26043`)
 - Improved performance of :class:`DatetimeIndex` and :class:`PeriodIndex` slicing given non-unique, monotonic data (:issue:`27136`).
+- Improved performance of :meth:`pd.read_json` for index-oriented data. (:issue:`26773`)
 
 .. _whatsnew_0250.bug_fixes:
 
@@ -995,6 +1037,7 @@ Timezones
 - Bug in :func:`DataFrame.join` where joining a timezone aware index with a timezone aware column would result in a column of ``NaN`` (:issue:`26335`)
 - Bug in :func:`date_range` where ambiguous or nonexistent start or end times were not handled by the ``ambiguous`` or ``nonexistent`` keywords respectively (:issue:`27088`)
 - Bug in :meth:`DatetimeIndex.union` when combining a timezone aware and timezone unaware :class:`DatetimeIndex` (:issue:`21671`)
+- Bug when applying a numpy reduction function (e.g. :meth:`numpy.minimum`) to a timezone aware :class:`Series` (:issue:`15552`)
 
 Numeric
 ^^^^^^^
@@ -1054,7 +1097,8 @@ Indexing
 - Bug in :class:`CategoricalIndex` and :class:`Categorical` incorrectly raising ``ValueError`` instead of ``TypeError`` when a list is passed using the ``in`` operator (``__contains__``) (:issue:`21729`)
 - Bug in setting a new value in a :class:`Series` with a :class:`Timedelta` object incorrectly casting the value to an integer (:issue:`22717`)
 - Bug in :class:`Series` setting a new key (``__setitem__``) with a timezone-aware datetime incorrectly raising ``ValueError`` (:issue:`12862`)
--
+- Bug in :meth:`DataFrame.iloc` when indexing with a read-only indexer (:issue:`17192`)
+- Bug in :class:`Series` setting an existing tuple key (``__setitem__``) with timezone-aware datetime values incorrectly raising ``TypeError`` (:issue:`20441`)
 
 Missing
 ^^^^^^^
@@ -1086,7 +1130,6 @@ I/O
 - Bug in :meth:`DataFrame.to_html` where header numbers would ignore display options when rounding (:issue:`17280`)
 - Bug in :func:`read_hdf` where reading a table from an HDF5 file written directly with PyTables fails with a ``ValueError`` when using a sub-selection via the ``start`` or ``stop`` arguments (:issue:`11188`)
 - Bug in :func:`read_hdf` not properly closing store after a ``KeyError`` is raised (:issue:`25766`)
-- Bug in ``read_csv`` which would not raise ``ValueError`` if a column index in ``usecols`` was out of bounds (:issue:`25623`)
 - Improved the explanation for the failure when value labels are repeated in Stata dta files and suggested work-arounds (:issue:`25772`)
 - Improved :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` to read incorrectly formatted 118 format files saved by Stata (:issue:`25960`)
 - Improved the ``col_space`` parameter in :meth:`DataFrame.to_html` to accept a string so CSS length values can be set correctly (:issue:`25941`)
@@ -1103,6 +1146,7 @@ I/O
 - Bug in :meth:`read_hdf` where reading a timezone aware :class:`DatetimeIndex` would raise a ``TypeError`` (:issue:`11926`)
 - Bug in :meth:`to_msgpack` and :meth:`read_msgpack` which would raise a ``ValueError`` rather than a ``FileNotFoundError`` for an invalid path (:issue:`27160`)
 - Fixed bug in :meth:`DataFrame.to_parquet` which would raise a ``ValueError`` when the dataframe had no columns (:issue:`27339`)
+- Allow parsing of :class:`PeriodDtype` columns when using :func:`read_csv` (:issue:`26934`)
 
 Plotting
 ^^^^^^^^
@@ -1111,7 +1155,7 @@ Plotting
 - Bug in an error message in :meth:`DataFrame.plot`. Improved the error message if non-numerics are passed to :meth:`DataFrame.plot` (:issue:`25481`)
 - Bug in incorrect ticklabel positions when plotting an index that are non-numeric / non-datetime (:issue:`7612`, :issue:`15912`, :issue:`22334`)
 - Fixed bug causing plots of :class:`PeriodIndex` timeseries to fail if the frequency is a multiple of the frequency rule code (:issue:`14763`)
--
+- Fixed bug when plotting a :class:`DatetimeIndex` with ``datetime.timezone.utc`` timezone (:issue:`17173`)
 -
 -
 

mypy.ini

@@ -148,7 +148,7 @@ def get_dispatch(dtypes):
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cdef inline take_1d_{{name}}_{{dest}}_memview({{c_type_in}}[:] values,
-                                              int64_t[:] indexer,
+                                              const int64_t[:] indexer,
                                               {{c_type_out}}[:] out,
                                               fill_value=np.nan):
 
@@ -159,7 +159,7 @@ cdef inline take_1d_{{name}}_{{dest}}_memview({{c_type_in}}[:] values,
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def take_1d_{{name}}_{{dest}}(ndarray[{{c_type_in}}, ndim=1] values,
-                              int64_t[:] indexer,
+                              const int64_t[:] indexer,
                               {{c_type_out}}[:] out,
                               fill_value=np.nan):
 
@@ -178,7 +178,7 @@ def take_1d_{{name}}_{{dest}}(ndarray[{{c_type_in}}, ndim=1] values,
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cdef inline take_2d_axis0_{{name}}_{{dest}}_memview({{c_type_in}}[:, :] values,
-                                                    int64_t[:] indexer,
+                                                    const int64_t[:] indexer,
                                                     {{c_type_out}}[:, :] out,
                                                     fill_value=np.nan):
 {{inner_take_2d_axis0}}
@@ -205,7 +205,7 @@ def take_2d_axis0_{{name}}_{{dest}}(ndarray[{{c_type_in}}, ndim=2] values,
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cdef inline take_2d_axis1_{{name}}_{{dest}}_memview({{c_type_in}}[:, :] values,
-                                                    int64_t[:] indexer,
+                                                    const int64_t[:] indexer,
                                                     {{c_type_out}}[:, :] out,
                                                     fill_value=np.nan):
 {{inner_take_2d_axis1}}