BUG: Slice Arrow buffer before passing it to numpy (#40896)

Merge branch 'master' into issue-40896

Author: ThomasBlauthQC

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
         exclude: ^pandas/_libs/src/(klib|headers)/
         args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.0
+    rev: 3.9.1
     hooks:
     -   id: flake8
         additional_dependencies:
@@ -75,7 +75,7 @@ repos:
     hooks:
     -   id: yesqa
         additional_dependencies:
-            - flake8==3.9.0
+            - flake8==3.9.1
             - flake8-comprehensions==3.1.0
             - flake8-bugbear==21.3.2
             - pandas-dev-flaker==0.2.0

LICENSE

@@ -3,7 +3,7 @@ BSD 3-Clause License
 Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
 All rights reserved.
 
-Copyright (c) 2011-2020, Open source contributors.
+Copyright (c) 2011-2021, Open source contributors.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

asv_bench/benchmarks/groupby.py

@@ -505,6 +505,34 @@ def time_frame_agg(self, dtype, method):
         self.df.groupby("key").agg(method)
 
 
+class CumminMax:
+    param_names = ["dtype", "method"]
+    params = [
+        ["float64", "int64", "Float64", "Int64"],
+        ["cummin", "cummax"],
+    ]
+
+    def setup(self, dtype, method):
+        N = 500_000
+        vals = np.random.randint(-10, 10, (N, 5))
+        null_vals = vals.astype(float, copy=True)
+        null_vals[::2, :] = np.nan
+        null_vals[::3, :] = np.nan
+        df = DataFrame(vals, columns=list("abcde"), dtype=dtype)
+        null_df = DataFrame(null_vals, columns=list("abcde"), dtype=dtype)
+        keys = np.random.randint(0, 100, size=N)
+        df["key"] = keys
+        null_df["key"] = keys
+        self.df = df
+        self.null_df = null_df
+
+    def time_frame_transform(self, dtype, method):
+        self.df.groupby("key").transform(method)
+
+    def time_frame_transform_many_nulls(self, dtype, method):
+        self.null_df.groupby("key").transform(method)
+
+
 class RankWithTies:
     # GH 21237
     param_names = ["dtype", "tie_method"]

doc/source/_static/style/hq_ax1.png

@@ -71,8 +71,8 @@ instead of comparing as False).
 
 Long term, we want to introduce consistent missing data handling for all data
 types. This includes consistent behavior in all operations (indexing, arithmetic
-operations, comparisons, etc.). We want to eventually make the new semantics the
-default.
+operations, comparisons, etc.). There has been discussion of eventually making
+the new semantics the default.
 
 This has been discussed at
 `github #28095 <https://github.com/pandas-dev/pandas/issues/28095>`__ (and

doc/source/_static/style/hq_axNone.png

@@ -362,6 +362,21 @@ pyarrow                   0.15.0             Parquet, ORC, and feather reading /
 pyreadstat                                   SPSS files (.sav) reading
 ========================= ================== =============================================================
 
+.. _install.warn_orc:
+
+.. warning::
+
+    * If you want to use :func:`~pandas.read_orc`, it is highly recommended to install pyarrow using conda.
+      The following is a summary of the environment in which :func:`~pandas.read_orc` can work.
+
+      ========================= ================== =============================================================
+      System                    Conda              PyPI
+      ========================= ================== =============================================================
+      Linux                     Successful         Failed(pyarrow==3.0 Successful)
+      macOS                     Successful         Failed
+      Windows                   Failed             Failed
+      ========================= ================== =============================================================
+
 Access data in the cloud
 ^^^^^^^^^^^^^^^^^^^^^^^^
 

doc/source/_static/style/hq_props.png

@@ -176,7 +176,7 @@ these are by default not taken into account by the :func:`~DataFrame.describe` m
 
 Many pandas operations return a ``DataFrame`` or a ``Series``. The
 :func:`~DataFrame.describe` method is an example of a pandas operation returning a
-pandas ``Series``.
+pandas ``Series`` or a pandas ``DataFrame``.
 
 .. raw:: html
 

doc/source/development/roadmap.rst

@@ -5443,6 +5443,11 @@ Similar to the :ref:`parquet <io.parquet>` format, the `ORC Format <https://orc.
 for data frames. It is designed to make reading data frames efficient. pandas provides *only* a reader for the
 ORC format, :func:`~pandas.read_orc`. This requires the `pyarrow <https://arrow.apache.org/docs/python/>`__ library.
 
+.. warning::
+
+   * It is *highly recommended* to install pyarrow using conda due to some issues occurred by pyarrow.
+   * :func:`~pandas.read_orc` is not supported on Windows yet, you can find valid environments on :ref:`install optional dependencies <install.warn_orc>`.
+
 .. _io.sql:
 
 SQL queries