From a074c6e1d11ceeaa0e1666f1c82ef265cdce7161 Mon Sep 17 00:00:00 2001 From: Jan Werkmann Date: Mon, 20 Nov 2017 23:00:07 +0100 Subject: [PATCH 1/5] Fixed numpy bool serialization to msgpack This allows for running to_msgpack with dataframes containing field of numpy.bool_ datatype --- pandas/io/msgpack/_packer.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/io/msgpack/_packer.pyx b/pandas/io/msgpack/_packer.pyx index f6383b42d4975..93831814b4505 100644 --- a/pandas/io/msgpack/_packer.pyx +++ b/pandas/io/msgpack/_packer.pyx @@ -8,6 +8,7 @@ from libc.limits cimport * from pandas.io.msgpack.exceptions import PackValueError from pandas.io.msgpack import ExtType +from numpy import bool_ cdef extern from "../../src/msgpack/pack.h": @@ -133,7 +134,7 @@ cdef class Packer(object): while True: if o is None: ret = msgpack_pack_nil(&self.pk) - elif isinstance(o, bool): + elif isinstance(o, (bool, bool_)): if o: ret = msgpack_pack_true(&self.pk) else: From 7b0c2a4a945a0995010545442f3af70092975b36 Mon Sep 17 00:00:00 2001 From: Jan Werkmann Date: Mon, 20 Nov 2017 23:11:05 +0100 Subject: [PATCH 2/5] Added to test --- pandas/tests/io/msgpack/test_pack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/io/msgpack/test_pack.py b/pandas/tests/io/msgpack/test_pack.py index c0b3e1b24674f..7784871fa45dc 100644 --- a/pandas/tests/io/msgpack/test_pack.py +++ b/pandas/tests/io/msgpack/test_pack.py @@ -7,6 +7,7 @@ from pandas import compat from pandas.compat import u, OrderedDict from pandas.io.msgpack import packb, unpackb, Unpacker, Packer +from numpy import bool_ class TestPack(object): @@ -25,6 +26,7 @@ def testPack(self): (), ((),), ((), None,), {None: 0}, (1 << 23), + bool_(1), bool_(0), ] for td in test_data: self.check(td) From 487484824a47f30645282ccb92727c44c74d622a Mon Sep 17 00:00:00 2001 From: Jan Werkmann Date: Mon, 20 Nov 2017 23:23:13 +0100 Subject: [PATCH 3/5] Added whatsnew entry --- doc/source/whatsnew/v0.21.1.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.21.1.txt b/doc/source/whatsnew/v0.21.1.txt index 5829481cdb731..ab7ffecaebc4f 100644 --- a/doc/source/whatsnew/v0.21.1.txt +++ b/doc/source/whatsnew/v0.21.1.txt @@ -87,6 +87,7 @@ I/O - :func:`read_parquet` now allows to specify the columns to read from a parquet file (:issue:`18154`) - :func:`read_parquet` now allows to specify kwargs which are passed to the respective engine (:issue:`18216`) - Bug in parsing integer datetime-like columns with specified format in ``read_sql`` (:issue:`17855`). +- Bug in :meth:`DataFrame.to_msgpack` when serializing data of the numpy.bool_ datatype (:issue:`18390`) Plotting From 3e19bce6bea161e04a137ed8c682ff880b4feaff Mon Sep 17 00:00:00 2001 From: Jan Werkmann Date: Tue, 21 Nov 2017 18:57:03 +0100 Subject: [PATCH 4/5] Added round-trip Tests, changend bool_ to np.bool_ Added test for scaler numpy bool, added numpy bool to mixed list test, removed modifications to pack test and changend bool_ to np.bool_ --- pandas/io/msgpack/_packer.pyx | 4 ++-- pandas/tests/io/msgpack/test_pack.py | 2 -- pandas/tests/io/test_packers.py | 11 ++++++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pandas/io/msgpack/_packer.pyx b/pandas/io/msgpack/_packer.pyx index 93831814b4505..c81069c8e04c0 100644 --- a/pandas/io/msgpack/_packer.pyx +++ b/pandas/io/msgpack/_packer.pyx @@ -8,7 +8,7 @@ from libc.limits cimport * from pandas.io.msgpack.exceptions import PackValueError from pandas.io.msgpack import ExtType -from numpy import bool_ +import numpy as np cdef extern from "../../src/msgpack/pack.h": @@ -134,7 +134,7 @@ cdef class Packer(object): while True: if o is None: ret = msgpack_pack_nil(&self.pk) - elif isinstance(o, (bool, bool_)): + elif isinstance(o, (bool, np.bool_)): if o: ret = msgpack_pack_true(&self.pk) else: diff --git a/pandas/tests/io/msgpack/test_pack.py b/pandas/tests/io/msgpack/test_pack.py index 7784871fa45dc..c0b3e1b24674f 100644 --- a/pandas/tests/io/msgpack/test_pack.py +++ b/pandas/tests/io/msgpack/test_pack.py @@ -7,7 +7,6 @@ from pandas import compat from pandas.compat import u, OrderedDict from pandas.io.msgpack import packb, unpackb, Unpacker, Packer -from numpy import bool_ class TestPack(object): @@ -26,7 +25,6 @@ def testPack(self): (), ((),), ((), None,), {None: 0}, (1 << 23), - bool_(1), bool_(0), ] for td in test_data: self.check(td) diff --git a/pandas/tests/io/test_packers.py b/pandas/tests/io/test_packers.py index a28adcf1ee771..1cbe9729c93db 100644 --- a/pandas/tests/io/test_packers.py +++ b/pandas/tests/io/test_packers.py @@ -180,6 +180,15 @@ def test_scalar_float(self): x_rec = self.encode_decode(x) tm.assert_almost_equal(x, x_rec) + def test_scalar_bool(self): + x = np.bool_(1) + x_rec = self.encode_decode(x) + tm.assert_almost_equal(x, x_rec) + + x = np.bool_(0) + x_rec = self.encode_decode(x) + tm.assert_almost_equal(x, x_rec) + def test_scalar_complex(self): x = np.random.rand() + 1j * np.random.rand() x_rec = self.encode_decode(x) @@ -263,7 +272,7 @@ def test_numpy_array_complex(self): x.dtype == x_rec.dtype) def test_list_mixed(self): - x = [1.0, np.float32(3.5), np.complex128(4.25), u('foo')] + x = [1.0, np.float32(3.5), np.complex128(4.25), u('foo'), np.bool_(1)] x_rec = self.encode_decode(x) # current msgpack cannot distinguish list/tuple tm.assert_almost_equal(tuple(x), x_rec) From 378a218067493e55fbe8cfece48720d77c651371 Mon Sep 17 00:00:00 2001 From: Jan Werkmann Date: Wed, 22 Nov 2017 10:18:05 +0100 Subject: [PATCH 5/5] Added test for mixed Series with numpy bool --- pandas/tests/io/test_packers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/io/test_packers.py b/pandas/tests/io/test_packers.py index 1cbe9729c93db..bc58ea1c7c228 100644 --- a/pandas/tests/io/test_packers.py +++ b/pandas/tests/io/test_packers.py @@ -410,6 +410,7 @@ def setup_method(self, method): 'G': [Timestamp('20130102', tz='US/Eastern')] * 5, 'H': Categorical([1, 2, 3, 4, 5]), 'I': Categorical([1, 2, 3, 4, 5], ordered=True), + 'J': (np.bool_(1), 2, 3, 4, 5), } self.d['float'] = Series(data['A']) @@ -419,6 +420,7 @@ def setup_method(self, method): self.d['dt_tz'] = Series(data['G']) self.d['cat_ordered'] = Series(data['H']) self.d['cat_unordered'] = Series(data['I']) + self.d['numpy_bool_mixed'] = Series(data['J']) def test_basic(self):