Added publish to dataframe repr

Author: TomAugspurger

ci/requirements-2.7.pip

@@ -7,3 +7,5 @@ pathlib
 backports.lzma
 py
 PyCrypto
+mock
+ipython

ci/requirements-3.5.run

@@ -18,3 +18,4 @@ pymysql
 psycopg2
 s3fs
 beautifulsoup4
+ipython

ci/requirements-3.6.run

@@ -18,3 +18,4 @@ pymysql
 beautifulsoup4
 s3fs
 xarray
+ipython

doc/source/options.rst

@@ -392,6 +392,9 @@ display.width              80           Width of the display in characters.
                                         IPython qtconsole, or IDLE do not run in a
                                         terminal and hence it is not possible
                                         to correctly detect the width.
+display.html.table_schema  True         Whether to publish a Table Schema
+                                        representation for frontends that
+                                        support it.
 html.border                1            A ``border=value`` attribute is
                                         inserted in the ``<table>`` tag
                                         for the DataFrame HTML repr.
@@ -516,7 +519,7 @@ Table Schema Display
 .. versionadded:: 0.20.0
 
 ``DataFrame`` and ``Series`` will publish a Table Schema representation
-by default. This can be disabled globally with the ``display.table_schema``
+by default. This can be disabled globally with the ``display.html.table_schema``
 option:
 
 .. ipython:: python

doc/source/whatsnew/v0.20.0.txt

@@ -142,7 +142,20 @@ the data.
    df
    df.to_json(orient='table')
 
+
+See :ref:`IO: Table Schema for more<io.table_schema>`.
+
+Additionally, the repr for ``DataFrame`` and ``Series`` can now publish
+this JSON Table schema representation of the Series or DataFrame if you are
+using IPython (or another frontend like `nteract`_ using the Jupyter messaging
+protocol).
+This gives frontends like the Jupyter notebook and `nteract`_
+more flexiblity in how they display pandas objects, since they have
+more information about the data.
+You must enable this by setting the ``display.html.table_schema`` option to True.
+
 .. _Table Schema: http://specs.frictionlessdata.io/json-table-schema/
+.. _nteract: http://nteract.io/
 
 .. _whatsnew_0200.enhancements.other:
 

pandas/core/config_init.py

@@ -164,6 +164,13 @@
     (default: False)
 """
 
+pc_table_schema_doc = """
+: boolean
+    Whether to publish a Table Schema representation for frontends
+    that support it.
+    (default: False)
+"""
+
 pc_line_width_deprecation_warning = """\
 line_width has been deprecated, use display.width instead (currently both are
 identical)
@@ -339,6 +346,8 @@ def mpl_style_cb(key):
                        validator=is_bool)
     cf.register_option('latex.longtable', False, pc_latex_longtable,
                        validator=is_bool)
+    cf.register_option('html.table_schema', False, pc_table_schema_doc,
+                       validator=is_bool)
 
 cf.deprecate_option('display.line_width',
                     msg=pc_line_width_deprecation_warning,

pandas/core/generic.py

@@ -4,6 +4,7 @@
 import operator
 import weakref
 import gc
+import json
 
 import numpy as np
 import pandas.lib as lib
@@ -129,6 +130,37 @@ def __init__(self, data, axes=None, copy=False, dtype=None,
         object.__setattr__(self, '_data', data)
         object.__setattr__(self, '_item_cache', {})
 
+    def _ipython_display_(self):
+        try:
+            from IPython.display import display
+        except ImportError:
+            return None
+
+        # Series doesn't define _repr_html_ or _repr_latex_
+        latex = self._repr_latex_() if hasattr(self, '_repr_latex_') else None
+        html = self._repr_html_() if hasattr(self, '_repr_html_') else None
+        table_schema = self._repr_table_schema_()
+        # We need the inital newline since we aren't going through the
+        # usual __repr__. See
+        # https://github.com/pandas-dev/pandas/pull/14904#issuecomment-277829277
+        text = "\n" + repr(self)
+
+        reprs = {"text/plain": text, "text/html": html, "text/latex": latex,
+                 "application/vnd.dataresource+json": table_schema}
+        reprs = {k: v for k, v in reprs.items() if v}
+        display(reprs, raw=True)
+
+    def _repr_table_schema_(self):
+        """
+        Not a real Jupyter special repr method, but we use the same
+        naming convention.
+        """
+        if config.get_option("display.html.table_schema"):
+            data = self.head(config.get_option('display.max_rows'))
+            payload = json.loads(data.to_json(orient='table'),
+                                 object_pairs_hook=collections.OrderedDict)
+            return payload
+
     def _validate_dtype(self, dtype):
         """ validate the passed dtype """
 
@@ -1088,10 +1120,9 @@ def __setstate__(self, state):
     strings before writing.
     """
 
-    def to_json(self, path_or_buf=None, orient=None, date_format='epoch',
-                timedelta_format='epoch', double_precision=10,
-                force_ascii=True, date_unit='ms', default_handler=None,
-                lines=False):
+    def to_json(self, path_or_buf=None, orient=None, date_format=None,
+                double_precision=10, force_ascii=True, date_unit='ms',
+                default_handler=None, lines=False):
         """
         Convert the object to a JSON string.
 
@@ -1124,19 +1155,19 @@ def to_json(self, path_or_buf=None, orient=None, date_format='epoch',
               - index : dict like {index -> {column -> value}}
               - columns : dict like {column -> {index -> value}}
               - values : just the values array
-            - ``'table'`` dict like
+              - table : dict like {'schema': {schema}, 'data': {data}}
               ``{'schema': [schema], 'data': [data]}``
-              where the schema component is a Table Schema
-              describing the data, and the data component is
-              like ``orient='records'``.
+                the schema component is a `Table Schema_`
+                describing the data, and the data component is
+                like ``orient='records'``.
 
                 .. versionchanged:: 0.20.0
 
-        date_format : {'epoch', 'iso'}
+        date_format : {None, 'epoch', 'iso'}
             Type of date conversion. `epoch` = epoch milliseconds,
-            `iso` = ISO8601. Default is epoch, except when orient is
-            table_schema, in which case this parameter is ignored
-            and iso formatting is always used.
+            `iso` = ISO8601. The default depends on the `orient`. For
+            `orient='table'`, the default is `'iso'`. For all other orients,
+            the default is `'epoch'`.
         double_precision : The number of decimal places to use when encoding
             floating point values, default 10.
         force_ascii : force encoded string to be ASCII, default True.
@@ -1155,6 +1186,7 @@ def to_json(self, path_or_buf=None, orient=None, date_format='epoch',
 
             .. versionadded:: 0.19.0
 
+        .. _Table Schema: http://specs.frictionlessdata.io/json-table-schema/
 
         Returns
         -------
@@ -1199,6 +1231,10 @@ def to_json(self, path_or_buf=None, orient=None, date_format='epoch',
         """
 
         from pandas.io import json
+        if date_format is None and orient == 'table':
+            date_format = 'iso'
+        elif date_format is None:
+            date_format = 'epoch'
         return json.to_json(path_or_buf=path_or_buf, obj=self, orient=orient,
                             date_format=date_format,
                             double_precision=double_precision,

pandas/tests/formats/test_printing.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from pandas import compat
+import pandas as pd
 import pandas.formats.printing as printing
 import pandas.formats.format as fmt
 import pandas.util.testing as tm
@@ -118,6 +119,63 @@ def test_ambiguous_width(self):
         self.assertEqual(adjoined, expected)
 
 
+class TestTableSchemaRepr(tm.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        tm._skip_if_no_mock()
+        tm._skip_if_no_ipython()
+        try:
+            import mock
+        except ImportError:
+            from unittest import mock
+        cls.mock = mock
+
+    def test_publishes(self):
+        df = pd.DataFrame({"A": [1, 2]})
+        objects = [df['A'], df, df]  # dataframe / series
+        expected_keys = [
+            {'text/plain', 'application/vnd.dataresource+json'},
+            {'text/plain', 'text/html', 'application/vnd.dataresource+json'},
+        ]
+
+        make_patch = self.mock.patch('IPython.display.display')
+        opt = pd.option_context('display.html.table_schema', True)
+        for obj, expected in zip(objects, expected_keys):
+            with opt, make_patch as mock_display:
+                handle = obj._ipython_display_()
+                self.assertEqual(mock_display.call_count, 1)
+                self.assertIsNone(handle)
+                args, kwargs = mock_display.call_args
+                arg, = args  # just one argument
+
+            self.assertEqual(kwargs, {"raw": True})
+            self.assertEqual(set(arg.keys()), expected)
+
+        with_latex = pd.option_context('display.latex.repr', True)
+
+        with opt, with_latex, make_patch as mock_display:
+            handle = obj._ipython_display_()
+            args, kwargs = mock_display.call_args
+            arg, = args
+
+        expected = {'text/plain', 'text/html', 'text/latex',
+                    'application/vnd.dataresource+json'}
+        self.assertEqual(set(arg.keys()), expected)
+
+    def test_config_on(self):
+        df = pd.DataFrame({"A": [1, 2]})
+        with pd.option_context("display.html.table_schema", True):
+            result = df._repr_table_schema_()
+        self.assertIsNotNone(result)
+
+    def test_config_default_off(self):
+        df = pd.DataFrame({"A": [1, 2]})
+        with pd.option_context("display.html.table_schema", False):
+            result = df._repr_table_schema_()
+        self.assertIsNone(result)
+
+
 # TODO: fix this broken test
 
 # def test_console_encode():

pandas/util/testing.py

@@ -415,6 +415,25 @@ def _skip_if_not_us_locale():
         import pytest
         pytest.skip("Specific locale is set {0}".format(lang))
 
+
+def _skip_if_no_mock():
+    try:
+        import mock  # noqa
+    except ImportError:
+        try:
+            from unittest import mock  # noqa
+        except ImportError:
+            import nose
+            raise nose.SkipTest("mock is not installed")
+
+
+def _skip_if_no_ipython():
+    try:
+        import IPython  # noqa
+    except ImportError:
+        import nose
+        raise nose.SkipTest("IPython not installed")
+
 # -----------------------------------------------------------------------------
 # locale utilities