Generalize NA Compat

Author: jschendel

doc/source/whatsnew/v0.22.0.txt

@@ -77,6 +77,7 @@ Other API Changes
 - :func:`Series.truncate` and :func:`DataFrame.truncate` will raise a ``ValueError`` if the index is not sorted instead of an unhelpful ``KeyError`` (:issue:`17935`)
 - :func:`Dataframe.unstack` will now default to filling with ``np.nan`` for ``object`` columns. (:issue:`12815`)
 - :class:`IntervalIndex` constructor will raise if the ``closed`` parameter conflicts with how the input data is inferred to be closed (:issue:`18421`)
+- Inserting missing values into indexes will work for all types of indexes and automatically insert the correct type of missing value (``NaN``, ``NaT``, etc.) regardless of the type passed in (:issue:`18295`)
 
 
 .. _whatsnew_0220.deprecations:

pandas/core/indexes/base.py

@@ -3734,6 +3734,10 @@ def insert(self, loc, item):
         -------
         new_index : Index
         """
+        if lib.checknull(item):
+            # GH 18295
+            item = self._na_value
+
         _self = np.asarray(self)
         item = self._coerce_scalar_to_index(item)._values
         idx = np.concatenate((_self[:loc], item, _self[loc:]))

pandas/core/indexes/category.py

@@ -1,5 +1,5 @@
 import numpy as np
-from pandas._libs import index as libindex
+from pandas._libs import index as libindex, lib
 
 from pandas import compat
 from pandas.compat.numpy import function as nv
@@ -690,7 +690,7 @@ def insert(self, loc, item):
 
         """
         code = self.categories.get_indexer([item])
-        if (code == -1):
+        if (code == -1) and not lib.checknull(item):
             raise TypeError("cannot insert an item into a CategoricalIndex "
                             "that is not already an existing category")
 

pandas/core/indexes/datetimes.py

@@ -1757,6 +1757,9 @@ def insert(self, loc, item):
         -------
         new_index : Index
         """
+        if lib.checknull(item):
+            # GH 18295
+            item = self._na_value
 
         freq = None
 
@@ -1773,14 +1776,14 @@ def insert(self, loc, item):
                 elif (loc == len(self)) and item - self.freq == self[-1]:
                     freq = self.freq
             item = _to_m8(item, tz=self.tz)
+
         try:
             new_dates = np.concatenate((self[:loc].asi8, [item.view(np.int64)],
                                         self[loc:].asi8))
             if self.tz is not None:
                 new_dates = conversion.tz_convert(new_dates, 'UTC', self.tz)
             return DatetimeIndex(new_dates, name=self.name, freq=freq,
                                  tz=self.tz)
-
         except (AttributeError, TypeError):
 
             # fall back to object index

pandas/core/indexes/interval.py

@@ -23,7 +23,7 @@
     Index, _ensure_index,
     default_pprint, _index_shared_docs)
 
-from pandas._libs import Timestamp, Timedelta
+from pandas._libs import lib, Timestamp, Timedelta
 from pandas._libs.interval import (
     Interval, IntervalMixin, IntervalTree,
     intervals_to_interval_bounds)
@@ -1007,12 +1007,8 @@ def insert(self, loc, item):
                                  'side as the index')
             left_insert = item.left
             right_insert = item.right
-        elif is_scalar(item) and isna(item):
+        elif lib.checknull(item):
             # GH 18295
-            if item is not self.left._na_value:
-                raise TypeError('cannot insert with incompatible NA value: '
-                                'got {item}, expected {na}'
-                                .format(item=item, na=self.left._na_value))
             left_insert = right_insert = item
         else:
             raise ValueError('can only insert Interval objects and NA into '

pandas/core/indexes/timedeltas.py

@@ -852,16 +852,18 @@ def insert(self, loc, item):
         -------
         new_index : Index
         """
-
         # try to convert if possible
         if _is_convertible_to_td(item):
             try:
                 item = Timedelta(item)
             except Exception:
                 pass
+        elif lib.checknull(item):
+            # GH 18295
+            item = self._na_value
 
         freq = None
-        if isinstance(item, Timedelta) or item is NaT:
+        if isinstance(item, Timedelta) or (item is self._na_value):
 
             # check freq can be preserved on edge cases
             if self.freq is not None:

pandas/tests/indexes/datetimes/test_indexing.py

@@ -145,6 +145,13 @@ def test_insert(self):
                 assert result.tz == expected.tz
                 assert result.freq is None
 
+        # GH 18295 (test missing)
+        expected = DatetimeIndex(
+            ['20170101', pd.NaT, '20170102', '20170103', '20170104'])
+        for na in (np.nan, pd.NaT, None):
+            result = date_range('20170101', periods=4).insert(1, na)
+            tm.assert_index_equal(result, expected)
+
     def test_delete(self):
         idx = date_range(start='2000-01-01', periods=5, freq='M', name='idx')
 

pandas/tests/indexes/period/test_period.py

@@ -697,3 +697,11 @@ def test_join_self(self, how):
         index = period_range('1/1/2000', periods=10)
         joined = index.join(index, how=how)
         assert index is joined
+
+    def test_insert(self):
+        # GH 18295 (test missing)
+        expected = PeriodIndex(
+            ['2017Q1', pd.NaT, '2017Q2', '2017Q3', '2017Q4'], freq='Q')
+        for na in (np.nan, pd.NaT, None):
+            result = period_range('2017Q1', periods=4, freq='Q').insert(1, na)
+            tm.assert_index_equal(result, expected)

pandas/tests/indexes/test_base.py

@@ -457,6 +457,12 @@ def test_insert(self):
         null_index = Index([])
         tm.assert_index_equal(Index(['a']), null_index.insert(0, 'a'))
 
+        # GH 18295 (test missing)
+        expected = Index(['a', np.nan, 'b', 'c'])
+        for na in (np.nan, pd.NaT, None):
+            result = Index(list('abc')).insert(1, na)
+            tm.assert_index_equal(result, expected)
+
     def test_delete(self):
         idx = Index(['a', 'b', 'c', 'd'], name='idx')
 

pandas/tests/indexes/test_category.py

@@ -344,6 +344,12 @@ def test_insert(self):
         # invalid
         pytest.raises(TypeError, lambda: ci.insert(0, 'd'))
 
+        # GH 18295 (test missing)
+        expected = CategoricalIndex(['a', np.nan, 'a', 'b', 'c', 'b'])
+        for na in (np.nan, pd.NaT, None):
+            result = CategoricalIndex(list('aabcb')).insert(1, na)
+            tm.assert_index_equal(result, expected)
+
     def test_delete(self):
 
         ci = self.create_index()