BUG: (GH4134) Fix arithmetic with series/datetimeindex and np.timedelta64 not working the same

TST: add in @cpcloud tests related to GH4135

Author: jreback

doc/source/release.rst

@@ -168,6 +168,8 @@ pandas 0.13
   - Fixed (:issue:`3334`) in pivot_table. Margins did not compute if values is the index.
   - Fix bug in having a rhs of ``np.timedelta64`` or ``np.offsets.DateOffset`` when operating
     with datetimes (:issue:`4532`)
+  - Fix arithmetic with series/datetimeindex and ``np.timedelta64`` not working the same (:issue:`4134`)
+    and buggy timedelta in numpy 1.6 (:issue:`4135`)
 
 pandas 0.12
 ===========

pandas/core/common.py

@@ -11,9 +11,11 @@
 import pandas.algos as algos
 import pandas.lib as lib
 import pandas.tslib as tslib
-
+from distutils.version import LooseVersion
 from pandas import compat
 from pandas.compat import StringIO, BytesIO, range, long, u, zip, map
+from datetime import timedelta
+
 from pandas.core.config import get_option
 from pandas.core import array as pa
 
@@ -33,6 +35,10 @@ class PandasError(Exception):
 class AmbiguousIndexError(PandasError, KeyError):
     pass
 
+# versioning
+_np_version = np.version.short_version
+_np_version_under1p6 = LooseVersion(_np_version) < '1.6'
+_np_version_under1p7 = LooseVersion(_np_version) < '1.7'
 
 _POSSIBLY_CAST_DTYPES = set([ np.dtype(t) for t in ['M8[ns]','m8[ns]','O','int8','uint8','int16','uint16','int32','uint32','int64','uint64'] ])
 _NS_DTYPE = np.dtype('M8[ns]')
@@ -1144,7 +1150,45 @@ def _possibly_convert_platform(values):
 def _possibly_cast_to_timedelta(value, coerce=True):
     """ try to cast to timedelta64, if already a timedeltalike, then make
         sure that we are [ns] (as numpy 1.6.2 is very buggy in this regards,
-        don't force the conversion unless coerce is True """
+        don't force the conversion unless coerce is True
+
+        if coerce='compat' force a compatibilty coercerion (to timedeltas) if needeed
+        """
+
+    # coercion compatability
+    if coerce == 'compat' and _np_version_under1p7:
+
+        def convert(td, type):
+
+            # we have an array with a non-object dtype
+            if hasattr(td,'item'):
+                td = td.astype(np.int64).item()
+                if td == tslib.iNaT:
+                    return td
+                if dtype == 'm8[us]':
+                    td *= 1000
+                return td
+
+            if td == tslib.compat_NaT:
+                return tslib.iNaT
+
+            # convert td value to a nanosecond value
+            d = td.days
+            s = td.seconds
+            us = td.microseconds
+
+            if dtype == 'object' or dtype == 'm8[ns]':
+                td = 1000*us + (s + d * 24 * 3600) * 10 ** 9
+            else:
+                raise ValueError("invalid conversion of dtype in np < 1.7 [%s]" % dtype)
+
+            return td
+
+        # < 1.7 coercion
+        if not is_list_like(value):
+            value = np.array([ value ])
+        dtype = value.dtype
+        return np.array([ convert(v,dtype) for v in value ], dtype='m8[ns]')
 
     # deal with numpy not being able to handle certain timedelta operations
     if isinstance(value,np.ndarray) and value.dtype.kind == 'm':
@@ -1154,6 +1198,7 @@ def _possibly_cast_to_timedelta(value, coerce=True):
 
     # we don't have a timedelta, but we want to try to convert to one (but don't force it)
     if coerce:
+
         new_value = tslib.array_to_timedelta64(value.astype(object), coerce=False)
         if new_value.dtype == 'i8':
             value = np.array(new_value,dtype='timedelta64[ns]')

pandas/core/series.py

@@ -94,6 +94,8 @@ def wrapper(self, other, name=name):
 
         if is_datetime_lhs or is_timedelta_lhs:
 
+            coerce = 'compat' if _np_version_under1p7 else True
+
             # convert the argument to an ndarray
             def convert_to_array(values):
                 if not is_list_like(values):
@@ -105,19 +107,20 @@ def convert_to_array(values):
                         values = tslib.array_to_datetime(values)
                 elif inferred_type in set(['timedelta']):
                     # have a timedelta, convert to to ns here
-                    if not (isinstance(values, pa.Array) and com.is_timedelta64_dtype(values)):
-                        values = com._possibly_cast_to_timedelta(values)
+                    values = com._possibly_cast_to_timedelta(values, coerce=coerce)
                 elif inferred_type in set(['timedelta64']):
                     # have a timedelta64, make sure dtype dtype is ns
-                    values = com._possibly_cast_to_timedelta(values)
+                    values = com._possibly_cast_to_timedelta(values, coerce=coerce)
                 elif inferred_type in set(['integer']):
                     # py3 compat where dtype is 'm' but is an integer
                     if values.dtype.kind == 'm':
                         values = values.astype('timedelta64[ns]')
+                    else:
+                        raise ValueError("incompatible type for a datetime/timedelta operation")
                 elif isinstance(values[0],DateOffset):
                     # handle DateOffsets
                     values = pa.array([ v.delta for v in values ])
-                    values = com._possibly_cast_to_timedelta(values)
+                    values = com._possibly_cast_to_timedelta(values, coerce=coerce)
                 else:
                     values = pa.array(values)
                 return values
@@ -126,8 +129,8 @@ def convert_to_array(values):
             lvalues = convert_to_array(lvalues)
             rvalues = convert_to_array(rvalues)
 
-            is_timedelta_rhs = com.is_timedelta64_dtype(rvalues)
             is_datetime_rhs  = com.is_datetime64_dtype(rvalues)
+            is_timedelta_rhs = com.is_timedelta64_dtype(rvalues) or (not is_datetime_rhs and _np_version_under1p7)
 
             # 2 datetimes or 2 timedeltas
             if (is_timedelta_lhs and is_timedelta_rhs) or (is_datetime_lhs and
@@ -141,7 +144,6 @@ def convert_to_array(values):
 
                 dtype = 'timedelta64[ns]'
 
-                # we may have to convert to object unfortunately here
                 mask = isnull(lvalues) | isnull(rvalues)
                 if mask.any():
                     def wrap_results(x):
@@ -150,13 +152,20 @@ def wrap_results(x):
                         return x
 
             # datetime and timedelta
-            elif (is_timedelta_lhs and is_datetime_rhs) or (is_timedelta_rhs and is_datetime_lhs):
+            elif is_timedelta_rhs and is_datetime_lhs:
 
                 if name not in ['__add__','__sub__']:
-                    raise TypeError("can only operate on a timedelta and a datetime for "
+                    raise TypeError("can only operate on a datetime with a rhs of a timedelta for "
                                     "addition and subtraction, but the operator [%s] was passed" % name)
                 dtype = 'M8[ns]'
 
+            elif is_timedelta_lhs and is_datetime_rhs:
+
+                if name not in ['__add__']:
+                    raise TypeError("can only operate on a timedelta and a datetime for "
+                                    "addition, but the operator [%s] was passed" % name)
+                dtype = 'M8[ns]'
+
             else:
                 raise ValueError('cannot operate on a series with out a rhs '
                                  'of a series/ndarray of type datetime64[ns] '
@@ -166,8 +175,11 @@ def wrap_results(x):
             rvalues = rvalues.view('i8')
 
         if isinstance(rvalues, Series):
-            lvalues = lvalues.values
-            rvalues = rvalues.values
+
+            if hasattr(lvalues,'values'):
+                lvalues = lvalues.values
+            if hasattr(rvalues,'values'):
+                rvalues = rvalues.values
 
             if self.index.equals(other.index):
                 name = _maybe_match_name(self, other)