BUG: removed table check in select if where is provided (convience really)

     allow types in Term that are datetime-like (e.g. can provide a timetuple method)
     added a warning if you try to select/remove with a where criteria on a legacy table (which isn't supported),
       you must convert to new format
     added versioning ability, 'pandas_version', can't detect future format changes (not a required attribute)

Author: jreback

pandas/io/pytables.py

@@ -10,6 +10,7 @@
 import re
 import copy
 import itertools
+import warnings
 
 import numpy as np
 from pandas import (
@@ -34,6 +35,11 @@
 
 from contextlib import contextmanager
 
+# versioning attribute
+_version = '0.10'
+
+class IncompatibilityWarning(Warning): pass
+
 # reading and writing the full object in one go
 _TYPE_MAP = {
     Series: 'series',
@@ -341,8 +347,6 @@ def select(self, key, where=None):
         group = self.get_node(key)
         if group is None:
             raise KeyError('No object named %s in the file' % key)
-        if where is not None and not _is_table_type(group):
-            raise Exception('can only select with where on objects written as tables')
         return self._read_group(group, where)
 
     def put(self, key, value, table=False, append=False,
@@ -498,6 +502,7 @@ def _write_to_group(self, key, value, table=False, append=False,
 
         wrapper(value)
         group._v_attrs.pandas_type = kind
+        group._v_attrs.pandas_version = _version
 
     def _write_series(self, group, series):
         self._write_index(group, 'index', series.index)
@@ -1123,6 +1128,7 @@ class Table(object):
     def __init__(self, parent, group):
         self.parent      = parent
         self.group       = group
+        self.version     = getattr(group._v_attrs,'version',None)
         self.index_axes     = []
         self.non_index_axes = []
         self.values_axes    = []
@@ -1475,6 +1481,11 @@ def write(self, **kwargs):
     def read(self, where=None):
         """ we have n indexable columns, with an arbitrary number of data axes """
 
+        # are we trying to operate on an old version?
+        if where is not None:
+            if self.version is None or self.version < 0.1:
+                warnings.warn("where criteria is being ignored as we this version is too old (or not-defined) [%s]" % self.version, IncompatibilityWarning)
+
         if not self.read_axes(where): return None
 
         indicies = [ i.values for i in self.index_axes ]
@@ -2114,7 +2125,7 @@ def convert_value(self, v):
         if self.field == 'index' or self.field == 'major_axis':
             if self.kind == 'datetime64' :
                 return [lib.Timestamp(v).value, None]
-            elif isinstance(v, datetime):
+            elif isinstance(v, datetime) or hasattr(v,'timetuple'):
                 return [time.mktime(v.timetuple()), None]
         elif not isinstance(v, basestring):
             return [str(v), None]

pandas/io/tests/test_pytables.py

@@ -2,13 +2,14 @@
 import unittest
 import os
 import sys
+import warnings
 
 from datetime import datetime
 import numpy as np
 
 from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
                     date_range, Index)
-from pandas.io.pytables import HDFStore, get_store, Term
+from pandas.io.pytables import HDFStore, get_store, Term, IncompatibilityWarning
 import pandas.util.testing as tm
 from pandas.tests.test_series import assert_series_equal
 from pandas.tests.test_frame import assert_frame_equal
@@ -85,6 +86,17 @@ def test_contains(self):
         self.assert_('/foo/b' not in self.store)
         self.assert_('bar' not in self.store)
 
+    def test_versioning(self):
+        self.store['a'] = tm.makeTimeSeries()
+        self.store['b'] = tm.makeDataFrame()
+        df = tm.makeTimeDataFrame()
+        self.store.remove('df1')
+        self.store.append('df1', df[:10])
+        self.store.append('df1', df[10:])
+        self.assert_(self.store.root.a._v_attrs.pandas_version == '0.10')
+        self.assert_(self.store.root.b._v_attrs.pandas_version == '0.10')
+        self.assert_(self.store.root.df1._v_attrs.pandas_version == '0.10')
+
     def test_reopen_handle(self):
         self.store['a'] = tm.makeTimeSeries()
         self.store.open('w', warn=False)
@@ -176,8 +188,6 @@ def test_append(self):
         tm.assert_frame_equal(self.store['df3'], df)
 
         # this is allowed by almost always don't want to do it
-        import warnings
-        import tables
         warnings.filterwarnings('ignore', category=tables.NaturalNameWarning)
         self.store.remove('/df3 foo')
         self.store.append('/df3 foo', df[:10])
@@ -451,9 +461,9 @@ def test_remove_where(self):
                           'wp', ['foo'])
 
         # selectin non-table with a where
-        self.store.put('wp2', wp, table=False)
-        self.assertRaises(Exception, self.store.remove,
-                          'wp2', [('column', ['A', 'D'])])
+        #self.store.put('wp2', wp, table=False)
+        #self.assertRaises(Exception, self.store.remove,
+        #                  'wp2', [('column', ['A', 'D'])])
 
 
     def test_remove_crit(self):
@@ -890,8 +900,8 @@ def test_select(self):
         self.store.select('wp2')
 
         # selectin non-table with a where
-        self.assertRaises(Exception, self.store.select,
-                          'wp2', ('column', ['A', 'D']))
+        #self.assertRaises(Exception, self.store.select,
+        #                  'wp2', ('column', ['A', 'D']))
 
     def test_panel_select(self):
         wp = tm.makePanel()
@@ -927,9 +937,9 @@ def test_frame_select(self):
         tm.assert_frame_equal(result, expected)
 
         # can't select if not written as table
-        self.store['frame'] = df
-        self.assertRaises(Exception, self.store.select,
-                          'frame', [crit1, crit2])
+        #self.store['frame'] = df
+        #self.assertRaises(Exception, self.store.select,
+        #                  'frame', [crit1, crit2])
 
     def test_select_filter_corner(self):
         df = DataFrame(np.random.randn(50, 100))
@@ -1004,6 +1014,13 @@ def test_legacy_table_read(self):
         store.select('df1')
         store.select('df2')
         store.select('wp1')
+
+        # old version (this still throws an exception though)
+        import warnings
+        warnings.filterwarnings('ignore', category=IncompatibilityWarning)
+        self.assertRaises(Exception, store.select, 'wp1', Term('minor_axis','=','B'))
+        warnings.filterwarnings('always', category=IncompatibilityWarning)
+
         store.close()
 
     def test_legacy_table_write(self):