BUG: fixed string appending when length of subsequent is longer/shorter that existing

     removed meta data saving
     disable memory tests (and put a try:except: around it)

Author: jreback

pandas/io/pytables.py

@@ -21,7 +21,7 @@
 from pandas.tseries.api import PeriodIndex, DatetimeIndex
 from pandas.core.common import adjoin
 from pandas.core.algorithms import match, unique
-
+from pandas.core.strings import str_len
 from pandas.core.categorical import Factor
 from pandas.core.common import _asarray_tuplesafe, _try_sort
 from pandas.core.internals import BlockManager, make_block, form_blocks
@@ -507,7 +507,7 @@ def _write_to_group(self, key, value, table=False, append=False,
         wrapper(value)
         group._v_attrs.pandas_type = kind
         group._v_attrs.pandas_version = _version
-        group._v_attrs.meta = getattr(value,'meta',None)
+        #group._v_attrs.meta = getattr(value,'meta',None)
 
     def _write_series(self, group, series):
         self._write_index(group, 'index', series.index)
@@ -848,10 +848,10 @@ def _read_group(self, group, where=None, **kwargs):
         kind = _LEGACY_MAP.get(kind, kind)
         handler = self._get_handler(op='read', kind=kind)
         v = handler(group, where, **kwargs)
-        if v is not None:
-            meta = getattr(group._v_attrs,'meta',None)
-            if meta is not None:
-                v.meta = meta
+        #if v is not None:
+        #    meta = getattr(group._v_attrs,'meta',None)
+        #    if meta is not None:
+        #        v.meta = meta
         return v
 
     def _read_series(self, group, where=None):
@@ -1001,16 +1001,22 @@ def validate_and_set(self, table, append, **kwargs):
         self.validate_attr(append)
         self.set_attr()
 
-    def validate_col(self):
-        """ validate this column & set table data for it """
+    def validate_col(self, itemsize = None):
+        """ validate this column: return the compared against itemsize """
 
         # validate this column for string truncation (or reset to the max size)
-        if self.kind == 'string':
+        dtype = getattr(self,'dtype',None)
+        if self.kind == 'string' or (dtype is not None and dtype.startswith('string')):
 
             c = self.col
             if c is not None:
-                if c.itemsize < self.itemsize:
-                    raise Exception("[%s] column has a min_itemsize of [%s] but itemsize [%s] is required!" % (self.cname,self.itemsize,c.itemsize))
+                if itemsize is None:
+                    itemsize = self.itemsize
+                if c.itemsize < itemsize:
+                    raise Exception("[%s] column has a min_itemsize of [%s] but itemsize [%s] is required!" % (self.cname,itemsize,c.itemsize))
+                return c.itemsize
+
+        return None
 
 
     def validate_attr(self, append):
@@ -1404,26 +1410,35 @@ def create_axes(self, axes, obj, validate = True, min_itemsize = None):
             # a string column
             if b.dtype.name == 'object':
 
+                # itemsize is the maximum length of a string (along any dimension)
+                itemsize = _itemsize_string_array(values)
+
                 # specified min_itemsize?
                 if isinstance(min_itemsize, dict):
-                    min_itemsize = int(min_itemsize.get('values'))
+                    itemsize = max(int(min_itemsize.get('values')),itemsize)
+
+                # check for column in the values conflicts
+                if existing_table is not None and validate:
+                    eci = existing_table.values_axes[i].validate_col(itemsize)
+                    if eci > itemsize:
+                        itemsize = eci
 
-                if min_itemsize is None:
-                    min_itemsize = values.dtype.itemsize
+                atom  = _tables().StringCol(itemsize = itemsize, shape = shape)
+                utype = 'S%s' % itemsize
+                kind  = 'string'
 
-                atom  = _tables().StringCol(itemsize = min_itemsize, shape = shape)
-                utype = 'S%s' % min_itemsize
             else:
                 atom  = getattr(_tables(),"%sCol" % b.dtype.name.capitalize())(shape = shape)
                 utype = atom._deftype
+                kind  = b.dtype.name
 
             # coerce data to this type
             try:
                 values = values.astype(utype)
             except (Exception), detail:
                 raise Exception("cannot coerce data type -> [dtype->%s]" % b.dtype.name)
 
-            dc = DataCol.create_for_block(i = i, values = list(b.items), kind = b.dtype.name, typ = atom, data = values, pos = j)
+            dc = DataCol.create_for_block(i = i, values = list(b.items), kind = kind, typ = atom, data = values, pos = j)
             j += 1
             self.values_axes.append(dc)
 
@@ -1663,7 +1678,6 @@ def write_data(self):
         """ fast writing of data: requires specific cython routines each axis shape """
 
         # create the masks & values
-        #import pdb; pdb.set_trace()
         masks  = []
         for a in self.values_axes:
 
@@ -1694,7 +1708,6 @@ def write_data(self):
             if len(rows):
                 self.table.append(rows)
         except (Exception), detail:
-            #import pdb; pdb.set_trace()
             raise Exception("tables cannot write this data -> %s" % str(detail))
 
     def delete(self, where = None):
@@ -1849,6 +1862,10 @@ def create_table(parent, group, typ = None, **kwargs):
     return _TABLE_MAP.get(tt)(parent, group, **kwargs)
 
 
+def _itemsize_string_array(arr):
+    """ return the maximum size of elements in a strnig array """
+    return max([ str_len(arr[v]).max() for v in range(arr.shape[0]) ])
+
 def _convert_index(index):
     if isinstance(index, DatetimeIndex):
         converted = index.asi8
@@ -2247,14 +2264,20 @@ def f(values, freq=None, tz=None):
 
 def create_debug_memory(parent):
     _debug_memory = getattr(parent,'_debug_memory',False)
+    def get_memory(s):
+        pass
+  
     if not _debug_memory:
-        def get_memory(s):
-            pass
+        pass
     else:
-        import psutil, os
-        def get_memory(s):
-            p = psutil.Process(os.getpid())
-            (rss,vms) = p.get_memory_info()
-            mp = p.get_memory_percent()
-            print "[%s] cur_mem->%.2f (MB),per_mem->%.2f" % (s,rss/1000000.0,mp)
+        try:
+            import psutil, os
+            def get_memory(s):
+                p = psutil.Process(os.getpid())
+                (rss,vms) = p.get_memory_info()
+                mp = p.get_memory_percent()
+                print "[%s] cur_mem->%.2f (MB),per_mem->%.2f" % (s,rss/1000000.0,mp)
+        except:
+            pass
+
     return get_memory

pandas/io/tests/test_pytables.py

@@ -98,6 +98,8 @@ def test_versioning(self):
         self.assert_(self.store.root.df1._v_attrs.pandas_version == '0.10')
 
     def test_meta(self):
+        raise nose.SkipTest('no meta')
+
         meta = { 'foo' : [ 'I love pandas ' ] }
         s = tm.makeTimeSeries()
         s.meta = meta
@@ -167,6 +169,29 @@ def test_put(self):
         self.store.put('c', df[:10], table=True, append=False)
         tm.assert_frame_equal(df[:10], self.store['c'])
 
+    def test_put_string_index(self):
+
+        index = Index([ "I am a very long string index: %s" % i for i in range(20) ])
+        s  = Series(np.arange(20), index = index)
+        df = DataFrame({ 'A' : s, 'B' : s })
+
+        self.store['a'] = s
+        tm.assert_series_equal(self.store['a'], s)
+
+        self.store['b'] = df
+        tm.assert_frame_equal(self.store['b'], df)
+
+        # mixed length
+        index = Index(['abcdefghijklmnopqrstuvwxyz1234567890'] + [ "I am a very long string index: %s" % i for i in range(20) ])
+        s  = Series(np.arange(21), index = index)
+        df = DataFrame({ 'A' : s, 'B' : s })
+        self.store['a'] = s
+        tm.assert_series_equal(self.store['a'], s)
+
+        self.store['b'] = df
+        tm.assert_frame_equal(self.store['b'], df)
+
+
     def test_put_compression(self):
         df = tm.makeTimeDataFrame()
 
@@ -325,11 +350,22 @@ def test_append_with_strings(self):
         self.store.append('df_big',df, min_itemsize = { 'values' : 1024 })
         tm.assert_frame_equal(self.store.select('df_big'), df)
 
+        # appending smaller string ok
+        df2 = DataFrame([[124,'asdqy'], [346,'dggnhefbdfb']])
+        self.store.append('df_big',df2)
+        expected = concat([ df, df2 ])
+        tm.assert_frame_equal(self.store.select('df_big'), expected)
+
         # avoid truncation on elements
         df = DataFrame([[123,'asdqwerty'], [345,'dggnhebbsdfbdfb']])
-        self.store.append('df_big2',df, min_itemsize = { 'values' : 300 })
+        self.store.append('df_big2',df, min_itemsize = { 'values' : 10 })
         tm.assert_frame_equal(self.store.select('df_big2'), df)
 
+        # bigger string on next append
+        self.store.append('df_new',df, min_itemsize = { 'values' : 16 })
+        df_new  = DataFrame([[124,'abcdefqhij'], [346, 'abcdefghijklmnopqrtsuvwxyz']])
+        self.assertRaises(Exception, self.store.append, 'df_new',df_new)
+
     def test_create_table_index(self):
         wp = tm.makePanel()
         self.store.append('p5', wp)
@@ -375,7 +411,8 @@ def test_create_table_index(self):
 
 
     def test_big_table(self):
-        
+        raise nose.SkipTest('no big table')
+
         # create and write a big table
         wp = Panel(np.random.randn(20, 1000, 1000), items= [ 'Item%s' % i for i in xrange(20) ],
                    major_axis=date_range('1/1/2000', periods=1000), minor_axis = [ 'E%s' % i for i in xrange(1000) ])