BUG: support duplicate columns in DataFrame.from_records. close #2179

Author: wesm

RELEASE.rst

@@ -46,6 +46,7 @@ pandas 0.10.0
   - Unstack/reshape algorithm rewrite to avoid high memory use in cases where
     the number of observed key-tuples is much smaller than the total possible
     number that could occur (#2278). Also improves performance in most cases.
+  - Support duplicate columns in DataFrame.from_records (#2179)
 
 **Bug fixes**
 

pandas/core/frame.py

@@ -896,11 +896,7 @@ def from_records(cls, data, index=None, exclude=None, columns=None,
         """
         # Make a copy of the input columns so we can modify it
         if columns is not None:
-            columns = list(columns)
-
-            if len(algos.unique(columns)) < len(columns):
-                raise ValueError('Non-unique columns not yet supported in '
-                                 'from_records')
+            columns = _ensure_index(columns)
 
         if com.is_iterator(data):
             if nrows == 0:
@@ -932,48 +928,66 @@ def from_records(cls, data, index=None, exclude=None, columns=None,
             else:
                 data = values
 
-        if isinstance(data, (np.ndarray, DataFrame, dict)):
-            keys, sdict = _rec_to_dict(data)
+        if isinstance(data, dict):
             if columns is None:
-                columns = keys
+                columns = arr_columns = _ensure_index(sorted(data))
+                arrays = [data[k] for k in columns]
             else:
-                sdict = dict((k, v) for k, v in sdict.iteritems()
-                             if k in columns)
+                arrays = []
+                arr_columns = []
+                for k, v in data.iteritems():
+                    if k in columns:
+                        arr_columns.append(k)
+                        arrays.append(v)
+
+        elif isinstance(data, (np.ndarray, DataFrame)):
+            arrays, columns = _to_arrays(data, columns)
+            if columns is not None:
+                columns = _ensure_index(columns)
+            arr_columns = columns
         else:
-            arrays, columns = _to_arrays(data, columns,
-                                         coerce_float=coerce_float)
-            columns=list(columns) # _to_arrays returns index, but we might mutate
-            sdict = dict(zip(columns, arrays))
+            arrays, arr_columns = _to_arrays(data, columns,
+                                             coerce_float=coerce_float)
+
+            arr_columns = _ensure_index(arr_columns)
+            if columns is not None:
+                columns = _ensure_index(columns)
+            else:
+                columns = arr_columns
 
         if exclude is None:
             exclude = set()
         else:
             exclude = set(exclude)
 
-        for col in exclude:
-            del sdict[col]
-            columns.remove(col)
-
         result_index = None
         if index is not None:
             if (isinstance(index, basestring) or
                 not hasattr(index, "__iter__")):
-                result_index = sdict.pop(index)
-                result_index = Index(result_index, name=index)
-                columns.remove(index)
+                i = columns.get_loc(index)
+                exclude.add(index)
+                result_index = Index(arrays[i], name=index)
             else:
                 try:
-                    arrays = []
-                    for field in index:
-                        arrays.append(sdict[field])
-                    for field in index:
-                        del sdict[field]
-                        columns.remove(field)
-                    result_index = MultiIndex.from_arrays(arrays, names=index)
+                    to_remove = [arr_columns.get_loc(field) for field in index]
+
+                    result_index = MultiIndex.from_arrays(
+                        [arrays[i] for i in to_remove], names=index)
+
+                    exclude.update(index)
                 except Exception:
                     result_index = index
 
-        return cls(sdict, index=result_index, columns=columns)
+        if any(exclude):
+            to_remove = [arr_columns.get_loc(col) for col in exclude]
+            arrays = [v for i, v in enumerate(arrays) if i not in to_remove]
+            arr_columns = arr_columns.drop(exclude)
+            columns = columns.drop(exclude)
+
+        mgr = _arrays_to_mgr(arrays, arr_columns, result_index,
+                             columns)
+
+        return DataFrame(mgr)
 
     def to_records(self, index=True):
         """
@@ -5217,9 +5231,18 @@ def _to_arrays(data, columns, coerce_float=False, dtype=None):
     """
     Return list of arrays, columns
     """
+    if isinstance(data, DataFrame):
+        if columns is not None:
+            arrays = [data.icol(i).values for i, col in enumerate(data.columns)
+                      if col in columns]
+        else:
+            columns = data.columns
+            arrays = [data.icol(i).values for i in range(len(columns))]
+
+        return arrays, columns
 
     if len(data) == 0:
-        return [], columns if columns is not None else []
+        return [], [] # columns if columns is not None else []
     if isinstance(data[0], (list, tuple)):
         return _list_to_arrays(data, columns, coerce_float=coerce_float,
                                dtype=dtype)
@@ -5231,6 +5254,10 @@ def _to_arrays(data, columns, coerce_float=False, dtype=None):
         return _list_of_series_to_arrays(data, columns,
                                          coerce_float=coerce_float,
                                          dtype=dtype)
+    elif isinstance(data, np.ndarray):
+        columns = list(data.dtype.names)
+        arrays = [data[k] for k in columns]
+        return arrays, columns
     else:
         # last ditch effort
         data = map(tuple, data)

pandas/tests/test_frame.py

@@ -2681,8 +2681,13 @@ def test_from_records_decimal(self):
         self.assert_(np.isnan(df['a'].values[-1]))
 
     def test_from_records_duplicates(self):
-        self.assertRaises(ValueError, DataFrame.from_records,
-                          [(1,2,3), (4,5,6)], columns=['a','b','a'])
+        result = DataFrame.from_records([(1,2,3), (4,5,6)],
+                                        columns=['a','b','a'])
+
+        expected = DataFrame([(1,2,3), (4,5,6)],
+                             columns=['a', 'b', 'a'])
+
+        assert_frame_equal(result, expected)
 
     def test_from_records_set_index_name(self):
         def create_dict(order_id):