CSV to sqlite fails with "TypeError: can only concatenate list (not "TextFileReader") to list"
- Dominant language
- Python
- Stars
- 1k
- Forks
- 131
- PR merge metrics
- No merged PRs in 30d
Description
Code run: `odo("./test.csv", "sqlite:///database.db::country_partner_year")`.
The offending line is at: https://github.com/blaze/odo/blob/39bf5770935c7cb50ec400aec378857379ba88bd/odo/backends/csv.py#L371
Pandas (v0.20.3) seems to return a TextFileReader object when setting `chunksize=blah`, and so first is of type list[DataFrame], and rest is of type TextFileReader, and adding them doesn't make any sense - this seems to be a job for `pd.concat()` or something similar, perhaps `itertools.chain` depending on what exactly is happening after that function.
Full backtrace:
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/odo.py in odo(source, target, **kwargs)
89 odo.append.append - Add things onto existing things
90 """
---> 91 return into(target, source, **kwargs)
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/multipledispatch/dispatcher.py in __call__(self, *args, **kwargs)
162 self._cache[types] = func
163 try:
--> 164 return func(*args, **kwargs)
165
166 except MDNotImplementedError:
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in wrapped(*args, **kwargs)
41 raise TypeError('dshape argument is not an instance of DataShape')
42 kwargs['dshape'] = dshape
---> 43 return f(*args, **kwargs)
44 return wrapped
45
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in into_string_string(a, b, **kwargs)
147 @validate
148 def into_string_string(a, b, **kwargs):
--> 149 return into(a, resource(b, **kwargs), **kwargs)
150
151
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/multipledispatch/dispatcher.py in __call__(self, *args, **kwargs)
162 self._cache[types] = func
163 try:
--> 164 return func(*args, **kwargs)
165
166 except MDNotImplementedError:
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in wrapped(*args, **kwargs)
41 raise TypeError('dshape argument is not an instance of DataShape')
42 kwargs['dshape'] = dshape
---> 43 return f(*args, **kwargs)
44 return wrapped
45
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in into_string(uri, b, dshape, **kwargs)
141
142 a = resource(uri, dshape=resource_ds, expected_dshape=dshape, **kwargs)
--> 143 return into(a, b, dshape=dshape, **kwargs)
144
145
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/multipledispatch/dispatcher.py in __call__(self, *args, **kwargs)
162 self._cache[types] = func
163 try:
--> 164 return func(*args, **kwargs)
165
166 except MDNotImplementedError:
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in wrapped(*args, **kwargs)
41 raise TypeError('dshape argument is not an instance of DataShape')
42 kwargs['dshape'] = dshape
---> 43 return f(*args, **kwargs)
44 return wrapped
45
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in into_object(target, source, dshape, **kwargs)
129 if dshape is None:
130 dshape = discover(source)
--> 131 return append(target, source, dshape=dshape, **kwargs)
132
133
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/multipledispatch/dispatcher.py in __call__(self, *args, **kwargs)
162 self._cache[types] = func
163 try:
--> 164 return func(*args, **kwargs)
165
166 except MDNotImplementedError:
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/backends/sql.py in append_anything_to_sql_Table(t, o, **kwargs)
525 @append.register(sa.Table, object)
526 def append_anything_to_sql_Table(t, o, **kwargs):
--> 527 return append(t, convert(Iterator, o, **kwargs), **kwargs)
528
529
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/core.py in __call__(self, *args, **kwargs)
81
82 def __call__(self, *args, **kwargs):
---> 83 return _transform(self.graph, *args, **kwargs)
84
85
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/core.py in _transform(graph, target, source, excluded_edges, ooc_types, **kwargs)
104 for convert_from, convert_to, f, cost in path_proxy:
105 try:
--> 106 x = f(x, excluded_edges=excluded_edges, **kwargs)
107 except NotImplementedError as e:
108 if kwargs.get('raise_on_errors'):
/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/backends/csv.py in CSV_to_chunks_of_dataframes(c, chunksize, **kwargs)
369 rest = []
370
--> 371 data = [first] + rest
372 return chunks(pd.DataFrame)(data)
373
TypeError: can only concatenate list (not "TextFileReader") to list
> /nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/backends/csv.py(371)CSV_to_chunks_of_dataframes()
369 rest = []
370
--> 371 data = [first] + rest
372 return chunks(pd.DataFrame)(data)
373
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.