blaze / blaze/odo

CSV to sqlite fails with "TypeError: can only concatenate list (not "TextFileReader") to list"

Open
#585 5 comments 8 reactions 0 assignees View on GitHub
Dominant language
Python
Stars
1k
Forks
131
PR merge metrics
No merged PRs in 30d

Description

Code run: `odo("./test.csv", "sqlite:///database.db::country_partner_year")`.

The offending line is at: https://github.com/blaze/odo/blob/39bf5770935c7cb50ec400aec378857379ba88bd/odo/backends/csv.py#L371

Pandas (v0.20.3) seems to return a TextFileReader object when setting `chunksize=blah`, and so first is of type list[DataFrame], and rest is of type TextFileReader, and adding them doesn't make any sense - this seems to be a job for `pd.concat()` or something similar, perhaps `itertools.chain` depending on what exactly is happening after that function.

Full backtrace:

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/odo.py in odo(source, target, **kwargs)
89 odo.append.append - Add things onto existing things
90 """
---> 91 return into(target, source, **kwargs)

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/multipledispatch/dispatcher.py in __call__(self, *args, **kwargs)
162 self._cache[types] = func
163 try:
--> 164 return func(*args, **kwargs)
165
166 except MDNotImplementedError:

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in wrapped(*args, **kwargs)
41 raise TypeError('dshape argument is not an instance of DataShape')
42 kwargs['dshape'] = dshape
---> 43 return f(*args, **kwargs)
44 return wrapped
45

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in into_string_string(a, b, **kwargs)
147 @validate
148 def into_string_string(a, b, **kwargs):
--> 149 return into(a, resource(b, **kwargs), **kwargs)
150
151

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/multipledispatch/dispatcher.py in __call__(self, *args, **kwargs)
162 self._cache[types] = func
163 try:
--> 164 return func(*args, **kwargs)
165
166 except MDNotImplementedError:

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in wrapped(*args, **kwargs)
41 raise TypeError('dshape argument is not an instance of DataShape')
42 kwargs['dshape'] = dshape
---> 43 return f(*args, **kwargs)
44 return wrapped
45

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in into_string(uri, b, dshape, **kwargs)
141
142 a = resource(uri, dshape=resource_ds, expected_dshape=dshape, **kwargs)
--> 143 return into(a, b, dshape=dshape, **kwargs)
144
145

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/multipledispatch/dispatcher.py in __call__(self, *args, **kwargs)
162 self._cache[types] = func
163 try:
--> 164 return func(*args, **kwargs)
165
166 except MDNotImplementedError:

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in wrapped(*args, **kwargs)
41 raise TypeError('dshape argument is not an instance of DataShape')
42 kwargs['dshape'] = dshape
---> 43 return f(*args, **kwargs)
44 return wrapped
45

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/into.py in into_object(target, source, dshape, **kwargs)
129 if dshape is None:
130 dshape = discover(source)
--> 131 return append(target, source, dshape=dshape, **kwargs)
132
133

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/multipledispatch/dispatcher.py in __call__(self, *args, **kwargs)
162 self._cache[types] = func
163 try:
--> 164 return func(*args, **kwargs)
165
166 except MDNotImplementedError:

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/backends/sql.py in append_anything_to_sql_Table(t, o, **kwargs)
525 @append.register(sa.Table, object)
526 def append_anything_to_sql_Table(t, o, **kwargs):
--> 527 return append(t, convert(Iterator, o, **kwargs), **kwargs)
528
529

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/core.py in __call__(self, *args, **kwargs)
81
82 def __call__(self, *args, **kwargs):
---> 83 return _transform(self.graph, *args, **kwargs)
84
85

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/core.py in _transform(graph, target, source, excluded_edges, ooc_types, **kwargs)
104 for convert_from, convert_to, f, cost in path_proxy:
105 try:
--> 106 x = f(x, excluded_edges=excluded_edges, **kwargs)
107 except NotImplementedError as e:
108 if kwargs.get('raise_on_errors'):

/nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/backends/csv.py in CSV_to_chunks_of_dataframes(c, chunksize, **kwargs)
369 rest = []
370
--> 371 data = [first] + rest
372 return chunks(pd.DataFrame)(data)
373

TypeError: can only concatenate list (not "TextFileReader") to list
> /nfs/projects_nobackup/c/cidgrowlab/Mali/intl-atlas-api/env/lib/python3.4/site-packages/odo/backends/csv.py(371)CSV_to_chunks_of_dataframes()
369 rest = []
370
--> 371 data = [first] + rest
372 return chunks(pd.DataFrame)(data)
373

Contributor guide

No contributing guide indexed for this repository

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.