Develop Reference

django-import-export | issue in importing data to database | ValueError: Field 'id' expected a number but got ''

I am using Django-import-export for importing data but facing an error given below.
i am using django 4.0.6 python 3.10.5 with PostgreSql and most latest version of django import export
Code settings i tried to import data to postgresql database by django-import-export
class MemberResource(resources.ModelResource):
Brand=Field()
class Meta:
model = model
fields=('id','title','Model_code','Chipset','chipset_description','image','Brand','Cat')
export_order=('id','title','Model_code','Chipset','chipset_description','image','Brand','Cat')
def dehydrate_Brand(self, obj):
return str(obj.Brand.title)
class modelAdmin(ImportExportModelAdmin):
resource_class = MemberResource
list_display=['id','title','Model_code','Chipset','chipset_description','Brand','categories']
search_fields = ['title','Model_code','Chipset',]
fields=('title','Model_code','Chipset','chipset_description','image','Brand','Cat')
admin.site.register(model,modelAdmin)
and got below error also attached the image where i exported the data from app and then edited the same and tried to import and stuck with below error.
[Line number: 1 - Field 'id' expected a number but got ''.
2, f9, sd, gf, kjkj, images/sample_3pahsfV.jfif, 1, 1
Traceback (most recent call last):
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\fields\related_descriptors.py", line 187, in _get_
rel_obj = self.field.get_cached_value(instance)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\fields\mixins.py", line 15, in get_cached_value
return instance._state.fields_cache\[cache_name\]
KeyError: 'Brand'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\fields\_init_.py", line 1988, in get_prep_value
return int(value)
ValueError: invalid literal for int() with base 10: ''
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\import_export\resources.py", line 707, in import_row
diff = self.get_diff_class()(self, original, new)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\import_export\resources.py", line 241, in _init_
self.left = self._export_resource_fields(resource, instance)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\import_export\resources.py", line 262, in _export_resource_fields
return \[resource.export_field(f, instance) if instance else "" for f in resource.get_user_visible_fields()\]
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\import_export\resources.py", line 262, in <listcomp>
return \[resource.export_field(f, instance) if instance else "" for f in resource.get_user_visible_fields()\]
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\import_export\resources.py", line 919, in export_field
return method(obj)
File "C:\Users\gsminfinity\Desktop\Master\admin\firmApp\admin.py", line 20, in dehydrate_Brand
return str(obj.Brand.title)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\fields\related_descriptors.py", line 205, in _get_
rel_obj = self.get_object(instance)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\fields\related_descriptors.py", line 168, in get_object
return qs.get(self.field.get_reverse_related_filter(instance))
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\query.py", line 482, in get
clone = self._chain() if self.query.combinator else self.filter(*args, **kwargs)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\query.py", line 1071, in filter
return self._filter_or_exclude(False, args, kwargs)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\query.py", line 1089, in _filter_or_exclude
clone._filter_or_exclude_inplace(negate, args, kwargs)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\query.py", line 1096, in _filter_or_exclude_inplace
self._query.add_q(Q(*args, **kwargs))
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\sql\query.py", line 1502, in add_q
clause, _ = self._add_q(q_object, self.used_aliases)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\sql\query.py", line 1532, in _add_q
child_clause, needed_inner = self.build_filter(
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\sql\query.py", line 1358, in build_filter
return self._add_q(
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\sql\query.py", line 1532, in _add_q
child_clause, needed_inner = self.build_filter(
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\sql\query.py", line 1448, in build_filter
condition = self.build_lookup(lookups, col, value)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\sql\query.py", line 1273, in build_lookup
lookup = lookup_class(lhs, rhs)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\lookups.py", line 27, in _init_
self.rhs = self.get_prep_lookup()
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\lookups.py", line 85, in get_prep_lookup
return self.lhs.output_field.get_prep_value(self.rhs)
File "C:\Users\gsminfinity\Desktop\Master\venv\lib\site-packages\django\db\models\fields\_init_.py", line 1990, in get_prep_value
raise e._class_(
ValueError: Field 'id' expected a number but got ''.][1]

the issue was dehydrating foreign key gives only read-only data which was for the purpose of export only to import we need to use the widgets which will resolve the foreign keys and import the data to database.
Link to Documentation
class MemberResource(resources.ModelResource):
Brand = fields.Field(
column_name='Brand',
attribute='Brand',
widget=ForeignKeyWidget(brand, 'title'))
class Meta:
model = model
fields=('id','title','Model_code','Chipset','chipset_description','image','Brand','Cat')

That means that there are missing values in the "id" column.
I don't know "Django import / export", sounds very good. An old way of mine with any import/export is to export some sample data and then open it in excel or pandas. Because of the following:
You have all the required columns.
The columns are in the correct order.
And if you have test data you can see the data-types of each column.
Take care using Excel because it changes True into TRUE and generates weird date formats.
Another issue is that after importing django u must also be aware of the last "id" number in order to give the correct AutoIncrement for a new record. That's usually corrected with SQL, but maybe "Django-import-export" corrects this automatically.

Why does the "browse" method return a wrong recordset using a non-existing id?

When I use the search method and I create a recordset looking for a non-existing id the result is a empty recordset as expected:
>>> self.env['account.invoice'].search([('id', 'in', [23232323123123123, ])])
account.invoice()
But if I do the same with the browse method, the result is a recordset with that id, but actually the record does not exist:
>>> o = self.env['account.invoice'].browse([23232323123123123])
>>> o
account.invoice(23232323123123123,)
>>> o.id
23232323123123123
>>> o.number
Traceback (most recent call last):
File "/path/to/odoo/odoo/fields.py", line 937, in __get__
value = record.env.cache.get(record, self)
File "/path/to/odoo/odoo/api.py", line 960, in get
value = self._data[field][record.id][key]
KeyError: (<odoo.sql_db.Cursor object at 0x7f4d2985e9e8>, 1)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.5/code.py", line 91, in runcode
exec(code, self.locals)
File "<console>", line 1, in <module>
File "/path/to/odoo/odoo/fields.py", line 937, in __get__
value = record.env.cache.get(record, self)
File "/path/to/odoo/odoo/api.py", line 961, in get
return value.get() if isinstance(value, SpecialValue) else value
File "/path/to/odoo/odoo/api.py", line 993, in getter
raise exception
File "/usr/lib/python3.5/code.py", line 91, in runcode
exec(code, self.locals)
File "<console>", line 1, in <module>
File "/path/to/odoo/odoo/fields.py", line 937, in __get__
value = record.env.cache.get(record, self)
File "/path/to/odoo/odoo/api.py", line 961, in get
return value.get() if isinstance(value, SpecialValue) else value
File "/path/to/odoo/odoo/api.py", line 993, in getter
raise exception
File "/usr/lib/python3.5/code.py", line 91, in runcode
exec(code, self.locals)
File "<console>", line 1, in <module>
File "/opt/odoo/odoo_11/src/linked/l10n_es_account_invoice_sequence/models/account_invoice.py", line 64, in unlink
self.filtered(lambda x: x.journal_id.invoice_sequence_id).write(
File "/path/to/odoo/odoo/models.py", line 4540, in filtered
return self.browse([rec.id for rec in self if func(rec)])
File "/path/to/odoo/odoo/models.py", line 4540, in <listcomp>
return self.browse([rec.id for rec in self if func(rec)])
File "/opt/odoo/odoo_11/src/linked/l10n_es_account_invoice_sequence/models/account_invoice.py", line 64, in <lambda>
self.filtered(lambda x: x.journal_id.invoice_sequence_id).write(
File "/path/to/odoo/odoo/fields.py", line 944, in __get__
value = record.env.cache.get(record, self)
File "/path/to/odoo/odoo/api.py", line 961, in get
return value.get() if isinstance(value, SpecialValue) else value
File "/path/to/odoo/odoo/api.py", line 993, in getter
raise exception
File "/path/to/odoo/odoo/models.py", line 2601, in read
values[name] = field.convert_to_read(record[name], record, use_name_get)
File "/path/to/odoo/odoo/models.py", line 4758, in __getitem__
return self._fields[key].__get__(self, type(self))
File "/path/to/odoo/odoo/fields.py", line 937, in __get__
value = record.env.cache.get(record, self)
File "/path/to/odoo/odoo/api.py", line 961, in get
return value.get() if isinstance(value, SpecialValue) else value
File "/path/to/odoo/odoo/api.py", line 993, in getter
raise exception
odoo.exceptions.MissingError: ('Record does not exist or has been deleted.', None)
This is the explanation in the documentation:
Takes a database id or a list of ids and returns a recordset, useful when record ids are obtained from outside Odoo (e.g. round-trip through external system) or when calling methods in the old API
Is this a normal behaviour? I see this method kind of useless
Do I have to check if all the records used in a browse exist in the database beforehand?

Yes, its the normal behaviour. Check the browe method. This line is commented
assert all(isinstance(id, IdType) for id in ids), "Browsing invalid ids: %s" % ids
But you have the method exists
exists()
Returns a new recordset containing only the records which exist in the database. Can be used to check whether a record (e.g. obtained externally) still exists:

blaze-odo error when trying to export an odo result to csv with MSSQL backend

I'm having an issue exporting some data from a MSSQL back-end to csv using blaze and odo.
The first thing that I do is create a blaze.data object that connects to the MSSQL db using a dns connection and pyodbc as the driver.
adjustment_statuses is a table in the database that the blaze.data object is connected to.
Then I use odo to take that result set and output it to a csv. When I output to a json format it works as expected. My hunch is that there is some issue with mssql support as an odo backend. Any odo devs have any insight here?
ez = bz.data(EZ_DB)
result = ez.adjustment_statuses
odo(result, os.path.join(ETL_STAGING_LOC, 'ez_adjustment.csv'))
Traceback (most recent call last):
File "<input>", line 25, in <module>
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\odo\odo.py", line 91, in odo
return into(target, source, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\multipledispatch\dispatcher.py", line 164, in __call__
return func(*args, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\blaze\interactive.py", line 406, in into
return into(a, result, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\multipledispatch\dispatcher.py", line 164, in __call__
return func(*args, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\odo\into.py", line 43, in wrapped
return f(*args, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\odo\into.py", line 143, in into_string
return into(a, b, dshape=dshape, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\multipledispatch\dispatcher.py", line 164, in __call__
return func(*args, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\odo\into.py", line 43, in wrapped
return f(*args, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\odo\into.py", line 131, in into_object
return append(target, source, dshape=dshape, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\multipledispatch\dispatcher.py", line 164, in __call__
return func(*args, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\odo\backends\sql.py", line 808, in append_table_to_csv
conn.execute(stmt)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\engine\base.py", line 945, in execute
return meth(self, multiparams, params)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\sql\elements.py", line 263, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\engine\base.py", line 1046, in _execute_clauseelement
if not self.schema_for_object.is_default else None)
File "<string>", line 1, in <lambda>
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\sql\elements.py", line 436, in compile
return self._compiler(dialect, bind=bind, **kw)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\sql\elements.py", line 442, in _compiler
return dialect.statement_compiler(dialect, self, **kw)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\dialects\mssql\base.py", line 1107, in __init__
super(MSSQLCompiler, self).__init__(*args, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\sql\compiler.py", line 435, in __init__
Compiled.__init__(self, dialect, statement, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\sql\compiler.py", line 216, in __init__
self.string = self.process(self.statement, **compile_kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\sql\compiler.py", line 242, in process
return obj._compiler_dispatch(self, **kwargs)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\ext\compiler.py", line 435, in <lambda>
lambda *arg, **kw: existing(*arg, **kw))
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\ext\compiler.py", line 474, in __call__
return fn(element, compiler, **kw)
File "C:\Users\jgenser\AppData\Local\Continuum\Anaconda2\envs\etl\lib\site-packages\sqlalchemy\ext\compiler.py", line 430, in _wrap_existing_dispatch
"compilation handler." % type(element))
CompileError: <class 'odo.backends.sql.CopyToCSV'> construct has no default compilation handler.

SQLAlchemy column with enum using PEP 435 compliant enum produces table creation error

I have an enum... defined very similarly to here.
When I try to create a table using the enum, as in the example, I get an error during table creation. It is a very vague error. Here is code that replicates the problem:
from sqlalchemy import Table, MetaData, Column, Enum, create_engine
import enum
class myEnum(enum.Enum):
one = 'one'
two = 'two'
three = 'three'
def main():
e = create_engine('sqlite:///:memory:')
e.echo = True
m = MetaData(bind = e)
t = Table('table', m, Column('my_enum', Enum(myEnum)))
t.create()
if __name__ == '__main__': main()
I get an AttributeError: replace when I run this code, with a seemingly useless stacktrace. I honestly don't even know where to begin debugging this and while I can conceive of a couple of possible workarounds, I am relatively new to SQLAlchemy so I'm not sure what the cleanest solution is. Surely basic enum support is a relatively simple expectation of a ORM framework?
Here is the full stack trace:
Traceback (most recent call last):
File "table_test.py", line 17, in <module>
if __name__ == '__main__': main()
File "table_test.py", line 16, in main
t.create()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\schema.py", line 725, in create
checkfirst=checkfirst)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1856, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1481, in _run_visitor
**kwargs).traverse_single(element)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\visitors.py", line 121, in traverse_single
return meth(obj, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\ddl.py", line 764, in visit_table
include_foreign_key_constraints=include_foreign_key_constraints
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 914, in execute
return meth(self, multiparams, params)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\ddl.py", line 68, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 962, in _execute_ddl
compiled = ddl.compile(dialect=dialect)
File "<string>", line 1, in <lambda>
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\elements.py", line 494, in compile
return self._compiler(dialect, bind=bind, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\ddl.py", line 26, in _compiler
return dialect.ddl_compiler(dialect, self, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 190, in __init__
self.string = self.process(self.statement, **compile_kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 213, in process
return obj._compiler_dispatch(self, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 2173, in visit_create_table
create.include_foreign_key_constraints)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 2220, in create_table_constraints
for constraint in constraints
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 2218, in <genexpr>
p for p in
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 2226, in <genexpr>
not getattr(constraint, 'use_alter', False)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 213, in process
return obj._compiler_dispatch(self, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\visitors.py", line 93, in _compiler_dispatch
return meth(self, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 2369, in visit_check_constraint
literal_binds=True)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 213, in process
return obj._compiler_dispatch(self, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 927, in visit_binary
return self._generate_generic_binary(binary, opstring, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 944, in _generate_generic_binary
binary.right._compiler_dispatch(self, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 527, in visit_grouping
return "(" + grouping.element._compiler_dispatch(self, **kwargs) + ")"
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 751, in visit_clauselist
for c in clauselist.clauses)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 748, in <genexpr>
s for s in
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 751, in <genexpr>
for c in clauselist.clauses)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 1071, in visit_bindparam
bindparam, within_columns_clause=True, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 1103, in render_literal_bindparam
return self.render_literal_value(value, bindparam.type)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\compiler.py", line 1118, in render_literal_value
return processor(value)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\sql\sqltypes.py", line 171, in process
value = value.replace("'", "''")
File "C:\Anaconda3\lib\enum.py", line 268, in __getattr__
raise AttributeError(name) from None
AttributeError: replace

PEP-435 enum support is being added in 1.1.
1.1.0b1 was recently released. You can upgrade to 1.1.0b1 but be wary of bugs. Based on release history I would say the stable version should be released in a month or two.
how to upgrade:
pip install 'sqlalchemy==1.1.0b3'

Streaming results with Blaze and SqlAlchemy

I am trying to use Blaze/Odo to read a large (~70M rows) result set from Redshift. By default SqlAlchemy witll try to read the whole result into memory, before starting to process it. This can be prevented by either
execution_options(stream_results=True) on the engine/session or yield_per(sane_number) on the query. When working from Blaze SqlAchemy queries are generated behind the covers, leaving the execution_options approach. Unfortunately the following throws and error.
from sqlalchemy import create_engine
from blaze import Data
redshift_params = (redshift_user, redshift_pass, redshift_endpoint, port, dbname)
engine_string = "redshift+psycopg2://%s:%s#%s:%d/%s" % redshift_params
engine = create_engine(engine_string,
execution_options=dict(stream_results=True)
)
db = Data(engine)
The exception is:
...
/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/engine/result.pyc in __buffer_rows(self)
1124 return
1125 size = getattr(self, '_bufsize', 1)
-> 1126 self.__rowbuffer = collections.deque(self.cursor.fetchmany(size))
1127 self._bufsize = self.size_growth.get(size, size)
1128 if self._max_row_buffer is not None:
InternalError: (psycopg2.InternalError) opening multiple cursors from within the same client connection is not allowed.
If I leave out the execution_options=dict(stream_results=True) then the above works, but doing something like
odo(db.mytable, 'mytable.bcolz')
will run out of memory for large tables.
Using execution_options(stream_results=True) does work with pandas.read_csv. The following code works fine, using only moderate amounts of memory:
from sqlalchemy import create_engine
import pandas as pd
redshift_params = (redshift_user, redshift_pass, redshift_endpoint, port, dbname)
engine_string = "postgresql+psycopg2://%s:%s#%s:%d/%s" % redshift_params
engine = create_engine(engine_string,
execution_options=dict(stream_results=True)
)
compression='bz2'
res = pd.read_sql_query(queryString
engine,
chunksize=2**20)
for i, df in enumerate(res):
df.to_csv('results-%s.csv.%s' % (i, compression), compression=compression)
This is the complete stack trace:
...
Data(engine)
No handlers could be found for logger "sqlalchemy.pool.QueuePool"
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/mahler/anaconda/lib/python2.7/site-packages/blaze/interactive.py", line 122, in Data
dshape = discover(data)
File "/home/mahler/anaconda/lib/python2.7/site-packages/multipledispatch/dispatcher.py", line 164, in __call__
return func(*args, **kwargs)
File "/home/mahler/anaconda/lib/python2.7/site-packages/odo/backends/sql.py", line 242, in discover
return discover(metadata)
File "/home/mahler/anaconda/lib/python2.7/site-packages/multipledispatch/dispatcher.py", line 164, in __call__
return func(*args, **kwargs)
File "/home/mahler/anaconda/lib/python2.7/site-packages/odo/backends/sql.py", line 248, in discover
metadata.reflect(views=metadata.bind.dialect.supports_views)
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/sql/schema.py", line 3623, in reflect
bind.dialect.get_view_names(conn, schema)
File "<string>", line 2, in get_view_names
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/engine/reflection.py", line 42, in cache
return fn(self, con, *args, **kw)
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/dialects/postgresql/base.py", line 2347, in get_view_names
for row in connection.execute(s)]
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/engine/result.py", line 713, in __iter__
row = self.fetchone()
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/engine/result.py", line 1026, in fetchone
self.cursor, self.context)
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1341, in _handle_dbapi_exception
exc_info
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 200, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/engine/result.py", line 1017, in fetchone
row = self._fetchone_impl()
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/engine/result.py", line 1139, in _fetchone_impl
self.__buffer_rows()
File "/home/mahler/anaconda/lib/python2.7/site-packages/sqlalchemy/engine/result.py", line 1126, in __buffer_rows
self.__rowbuffer = collections.deque(self.cursor.fetchmany(size))
sqlalchemy.exc.InternalError: (psycopg2.InternalError) opening multiple cursors from within the same client connection is not allowed.