Using Django 1.8 and Postgres 9.3 - I have a Django model as follows which contains log lines. I would like to extract all domain_name values that were never allowed access.
class Logs(models.Model):
date = DateTimeField(db_index=True)
action = TextField(default='a', null=True, blank=True, db_index=True)
url = TextField(null=True, blank=True)
stats = JsonField(null=True, blank=True)
domain_name = TextField(null=True, blank=True, db_index=True)
<snip>
This SQL query works nicely, but I'm having trouble translating it to a Django ORM queryset.
select domain_name from reporter_log
GROUP BY domain_name
HAVING COUNT(CASE WHEN action = 'a' OR action = 'ae' then 1 END) = 0;
I would have expected the following queryset would work:
LogLine.objects.annotate(
allowed=Count(Case(
When(action='a', then=Value(1)),
When(action='ae', then=Value(1)),
default=Value(0),
output_field=IntegerField(),
)
))
But I get a traceback:
ProgrammingError Traceback (most recent call last)
<ipython-input-4-feb056328fe8> in <module>()
4 When(action='ae', then=Value(1)),
5 default=Value(0),
----> 6 output_field=IntegerField(),
7 )
8 ))
/usr/lib/python3/dist-packages/IPython/core/displayhook.py in __call__(self, result)
245 self.start_displayhook()
246 self.write_output_prompt()
--> 247 format_dict, md_dict = self.compute_format_data(result)
248 self.write_format_data(format_dict, md_dict)
249 self.update_user_ns(result)
/usr/lib/python3/dist-packages/IPython/core/displayhook.py in compute_format_data(self, result)
155
156 """
--> 157 return self.shell.display_formatter.format(result)
158
159 def write_format_data(self, format_dict, md_dict=None):
/usr/lib/python3/dist-packages/IPython/core/formatters.py in format(self, obj, include, exclude)
150 md = None
151 try:
--> 152 data = formatter(obj)
153 except:
154 # FIXME: log the exception
/usr/lib/python3/dist-packages/IPython/core/formatters.py in __call__(self, obj)
478 type_pprinters=self.type_printers,
479 deferred_pprinters=self.deferred_printers)
--> 480 printer.pretty(obj)
481 printer.flush()
482 return stream.getvalue()
/usr/lib/python3/dist-packages/IPython/lib/pretty.py in pretty(self, obj)
361 if isinstance(meth, collections.Callable):
362 return meth(obj, self, cycle)
--> 363 return _default_pprint(obj, self, cycle)
364 finally:
365 self.end_group()
/usr/lib/python3/dist-packages/IPython/lib/pretty.py in _default_pprint(obj, p, cycle)
481 if getattr(klass, '__repr__', None) not in _baseclass_reprs:
482 # A user-provided repr.
--> 483 p.text(repr(obj))
484 return
485 p.begin_group(1, '<')
/usr/local/lib/python3.4/dist-packages/django/db/models/query.py in __repr__(self)
136
137 def __repr__(self):
--> 138 data = list(self[:REPR_OUTPUT_SIZE + 1])
139 if len(data) > REPR_OUTPUT_SIZE:
140 data[-1] = "...(remaining elements truncated)..."
/usr/local/lib/python3.4/dist-packages/django/db/models/query.py in __iter__(self)
160 - Responsible for turning the rows into model objects.
161 """
--> 162 self._fetch_all()
163 return iter(self._result_cache)
164
/usr/local/lib/python3.4/dist-packages/django/db/models/query.py in _fetch_all(self)
963 def _fetch_all(self):
964 if self._result_cache is None:
--> 965 self._result_cache = list(self.iterator())
966 if self._prefetch_related_lookups and not self._prefetch_done:
967 self._prefetch_related_objects()
/usr/local/lib/python3.4/dist-packages/django/db/models/query.py in iterator(self)
236 # Execute the query. This will also fill compiler.select, klass_info,
237 # and annotations.
--> 238 results = compiler.execute_sql()
239 select, klass_info, annotation_col_map = (compiler.select, compiler.klass_info,
240 compiler.annotation_col_map)
/usr/local/lib/python3.4/dist-packages/django/db/models/sql/compiler.py in execute_sql(self, result_type)
838 cursor = self.connection.cursor()
839 try:
--> 840 cursor.execute(sql, params)
841 except Exception:
842 cursor.close()
/usr/local/lib/python3.4/dist-packages/django/db/backends/utils.py in execute(self, sql, params)
77 start = time()
78 try:
---> 79 return super(CursorDebugWrapper, self).execute(sql, params)
80 finally:
81 stop = time()
/usr/local/lib/python3.4/dist-packages/django/db/backends/utils.py in execute(self, sql, params)
62 return self.cursor.execute(sql)
63 else:
---> 64 return self.cursor.execute(sql, params)
65
66 def executemany(self, sql, param_list):
/usr/local/lib/python3.4/dist-packages/django/db/utils.py in __exit__(self, exc_type, exc_value, traceback)
95 if dj_exc_type not in (DataError, IntegrityError):
96 self.wrapper.errors_occurred = True
---> 97 six.reraise(dj_exc_type, dj_exc_value, traceback)
98
99 def __call__(self, func):
/usr/local/lib/python3.4/dist-packages/django/utils/six.py in reraise(tp, value, tb)
656 value = tp()
657 if value.__traceback__ is not tb:
--> 658 raise value.with_traceback(tb)
659 raise value
660
/usr/local/lib/python3.4/dist-packages/django/db/backends/utils.py in execute(self, sql, params)
62 return self.cursor.execute(sql)
63 else:
---> 64 return self.cursor.execute(sql, params)
65
66 def executemany(self, sql, param_list):
ProgrammingError: could not identify an equality operator for type json
LINE 1: ...e"."domain_name", "reporter_logs"."rule_type", "reporter_...
^
Here is the output of the query that fails.
[{'sql': 'SELECT "reporter_logs"."id", "reporter_logs"."date", "reporter_logs"."userip", "reporter_logs"."action", "reporter_logs"."url", "reporter_logs"."method", "reporter_logs"."status", "reporter_logs"."mimetype", "reporter_logs"."content_length", "reporter_logs"."pruned", "reporter_logs"."page_title", "reporter_logs"."user_agent", "reporter_logs"."domain_name", "reporter_logs"."rule_type", "reporter_logs"."tally_stats", "reporter_logs"."cat_stats", "reporter_logs"."grade_stats", "reporter_logs"."ignored", "reporter_logs"."category", "reporter_logs"."genre", "reporter_logs"."grade", "reporter_logs"."top_grade", "reporter_logs"."category_confidence", "reporter_logs"."grade_confidence", COUNT(CASE WHEN "reporter_logs"."action" = \'a\' THEN 1 WHEN "reporter_logs"."action" = \'ae\' THEN 1 ELSE 0 END) AS "allowed" FROM "reporter_logs" GROUP BY "reporter_logs"."id", "reporter_logs"."date", "reporter_logs"."userip", "reporter_logs"."action", "reporter_logs"."url", "reporter_logs"."method", "reporter_logs"."status", "reporter_logs"."mimetype", "reporter_logs"."content_length", "reporter_logs"."pruned", "reporter_logs"."page_title", "reporter_logs"."user_agent", "reporter_logs"."domain_name", "reporter_logs"."rule_type", "reporter_logs"."tally_stats", "reporter_logs"."cat_stats", "reporter_logs"."grade_stats", "reporter_logs"."ignored", "reporter_logs"."category", "reporter_logs"."genre", "reporter_logs"."grade", "reporter_logs"."top_grade", "reporter_logs"."category_confidence", "reporter_logs"."grade_confidence" LIMIT 21', 'time': '0.002'}]
Any suggestions would be appreciated. Including suggestions about alternate ways to write the query.
Try by not using Q function, use separated When instead:
Logs.objects.annotate(
allowed=Count(Case(
When(action='a', then=1),
When(action='ae', then=1),
default=0,
output_field=IntegerField()
)
)).values('date')
Well, thanks #knbk and #Gocht for trying. I finally found the solution to my problem. I had to specify that I only wanted the domain_name column, which took care of the traceback.
Also, I had to use Sum instead of Count to return 0 for all domains that had no matches. Count returned 1 for nonmatching values.
Logs.objects.values('domain_name').annotate(
allowed=Sum(Case(
When(Q(action='a') |Q(action='ae'), then=Value(1)),
default=Value(0),
output_field=IntegerField()
)
))
Related
I have multiple fields that stores the same value type (price) from different sources.
class Product...
price_amazon = ...
price_ebay = ...
price_etsy = ...
#property
def price...
return self.price_amazon or self.price_ebay or self.price_etsy
I'm looking for a way to annotate price to each Product from queryset. It should behave exactly as the price property.
price is either price_amazon, price_ebay or price_etsy - the first not None value.
How would you do that?
I tried:
Product.objects.all().annotate(price=F('price_amazon') or F('price_ebay') or F('price_etsy'))
which raises:
AttributeError Traceback (most recent call last)
~/PycharmProjects/.virtualenvs/.cloud/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
700 type_pprinters=self.type_printers,
701 deferred_pprinters=self.deferred_printers)
--> 702 printer.pretty(obj)
703 printer.flush()
704 return stream.getvalue()
~/PycharmProjects/.virtualenvs/.cloud/lib/python3.8/site-packages/IPython/lib/pretty.py in pretty(self, obj)
392 if cls is not object \
393 and callable(cls.__dict__.get('__repr__')):
--> 394 return _repr_pprint(obj, self, cycle)
395
396 return _default_pprint(obj, self, cycle)
~/PycharmProjects/.virtualenvs/.cloud/lib/python3.8/site-packages/IPython/lib/pretty.py in _repr_pprint(obj, p, cycle)
698 """A pprint that just redirects to the normal repr function."""
699 # Find newlines and replace them with p.break_()
--> 700 output = repr(obj)
701 lines = output.splitlines()
702 with p.group():
~/PycharmProjects/.virtualenvs/.cloud/lib/python3.8/site-packages/django/db/models/query.py in __repr__(self)
261
262 def __repr__(self):
--> 263 data = list(self[:REPR_OUTPUT_SIZE + 1])
264 if len(data) > REPR_OUTPUT_SIZE:
265 data[-1] = "...(remaining elements truncated)..."
~/PycharmProjects/.virtualenvs/.cloud/lib/python3.8/site-packages/django/db/models/query.py in __len__(self)
267
268 def __len__(self):
--> 269 self._fetch_all()
270 return len(self._result_cache)
271
~/PycharmProjects/.virtualenvs/.cloud/lib/python3.8/site-packages/django/db/models/query.py in _fetch_all(self)
1306 def _fetch_all(self):
1307 if self._result_cache is None:
-> 1308 self._result_cache = list(self._iterable_class(self))
1309 if self._prefetch_related_lookups and not self._prefetch_done:
1310 self._prefetch_related_objects()
~/PycharmProjects/.virtualenvs/.cloud/lib/python3.8/site-packages/django/db/models/query.py in __iter__(self)
74 if annotation_col_map:
75 for attr_name, col_pos in annotation_col_map.items():
---> 76 setattr(obj, attr_name, row[col_pos])
77
78 # Add the known related objects to the model.
AttributeError: can't set attribute
This is what Coalesce [Django-doc] is all about. You can annotate with:
from django.db.models.functions import Coalesce
Product.objects.annotate(
price=Coalesce('price_amazon', 'price_ebay', 'price_etsy')
)
The script I'm writing requests news article metadata from an API. In response, it receives a page of results containing several news articles. It is designed to process the records one at a time, extracting the data fields from the json dict and inserting them into postgres.
However, when I run the insert operation, the function returns:
CompileError: Unconsumed column names: urlToImage, publishedAt
How do I get this insert operation to work?
Any help would be greatly appreciated!
Here's my code:
from sqlalchemy import MetaData # for getting table metadata
from sqlalchemy import Table # for interacting with tables
from sqlalchemy import create_engine # for creating db engine
from sqlalchemy.dialects import postgresql
from sqlalchemy.dialects.postgresql import insert # for getting alterate query method to work
# Create DB engine
engine = create_engine('postgresql+psycopg2://{user}:{password}#{hostip}/{db}'.format(**dbkeys))
# Get metadata objects for tables in database
metadata = MetaData(engine, reflect=True)
nstream = metadata.tables['nstream']
for item in response_page['articles']:
# Convert datetime strings from api into Python datetime format
dtp = datetime.strptime(item['publishedAt'], "%Y-%m-%dT%H:%M:%SZ")
inserttw = nstream.insert().values(source_id = item['source']['id'],
source_name = item['source']['name'],
author = item['author'],
title = item['title'],
description = item['description'],
url = item['url'],
urlToImage = item['urlToImage'],
publishedAt = dtp,
uploaded2db = datetime.now(),
content = item['content'])
engine.execute(inserttw)
And the full traceback:
CompileError Traceback (most recent call last)
<ipython-input-10-c5f4a6bff45e> in <module>
63 # 2. If query has more than one page, get additional pages
64
---> 65 get_results(tfrom, engine = engine, max_retries = 5)
<ipython-input-10-c5f4a6bff45e> in get_results(tfrom, engine, max_retries)
39
40 # Append the results to the database using the helper
---> 41 process_page(results)
42
43 # If there is an exception, add to the retry counter and then sleep.
<ipython-input-10-c5f4a6bff45e> in process_page(response_page)
26 uploaded2db = datetime.now(),
27 content = item['content'])
---> 28 engine.execute(inserttw)
29
30 def get_results(tfrom, engine = engine, max_retries = 5):
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py in execute(self, statement, *multiparams, **params)
2073
2074 connection = self.contextual_connect(close_with_result=True)
-> 2075 return connection.execute(statement, *multiparams, **params)
2076
2077 def scalar(self, statement, *multiparams, **params):
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py in execute(self, object, *multiparams, **params)
946 raise exc.ObjectNotExecutableError(object)
947 else:
--> 948 return meth(self, multiparams, params)
949
950 def _execute_function(self, func, multiparams, params):
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/elements.py in _execute_on_connection(self, connection, multiparams, params)
267 def _execute_on_connection(self, connection, multiparams, params):
268 if self.supports_execution:
--> 269 return connection._execute_clauseelement(self, multiparams, params)
270 else:
271 raise exc.ObjectNotExecutableError(self)
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _execute_clauseelement(self, elem, multiparams, params)
1051 inline=len(distilled_params) > 1,
1052 schema_translate_map=self.schema_for_object
-> 1053 if not self.schema_for_object.is_default else None)
1054
1055 ret = self._execute_context(
<string> in <lambda>(self, bind, dialect, **kw)
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/elements.py in compile(self, default, bind, dialect, **kw)
440 else:
441 dialect = default.StrCompileDialect()
--> 442 return self._compiler(dialect, bind=bind, **kw)
443
444 def _compiler(self, dialect, **kw):
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/elements.py in _compiler(self, dialect, **kw)
446 Dialect."""
447
--> 448 return dialect.statement_compiler(dialect, self, **kw)
449
450 def __str__(self):
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py in __init__(self, dialect, statement, column_keys, inline, **kwargs)
451 # dialect.label_length or dialect.max_identifier_length
452 self.truncated_names = {}
--> 453 Compiled.__init__(self, dialect, statement, **kwargs)
454
455 if (
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py in __init__(self, dialect, statement, bind, schema_translate_map, compile_kwargs)
217 if self.can_execute:
218 self.execution_options = statement._execution_options
--> 219 self.string = self.process(self.statement, **compile_kwargs)
220
221 #util.deprecated("0.7", ":class:`.Compiled` objects now compile "
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py in process(self, obj, **kwargs)
243
244 def process(self, obj, **kwargs):
--> 245 return obj._compiler_dispatch(self, **kwargs)
246
247 def __str__(self):
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/visitors.py in _compiler_dispatch(self, visitor, **kw)
79 raise exc.UnsupportedCompilationError(visitor, cls)
80 else:
---> 81 return meth(self, **kw)
82 else:
83 # The optimization opportunity is lost for this case because the
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py in visit_insert(self, insert_stmt, asfrom, **kw)
2057
2058 crud_params = crud._setup_crud_params(
-> 2059 self, insert_stmt, crud.ISINSERT, **kw)
2060
2061 if not crud_params and \
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/crud.py in _setup_crud_params(compiler, stmt, local_stmt_type, **kw)
55 try:
56 if local_stmt_type in (ISINSERT, ISUPDATE):
---> 57 return _get_crud_params(compiler, stmt, **kw)
58 finally:
59 if should_restore:
~/anaconda3/lib/python3.6/site-packages/sqlalchemy/sql/crud.py in _get_crud_params(compiler, stmt, **kw)
144 raise exc.CompileError(
145 "Unconsumed column names: %s" %
--> 146 (", ".join("%s" % c for c in check))
147 )
148
CompileError: Unconsumed column names: urlToImage, publishedAt
The problem, as it turned out was that I was capitalizing the column names wrong.
I figured this out by using SQLalchemy's inspector function. The column names were there, they were just in lower case.
from sqlalchemy import create_engine
from sqlalchemy.engine import reflection
insp = reflection.Inspector.from_engine(engine)
print(insp.get_columns(nstream))
The source of the confusion is that when you create tables in Postgres, Postgres will automatically lower case your column names unless you use quotes when naming them.
This is the working version of the insert command:
inserttw = nstream.insert().values(source_id = item['source']['id'],
source_name = item['source']['name'],
author = item['author'],
title = item['title'],
description = item['description'],
url = item['url'],
urltoimage = item['urlToImage'],
publishedat = dtp,
uploaded2db = datetime.now(),
content = item['content'])
I want to update the email id of some users to: "prefix" + "value of the user email"
Now I can do this for one user as follows:
User.objects.filter(pk=<id>).update(email=Concat(Value("prefix"), 'email'))
However, as soon as I filter on a pk list, I get a nasty error. The query is:
User.objects.filter(pk__in=<list_id>).update(email=Concat(Value("prefix"), 'email'))
The error is:
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/django/db/models/query.pyc in update(self, **kwargs)
561 query.add_update_values(kwargs)
562 with transaction.atomic(using=self.db, savepoint=False):
--> 563 rows = query.get_compiler(self.db).execute_sql(CURSOR)
564 self._result_cache = None
565 return rows
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/django/db/models/sql/compiler.pyc in execute_sql(self, result_type)
1060 related queries are not available.
1061 """
-> 1062 cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
1063 try:
1064 rows = cursor.rowcount if cursor else 0
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/django/db/models/sql/compiler.pyc in execute_sql(self, result_type)
838 cursor = self.connection.cursor()
839 try:
--> 840 cursor.execute(sql, params)
841 except Exception:
842 cursor.close()
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/django/db/backends/utils.pyc in execute(self, sql, params)
77 start = time()
78 try:
---> 79 return super(CursorDebugWrapper, self).execute(sql, params)
80 finally:
81 stop = time()
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/django/db/backends/utils.pyc in execute(self, sql, params)
62 return self.cursor.execute(sql)
63 else:
---> 64 return self.cursor.execute(sql, params)
65
66 def executemany(self, sql, param_list):
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/django/db/utils.pyc in __exit__(self, exc_type, exc_value, traceback)
96 if dj_exc_type not in (DataError, IntegrityError):
97 self.wrapper.errors_occurred = True
---> 98 six.reraise(dj_exc_type, dj_exc_value, traceback)
99
100 def __call__(self, func):
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/django/db/backends/utils.pyc in execute(self, sql, params)
62 return self.cursor.execute(sql)
63 else:
---> 64 return self.cursor.execute(sql, params)
65
66 def executemany(self, sql, param_list):
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/django/db/backends/mysql/base.pyc in execute(self, query, args)
122 try:
123 # args is None means no string interpolation
--> 124 return self.cursor.execute(query, args)
125 except Database.OperationalError as e:
126 # Map some error codes to IntegrityError, since they seem to be
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/MySQLdb/cursors.pyc in execute(self, query, args)
224 except Exception:
225 exc, value = sys.exc_info()[:2]
--> 226 self.errorhandler(self, exc, value)
227 self._executed = query
228 if not self._defer_warnings: self._warning_check()
/Users/zishanahmad/Devel/Env/venv_sliderule/lib/python2.7/site-packages/MySQLdb/connections.pyc in defaulterrorhandler(***failed resolving arguments***)
34 del connection
35 if isinstance(errorvalue, BaseException):
---> 36 raise errorvalue
37 if errorclass is not None:
38 raise errorclass(errorvalue)
OperationalError: (1093, "You can't specify target table 'auth_user' for update in FROM clause")
I don't really understand what the error message is trying to say. Any help would be appreciated.
EDIT:
MySQL version: 5.7.12
Django: 1.8
What does your <list_id> look like? What's the result of User.objects.filter(pk__in=<list_id>)?
In Django 1.10, I have no problem running:
>>> User.objects.filter(pk=1).update(first_name='Test')
1L
>>> User.objects.filter(pk__in=[1, 2]).update(first_name='Test')
2L
As for workarounds, obviously it isn't as efficient, but why not just cache the QuerySet and then iterate over it to update each object? This is (hopefully) a one-time query, so I'm not sure how much of a concern SQL efficiency is.
This should work :)
User.objects.filter(pk__in=<list_id>).update(email=Concat(Value("prefix"), F('email')))
I had the same problem, but solved by trying to convert queryset list_ids to a list(), like this:
ids = Transactions.objects.values_list('id', flat=True)
# This may raise error:
User.objects.filter(pk__in=ids).update(first_name='Test')
# But if convert ids to a list, should work:
list_ids = list(ids)
User.objects.filter(pk__in=list_ids).update(first_name='Test')
Hope it helps :)
So I need to iterate through millions of rows using django. This does not work with the way django will grab them from the DB, so I looked into server side cursors.
I attempted the exact code that the github suggests:
from djorm_core.postgresql import server_side_cursors
with server_side_cursors():
for item in MyModel.objects.all():
#do something
and I got the following error:
(I cannot show the actual line where the error started, but it started on the equivalent of for item in myModel.objects.all() above)
/usr/local/lib/python2.7/dist-packages/django/db/models/query.pyc in __iter__(self)
94 - Responsible for turning the rows into model objects.
95 """
---> 96 self._fetch_all()
97 return iter(self._result_cache)
98
/usr/local/lib/python2.7/dist-packages/django/db/models/query.pyc in _fetch_all(self)
852 def _fetch_all(self):
853 if self._result_cache is None:
--> 854 self._result_cache = list(self.iterator())
855 if self._prefetch_related_lookups and not self._prefetch_done:
856 self._prefetch_related_objects()
/usr/local/lib/python2.7/dist-packages/django/db/models/query.pyc in iterator(self)
941 names = extra_names + field_names + aggregate_names
942
--> 943 for row in self.query.get_compiler(self.db).results_iter():
944 yield dict(zip(names, row))
945
/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.pyc in results_iter(self)
707 fields = None
708 has_aggregate_select = bool(self.query.aggregate_select)
--> 709 for rows in self.execute_sql(MULTI):
710 for row in rows:
711 if has_aggregate_select:
/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.pyc in execute_sql(self, result_type)
780
781 cursor = self.connection.cursor()
--> 782 cursor.execute(sql, params)
783
784 if not result_type:
/usr/local/lib/python2.7/dist-packages/django/db/backends/util.pyc in execute(self, sql, params)
67 start = time()
68 try:
---> 69 return super(CursorDebugWrapper, self).execute(sql, params)
70 finally:
71 stop = time()
/usr/local/lib/python2.7/dist-packages/django/db/backends/util.pyc in execute(self, sql, params)
51 return self.cursor.execute(sql)
52 else:
---> 53 return self.cursor.execute(sql, params)
54
55 def executemany(self, sql, param_list):
/usr/local/lib/python2.7/dist-packages/django/db/utils.pyc in __exit__(self, exc_type, exc_value, traceback)
97 if dj_exc_type not in (DataError, IntegrityError):
98 self.wrapper.errors_occurred = True
---> 99 six.reraise(dj_exc_type, dj_exc_value, traceback)
100
101 def __call__(self, func):
/usr/local/lib/python2.7/dist-packages/django/db/backends/util.pyc in execute(self, sql, params)
51 return self.cursor.execute(sql)
52 else:
---> 53 return self.cursor.execute(sql, params)
54
55 def executemany(self, sql, param_list):
ProgrammingError: can't use a named cursor outside of transactions
is there another way of doing the equivalent? Is there a way to still use this module or is it simply broken as this is exactly what the github account suggests?
So this question has been asked before, but there is no answer. I know that joining an annotated queryset with a distinct one is not implemented in Django, but the question is: what would an alternate way of doing this be?
code example:
qs1 = Example.objects.filter(...).annotate(...)
qs2 = Example.objects.filter(...).distinct(...)
from itertools import chain
answer = chain(qs1,qs2)
but this will return the following error because "it is not implemented" in Django:
/Library/Python/2.7/site-packages/Django-1.6-py2.7.egg/django/db/models/query.pyc in __iter__(self)
94 - Responsible for turning the rows into model objects.
95 """
---> 96 self._fetch_all()
97 return iter(self._result_cache)
98
/Library/Python/2.7/site-packages/Django-1.6-py2.7.egg/django/db/models/query.pyc in _fetch_all(self)
852 def _fetch_all(self):
853 if self._result_cache is None:
--> 854 self._result_cache = list(self.iterator())
855 if self._prefetch_related_lookups and not self._prefetch_done:
856 self._prefetch_related_objects()
/Library/Python/2.7/site-packages/Django-1.6-py2.7.egg/django/db/models/query.pyc in iterator(self)
218 klass_info = get_klass_info(model, max_depth=max_depth,
219 requested=requested, only_load=only_load)
--> 220 for row in compiler.results_iter():
221 if fill_cache:
222 obj, _ = get_cached_row(row, index_start, db, klass_info,
/Library/Python/2.7/site-packages/Django-1.6-py2.7.egg/django/db/models/sql/compiler.pyc in results_iter(self)
708 fields = None
709 has_aggregate_select = bool(self.query.aggregate_select)
--> 710 for rows in self.execute_sql(MULTI):
711 for row in rows:
712 if has_aggregate_select:
/Library/Python/2.7/site-packages/Django-1.6-py2.7.egg/django/db/models/sql/compiler.pyc in execute_sql(self, result_type)
769 """
770 try:
--> 771 sql, params = self.as_sql()
772 if not sql:
773 raise EmptyResultSet
/Library/Python/2.7/site-packages/Django-1.6-py2.7.egg/django/db/models/sql/compiler.pyc in as_sql(self, with_limits, with_col_aliases)
119 if distinct_fields:
120 raise NotImplementedError(
--> 121 "annotate() + distinct(fields) not implemented.")
122 if not ordering:
123 ordering = self.connection.ops.force_no_ordering()
NotImplementedError: annotate() + distinct(fields) not implemented.
So, again, the question is: What is some way to accomplish chaining these querysets?
I had to do something like this some time back, so what your doing with iter tools is right, you will have to cast it to list.
from itertools import chain
cars = Cars.objects.all()
trucks = Truck.objects.all()
all_vechiles = chain( list(cars), list(trucks) )
source here:http://mushfiq.me/2013/08/04/django-merging-to-queryset-using-itertools/