I am trying to download multiple files as a zip in Django. It works well when file sizes are small. If the file size is more than 1GB, it's showing memory error:
Traceback (most recent call last):
x-xxx-xx: File "/usr/lib64/python3.7/zipfile.py", line 1764, in write
x-xxx-xx: shutil.copyfileobj(src, dest, 1024*8)
x-xxx-xx: File "/usr/lib64/python3.7/shutil.py", line 82, in copyfileobj
x-xxx-xx: fdst.write(buf)
x-xxx-xx: File "/usr/lib64/python3.7/zipfile.py", line 1131, in write
x-xxx-xx: self._fileobj.write(data)
x-xxx-xx: MemoryError
x-xxx-xx: During handling of the above exception, another exception occurred:
x-xxx-xx: Traceback (most recent call last):
x-xxx-xx: File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/django/core/handlers/exception.py", line 34, in inner
x-xxx-xx: response = get_response(request)
x-xxx-xx: File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/django/core/handlers/base.py", line 115, in _get_response
x-xxx-xx: response = self.process_exception_by_middleware(e, request)
x-xxx-xx: File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/django/core/handlers/base.py", line 113, in _get_response
x-xxx-xx: response = wrapped_callback(request, *callback_args, **callback_kwargs)
x-xxx-xx: File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/django/utils/decorators.py", line 130, in _wrapped_view
x-xxx-xx: response = view_func(request, *args, **kwargs)
x-xxx-xx: File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/django/views/decorators/cache.py", line 44, in _wrapped_view_func
x-xxx-xx: response = view_func(request, *args, **kwargs)
x-xxx-xx: File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/django/contrib/admin/sites.py", line 231, in inner
x-xxx-xx: return view(request, *args, **kwargs)
x-xxx-xx: File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/django/views/decorators/cache.py", line 31, in _cache_controlled
x-xxx-xx: response = viewfunc(request, *args, **kw)
x-xxx-xx: File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/django/contrib/auth/decorators.py", line 21, in _wrapped_view
x-xxx-xx: return view_func(request, *args, **kwargs)
x-xxx-xx: File "/var/app/current/fileupload/views.py", line 519, in downloadScanView
x-xxx-xx: zf.write(tmp.name, zip_path)
x-xxx-xx: File "/usr/lib64/python3.7/zipfile.py", line 1764, in write
x-xxx-xx: shutil.copyfileobj(src, dest, 1024*8)
x-xxx-xx: File "/usr/lib64/python3.7/zipfile.py", line 1143, in close
x-xxx-xx: self._fileobj.write(buf)
x-xxx-xx: ValueError: I/O operation on closed file.
This Django application is up and running in AWS. I understand that the problem arises because of insufficient memory allocation. I am using the following code to serve the files. Is it possible to pass the file content as a chunk to utilize the memory more efficiently? Any help would be appreciated.
zip_subdir = url.split('/')[-1]
zip_filename = zip_subdir + ".zip"
byte_stream = BytesIO()
zf = ZipFile(byte_stream, "w", zipfile.ZIP_DEFLATED)
for path in s3_file_path:
url = s3Client.generate_presigned_url('get_object', Params = {'Bucket': BUCKET_NAME, 'Key': path.key}, ExpiresIn = 100)
file_response = requests.get(url)
if file_response.status_code == 200:
try:
tmp = tempfile.NamedTemporaryFile()
tmp.name = path.key.split('/')[-1]
f1 = open(tmp.name, 'wb')
f1.write(file_response.content)
f1.close()
zip_path = os.path.join('/'.join(path.key.split('/')[1:-1]), tmp.name)
zf.write(tmp.name, zip_path)
finally:
os.remove(tmp.name)
zf.close()
response = HttpResponse(byte_stream.getvalue(), content_type="application/x-zip-compressed")
response['Content-Disposition'] = 'attachment; filename=%s' % zip_filename
return response
Related
i have a csv file below which works fine:
Test Case ID,summary
TC-16610,“verify that user is able to u_pdate 'active' attribute 'false ' on adding “new category records” using 'v3/definition/categories' PUT API on specifying the 'active' attribute 'true'”
TC-16609,“verify that user is able to u_pdate 'active' attribute 'true ' on adding “new category records” using 'v3/definition/categories' PUT API on specifying the 'active' attribute 'false'”
But if i add single comma it fails to parse:
Test Case ID,summary
,
TC-16610,“verify that user is able to u_pdate 'active' attribute 'false ' on adding “new category records” using 'v3/definition/categories' PUT API on specifying the 'active' attribute 'true'”
TC-16609,“verify that user is able to u_pdate 'active' attribute 'true ' on adding “new category records” using 'v3/definition/categories' PUT API on specifying the 'active' attribute 'false'”
i want to parse csv file even while having single comma in it. either it should skip and parse or validate to parse. can anyone help me with this.
My code... i am using this in django :
class CsvUpload(forms.Form):
csv_file = forms.FileField()
def clean_csv_file(self):
# Probably worth doing this check first anyway
value = self.cleaned_data['csv_file']
if not value.name.endswith('.csv'):
raise forms.ValidationError('Invalid file type')
try:
data = pd.read_csv(value.file, encoding = 'ISO-8859-1', engine='python')
data.columns= data.columns.str.strip().str.lower()
data=data.rename(columns = {'test case id':'Test Case ID'})
def transform(df):
my_new_string = re.sub('[^a-zA-Z0-9"''-_“” \n\.]', '', df)
return my_new_string
data['summary'] = data['summary'].apply(transform)
except KeyError:
raise forms.ValidationError(
'CSV file must have "summary" column and "Issue Key" column')
except Exception as e:
print('Error while parsing CSV file=> %s', e)
raise forms.ValidationError('Failed to parse the CSV file')
return data
EDIT: after parsing the file it leads to an exception 'Failed to parse the CSV file'and if i remove the last exception it leads to
Traceback (most recent call last):
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\core\handlers\exception.py", line 34, in inner
response = get_response(request)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\core\handlers\base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\core\handlers\base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\utils\decorators.py", line 130, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\views\decorators\cache.py", line 44, in _wrapped_view_func
response = view_func(request, *args, **kwargs)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\contrib\admin\sites.py", line 231, in inner
return view(request, *args, **kwargs)
File "C:\Users\ssuri\myproject\elixirdev\test_suite_optimizer\admin.py", line 282, in process_csv
if form.is_valid():
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\forms\forms.py", line 180, in is_valid
return self.is_bound and not self.errors
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\forms\forms.py", line 175, in errors
self.full_clean()
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\forms\forms.py", line 376, in full_clean
self._clean_fields()
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\forms\forms.py", line 397, in _clean_fields
value = getattr(self, 'clean_%s' % name)()
File "C:\Users\ssuri\myproject\elixirdev\test_suite_optimizer\forms.py", line 78, in clean_csv_file
data['summary'] = data['summary'].apply(transform)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\series.py", line 4045, in apply
mapped = lib.map_infer(values, f, convert=convert_dtype)
File "pandas/_libs/lib.pyx", line 2228, in pandas._libs.lib.map_infer
File "C:\Users\ssuri\myproject\elixirdev\test_suite_optimizer\forms.py", line 74, in transform
my_new_string = re.sub('[^a-zA-Z0-9"''-_“” \n\.]', '', df)
File "c:\users\ssuri\appdata\local\programs\python\python38\lib\re.py", line 210, in sub
return _compile(pattern, flags).sub(repl, string, count)
TypeError: expected string or bytes-like object
After using:
data = data.dropna()
it leads to:
Traceback (most recent call last):
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\indexes\base.py", line 2897, in get_loc
return self._engine.get_loc(key)
File "pandas/_libs/index.pyx", line 107, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 131, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 992, in pandas._libs.hashtable.Int64HashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 998, in pandas._libs.hashtable.Int64HashTable.get_item
KeyError: 0
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\core\handlers\exception.py", line 34, in inner
response = get_response(request)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\core\handlers\base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\core\handlers\base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\utils\decorators.py", line 130, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\views\decorators\cache.py", line 44, in _wrapped_view_func
response = view_func(request, *args, **kwargs)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\django\contrib\admin\sites.py", line 231, in inner
return view(request, *args, **kwargs)
File "C:\Users\ssuri\myproject\elixirdev\test_suite_optimizer\admin.py", line 287, in process_csv
data = handle_demo_ai(csv_data)
File "C:\Users\ssuri\myproject\elixirdev\test_suite_optimizer\ai\web_demo_test_suite_optimization.py", line 108, in handle_json_ai
X = df_tcs.loc[X_row, "Summary_lemmatized"]
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\indexing.py", line 1418, in __getitem__
return self._getitem_tuple(key)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\indexing.py", line 805, in _getitem_tuple
return self._getitem_lowerdim(tup)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\indexing.py", line 929, in _getitem_lowerdim
section = self._getitem_axis(key, axis=i)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\indexing.py", line 1850, in _getitem_axis
return self._get_label(key, axis=axis)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\indexing.py", line 160, in _get_label
return self.obj._xs(label, axis=axis)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\generic.py", line 3737, in xs
loc = self.index.get_loc(key)
File "C:\Users\ssuri\Envs\elixir\lib\site-packages\pandas\core\indexes\base.py", line 2899, in get_loc
return self._engine.get_loc(self._maybe_cast_indexer(key))
File "pandas/_libs/index.pyx", line 107, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 131, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 992, in pandas._libs.hashtable.Int64HashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 998, in pandas._libs.hashtable.Int64HashTable.get_item
KeyError: 0
When you read the file with the comma, first row has NAN values.
You need to drop NAN values before you try to apply transform()
use
data = data.dropna()
I have been trying to get this theme working for Open edX (based on Django) and I am getting this error in supervisor logs -
AttributeError: 'CourseOverview' object has no attribute 'start_datetime_text'
The whole error is this -
May 14 11:43:36 ip-172-26-15-154 [service_variant=lms][django.request][env:sandbox] ERROR [ip-172-26-15-154 20450] [base.py:256] - Internal Server Error: /
Traceback (most recent call last):
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/django/core/handlers/base.py", line 132, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/django/utils/decorators.py", line 145, in inner
return func(*args, **kwargs)
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/django/utils/decorators.py", line 110, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "/edx/app/edxapp/edx-platform/common/djangoapps/util/cache.py", line 78, in wrapper
response = view_func(request, *args, **kwargs)
File "/edx/app/edxapp/edx-platform/lms/djangoapps/branding/views.py", line 94, in index
return student.views.index(request, user=request.user)
File "/edx/app/edxapp/edx-platform/common/djangoapps/student/views.py", line 221, in index
return render_to_response('index.html', context)
File "/edx/app/edxapp/edx-platform/common/djangoapps/edxmako/shortcuts.py", line 198, in render_to_response
return HttpResponse(render_to_string(template_name, dictionary, context_instance, namespace, request), **kwargs)
File "/edx/app/edxapp/edx-platform/common/djangoapps/edxmako/shortcuts.py", line 188, in render_to_string
return template.render_unicode(**context_dictionary)
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/mako/template.py", line 454, in render_unicode
as_unicode=True)
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/mako/runtime.py", line 829, in _render
**_kwargs_for_callable(callable_, data))
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/mako/runtime.py", line 864, in _render_context
_exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/mako/runtime.py", line 890, in _exec_template
callable_(context, *args, **kwargs)
File "/tmp/mako_lms/c11f9c5f254718c770fcf021e95ac093/main.html.py", line 286, in render_body
__M_writer(filters.decode.utf8(self.body()))
File "/tmp/mako_lms/c11f9c5f254718c770fcf021e95ac093/marvel-theme-eucalyptus/lms/templates/index.html.py", line 54, in render_body
runtime._include_file(context, (courses_list), _template_uri)
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/mako/runtime.py", line 752, in _include_file
callable_(ctx, **_kwargs_for_include(callable_, context._data, **kwargs))
File "/tmp/mako_lms/c11f9c5f254718c770fcf021e95ac093/marvel-theme-eucalyptus/lms/templates/courses_list.html.py", line 44, in render_body
runtime._include_file(context, u'course.html', _template_uri, course=course)
File "/edx/app/edxapp/venvs/edxapp/local/lib/python2.7/site-packages/mako/runtime.py", line 752, in _include_file
callable_(ctx, **_kwargs_for_include(callable_, context._data, **kwargs))
File "/tmp/mako_lms/c11f9c5f254718c770fcf021e95ac093/marvel-theme-eucalyptus/lms/templates/course.html.py", line 60, in render_body
__M_writer(filters.html_escape(filters.decode.utf8(course.start_datetime_text())))
AttributeError: 'CourseOverview' object has no attribute 'start_datetime_text'
The file in question (the one that is giving the error - course.html.py) is here
https://pastebin.com/0S79hhDa
The start_datetime_text method was removed on november 30, 2016.
You should be using course.advertised_start. You can see the current version of the course.html template
I am trying to update a self datetime field of a model using F() object in django.
HeatWatchList.objects.filter(
user=request.user,
next_date_to__lt=datetime.combine(datetime.now().date(), time.min)
).update(
next_date_from = F('next_date_from') + relativedelta(months=1),
next_date_to = F('next_date_to') + relativedelta(months=1)
)
But when doing this i got AttributeError: relativedelta object has no attribute translate.
Here is the traceback i got when executing the code maybe it has an issue using the F() object when adding value with a datetime type relativedelta.
Traceback (most recent call last):
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\core\handlers\exception.py", line 39, in inner
response = get_response(request)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\core\handlers\base.py", line 187, in _get_response
response = self.process_exception_by_middleware(e, request)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\core\handlers\base.py", line 185, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\views\decorators\csrf.py", line 58, in wrapped_view
return view_func(*args, **kwargs)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\rest_framework\viewsets.py", line 83, in view
return self.dispatch(request, *args, **kwargs)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\rest_framework\views.py", line 477, in dispatch
response = self.handle_exception(exc)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\rest_framework\views.py", line 437, in handle_exception
self.raise_uncaught_exception(exc)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\rest_framework\views.py", line 474, in dispatch
response = handler(request, *args, **kwargs)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\farm_management\heat\views.py", line 192, in update_watchlist
next_date_to = F('next_date_to') + relativedelta(months=1)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\db\models\query.py", line 637, in update
rows = query.get_compiler(self.db).execute_sql(CURSOR)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\db\models\sql\compiler.py", line 1148, in execute_sql
cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\db\models\sql\compiler.py", line 835, in execute_sql
cursor.execute(sql, params)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\db\backends\utils.py", line 79, in execute
return super(CursorDebugWrapper, self).execute(sql, params)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\db\backends\utils.py", line 64, in execute
return self.cursor.execute(sql, params)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\django\db\backends\mysql\base.py", line 110, in execute
return self.cursor.execute(query, args)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\cursors.py", line 164, in execute
query = self.mogrify(query, args)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\cursors.py", line 143, in mogrify
query = query % self._escape_args(args, conn)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\cursors.py", line 118, in _escape_args
return tuple(conn.literal(arg) for arg in args)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\cursors.py", line 118, in <genexpr>
return tuple(conn.literal(arg) for arg in args)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\connections.py", line 800, in literal
return self.escape(obj, self.encoders)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\connections.py", line 793, in escape
return escape_item(obj, self.charset, mapping=mapping)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\converters.py", line 27, in escape_item
val = encoder(val, mapping)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\converters.py", line 110, in escape_unicode
return u"'%s'" % _escape_unicode(value)
File "C:\Users\Web\Desktop\PyDev\Envs\djangular\lib\site-packages\pymysql\converters.py", line 73, in _escape_unicode
return value.translate(_escape_table)
AttributeError: 'relativedelta' object has no attribute 'translate'
Seems like pymysql's converters does not support relativedelta objects.
One possible solution is to iterate over objects and update it (inefficient):
objects_list = HeatWatchList.objects.filter(
user=request.user,
next_date_to__lt=datetime.combine(datetime.now().date(), time.min)
)
for obj in object_list:
obj.next_date_from = onj.next_date_from + relativedelta(months=1)
obj.next_date_to = onj.next_date_to + relativedelta(months=1)
objsave()
Or better use datetime.timedelta instead of relativedelta if it possible:
HeatWatchList.objects.filter(
user=request.user,
next_date_to__lt=datetime.combine(datetime.now().date(), time.min)
).update(
next_date_from = F('next_date_from') + timedelta(days=31),
next_date_to = F('next_date_to') + timedelta(days=31)
)
I have custom pipeline in my django project. Following this example I'm trying to get access token (to get data from google plus) but it returns me TypeError: string indices must be integers.
Custom pipeline:
def user_details(strategy, details, response, user=None, *args, **kwargs):
....
if user:
if kwargs['is_new']:
if "google" in kwargs['backend'].redirect_uri:
...
try:
social = user.social_auth.get(provider='google-plus')
access_token = logger.warning(social.extra_data['access_token'])
....
except HTTPError:
pass
.....
Full traceback:
Traceback (most recent call last):
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/django/core/handlers/base.py", line 149, in get_response
response = self.process_exception_by_middleware(e, request)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/django/core/handlers/base.py", line 147, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/django/views/decorators/cache.py", line 57, in _wrapped_view_func
response = view_func(request, *args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
return view_func(*args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/apps/django_app/utils.py", line 51, in wrapper
return func(request, backend, *args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/apps/django_app/views.py", line 28, in complete
redirect_name=REDIRECT_FIELD_NAME, *args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/actions.py", line 43, in do_complete
user = backend.complete(user=user, *args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/backends/base.py", line 41, in complete
return self.auth_complete(*args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/utils.py", line 229, in wrapper
return func(*args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/backends/google.py", line 158, in auth_complete
*args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/utils.py", line 229, in wrapper
return func(*args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/backends/oauth.py", line 398, in do_auth
return self.strategy.authenticate(*args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/strategies/django_strategy.py", line 96, in authenticate
return authenticate(*args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/django/contrib/auth/__init__.py", line 74, in authenticate
user = backend.authenticate(**credentials)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/backends/base.py", line 82, in authenticate
return self.pipeline(pipeline, *args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/backends/base.py", line 85, in pipeline
out = self.run_pipeline(pipeline, pipeline_index, *args, **kwargs)
File "/home/linevich/.virtualenvs/tathros/lib/python3.4/site-packages/social/backends/base.py", line 112, in run_pipeline
result = func(*args, **out) or {}
File "/home/linevich/projects/tathros/proto/pipeline.py", line 54, in user_details
auth_token = social.extra_data['access_token']
TypeError: string indices must be integers
The solutuion was simple:
auth_token = json.loads(social.extra_data)['access_token']
I'm using django-rq, the django bindings for python-rq, to try generate a PDF asynchronously. The class TemplateProcesser initializes with two arguments and automatically generates the PDF in the __init__ function. This works fine synchronously, outside of django-rq, but with django-rq it fails with this error:
Error:
AttributeError: type object 'TemplateProcesser' has no attribute 'rsplit'
From this call:
django_rq.enqueue(TemplateProcesser, nail_order=serializer.object, user_photo=base64_image)
Any idea on how to correctly include the uninstantiated class in django-rq?
Class:
class TemplateProcesser(object):
def __init__(self, nail_order, user_photo, *args, **kwargs):
self.nail_order = nail_order
self.user_photo = user_photo
...
self.procces_template()
...
StackTrace:
Traceback (most recent call last):
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/worker.py", line 426, in perform_job
rv = job.perform()
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 386, in perform
self._result = self.func(*self.args, **self.kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 154, in func
return import_attribute(self.func_name)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/utils.py", line 168, in import_attribute
module_name, attribute = name.rsplit('.', 1)
AttributeError: type object 'TemplateProcesser' has no attribute 'rsplit'
Traceback after metaperture's answer:
Traceback (most recent call last):
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/core/handlers/base.py", line 114, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/views/generic/base.py", line 69, in view
return self.dispatch(request, *args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/views/decorators/csrf.py", line 57, in wrapped_view
return view_func(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rest_framework/views.py", line 400, in dispatch
response = self.handle_exception(exc)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rest_framework/views.py", line 397, in dispatch
response = handler(request, *args, **kwargs)
File "/Users/admin/dev/ncla-web/ncla/api/views.py", line 91, in post
django_rq.enqueue(self.template_processor_factory, **parameter_dict)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 162, in enqueue
return get_queue().enqueue(func, *args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 213, in enqueue
description=description, depends_on=depends_on)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 42, in enqueue_call
return self.original_enqueue_call(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 37, in original_enqueue_call
return super(DjangoRQ, self).enqueue_call(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 176, in enqueue_call
return self.enqueue_job(job)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 232, in enqueue_job
job.save()
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 360, in save
connection.hmset(key, self.dump())
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 329, in dump
obj['data'] = dumps(self.job_tuple)
File "/Users/admin/dev/ncla-web/env/bin/../lib/python2.7/copy_reg.py", line 70, in _reduce_ex
raise TypeError, "can't pickle %s objects" % base.__name__
TypeError: can't pickle BytesIO objects
Looks like python-rq expects a function as the first argument. I would try:
def template_processor_factory(*args, **kwargs):
return TemplateProcessor(*args, **kwargs)
django_rq.enqueue(template_processer_factory, nail_order=serializer.object,
user_photo=base64_image)