I have unittest for api use Marshmallow in flask 1.1.2 and python 3.8 as bellow
schema.py
class ListSchema(Schema):
id = fields.Integer()
name = fields.String()
gone_date = fields.Function(lambda data: data.gone_date.timestamp())
main_service.py
class MainService:
#classmethod:
def get_list(cls):
list_schema = ListSchema()
list_data, total = Class.function_get_list_data
list_data = list_schema.dump(list_data, many=True)
When i run pytest for function get_list
mock = [(1, 'Name_1', datetime.datetime(2020, 8, 20, 0, 0))], 10
mock_data = mock.Mock(return_value=mock)
with mock.patch.object(Class, 'function_get_list_data', mock_data):
response = MainService.get_list()
I alway get error add lambda function in schema
AttributeError: 'tuple' object has no attribute 'gone_date'
How to pass or test via function lambda in this case. I have try catch with raise unittest.skip but it not my expectation. My function still work normally, only error when apply unittest. Thanks for help.
Edit:
This trace back:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/unittest/case.py", line 60, in testPartExecutor
yield
File "/usr/local/lib/python3.8/unittest/case.py", line 676, in run
self._callTestMethod(testMethod)
File "/usr/local/lib/python3.8/unittest/case.py", line 633, in _callTestMethod
method()
File "/opt/project/app/test/service/test_main_service.py", line 79, in test_process_list
response = MainService.get_list()
File "/opt/project/app/main/service/main_service.py", line 18, in process_main_list
list_data = list_schema.dump(list_data, many=True)
File "/usr/local/lib/python3.8/site-packages/marshmallow/schema.py", line 557, in dump
result = self._serialize(processed_obj, many=many)
File "/usr/local/lib/python3.8/site-packages/marshmallow/schema.py", line 515, in _serialize
return [
File "/usr/local/lib/python3.8/site-packages/marshmallow/schema.py", line 516, in <listcomp>
self._serialize(d, many=False)
File "/usr/local/lib/python3.8/site-packages/marshmallow/schema.py", line 521, in _serialize
value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)
File "/usr/local/lib/python3.8/site-packages/marshmallow/fields.py", line 312, in serialize
return self._serialize(value, attr, obj, **kwargs)
File "/usr/local/lib/python3.8/site-packages/marshmallow/fields.py", line 1722, in _serialize
return self._call_or_raise(self.serialize_func, obj, attr)
File "/usr/local/lib/python3.8/site-packages/marshmallow/fields.py", line 1736, in _call_or_raise
return func(value)
File "/opt/project/app/main/schemas/schema.py", line 13, in <lambda>
gone_date = fields.Function(lambda data: data.gone_date.timestamp())
Related
My locust file has put requests but sometimes it's passing sometimes failing. Can anyone explain why that's happening?
Here is my locust file
def _generate_put_data(self) -> str:
if len(self.ids) > 0:
teacher_name = self.generate_random_string()
teacher_email = f"{self.generate_random_string()}#{self.generate_random_string()}.{self.generate_random_string()}"
teacher_email = teacher_email.replace("#", "%40")
teacher_id = str(random.choice(self.ids))
request_string = f"{self.path_all}/{teacher_id}?teacherName={teacher_name}&teacherEmail={teacher_email}"
return request_string
#task(2)
def put_request(self):
self.client.put(url=self._generate_put_data())
...
_generate_put_data method returns a string, which is a query.
Here is the error:
[2023-01-26 11:39:12,357] pop-os/ERROR/locust.user.task: expected string or bytes-like object
Traceback (most recent call last):
File "/home/XXX/.local/lib/python3.10/site-packages/locust/user/task.py", line 347, in run
self.execute_next_task()
File "/home/XXX/.local/lib/python3.10/site-packages/locust/user/task.py", line 372, in execute_next_task
self.execute_task(self._task_queue.pop(0))
File "/home/XXX/.local/lib/python3.10/site-packages/locust/user/task.py", line 493, in execute_task
task(self.user)
File "/home/XXX/Desktop/my-projects/spring-boot-app/performans-testing/locust.py", line 44, in put_request
self.client.put(url=self._generate_put_data())
File "/home/XXX/.local/lib/python3.10/site-packages/requests/sessions.py", line 647, in put
return self.request("PUT", url, data=data, **kwargs)
File "/home/XXX/.local/lib/python3.10/site-packages/locust/clients.py", line 131, in request
url = self._build_url(url)
File "/home/XXX/.local/lib/python3.10/site-packages/locust/clients.py", line 81, in _build_url
if absolute_http_url_regexp.match(path):
TypeError: expected string or bytes-like object
I am using google-api-python-client for inserting a json record to bigquery and when I try to unittest the method using python unittest, I am getting error in exactly this line
The code is as follows:
def write_to_bigquery(self, timeseries, metadata):
response = {}
json_msg_list = []
stats = {}
if not timeseries or "points" not in timeseries:
logging.debug("No timeseries data to write to BigQuery")
msgs_written = 0
metadata["msg_without_timeseries"] = 1
error_msg_cnt = 0
else:
rows = build_rows(timeseries, metadata)
print("rows", rows) //This gets printed
bigquery = build('bigquery', 'v2', cache_discovery=False)
print("after rows", rows) //Control does not reach here
body = {
"kind": "bigquery#tableDataInsertAllRequest",
"skipInvalidRows": "false",
"rows": json_row_list
}
logging.debug('body: {}'.format(json.dumps(body, sort_keys=True, indent=4)))
response = bigquery.tabledata().insertAll(
projectId=app_identity.get_application_id(),
datasetId=config.BIGQUERY_DATASET,
tableId=config.BIGQUERY_STATS_TABLE,
body=body
).execute()
logging.debug("BigQuery said... = {}".format(response))
and this is the error I get
Traceback (most recent call last):
File "/home/barumugham/.local/lib/python2.7/site-packages/webapp2.py", line 1535, in __call__
rv = self.handle_exception(request, response, e)
File "/home/barumugham/.local/lib/python2.7/site-packages/webapp2.py", line 1529, in __call__
rv = self.router.dispatch(request, response)
File "/home/barumugham/.local/lib/python2.7/site-packages/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "/home/barumugham/.local/lib/python2.7/site-packages/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "/home/barumugham/.local/lib/python2.7/site-packages/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "/home/barumugham/.local/lib/python2.7/site-packages/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "main.py", line 422, in post
File "/home/barumugham/.local/lib/python2.7/site-packages/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "main.py", line 422, in post
self.write_to_bigquery(data, metadata)
File "main.py", line 296, in write_to_bigquery
bigquery = build('bigquery', 'v2', cache_discovery=False)
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 258, in build
adc_key_path=adc_key_path,
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 423, in build_from_document
credentials = _auth.default_credentials()
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_auth.py", line 44, in default_credentials
credentials, project_id = checker()
File "/usr/local/lib/python2.7/dist-packages/google/auth/_default.py", line 186, in _get_gae_credentials
project_id = app_engine.get_project_id()
File "/usr/local/lib/python2.7/dist-packages/google/auth/app_engine.py", line 77, in get_project_id
return app_identity.get_application_id()
File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/api/app_identity/app_identity.py", line 455, in get_application_id
_, domain_name, display_app_id = _ParseFullAppId(full_app_id)
File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/api/app_identity/app_identity.py", line 436, in _ParseFullAppId
psep = app_id.find(_PARTITION_SEPARATOR)
AttributeError: 'NoneType' object has no attribute 'find'
I am new to python and bigquery so any help is appreciated thanks
I would recommend you using the BigQuery Python SDK
For that, you first need to install it in you Python. You can do that by running:
pip install google-cloud-bigquery
After that you use a code like this insert json records to your table:
from google.cloud import bigquery
# Construct a BigQuery client object.
client = bigquery.Client()
table_id = "project_id.dataset.table"
# Your JSON keys must correspond to your table column names
json_list = [{"your": "json", "data":"here"},{"your": "json", "data":"here"},{"your": "json", "data":"here"}, ...]
# Get table reference
table = client.get_table(table_id)
rows_to_insert = json_list
# Insert the data into your table
errors = client.insert_rows(table, rows_to_insert)
Finally, I'd like to say that Python 2 is considered deprecated already. If possible, update it to Python 3
Trying to perform an UPSERT on an array of values to a PostgreSQL > 9.5.
Trying to build the statement as in the SQLALchemy docs, but there is no explanation about how to do it for an array instead of a single row. The insert statement builds properly so I suppose it's possible to do so with the on_conflict_do_update function.
Having this code :
stock_table = Table("stock_history", metadata,
Column('date', sqlalchemy.types.NVARCHAR(length=255), primary_key=True),
Column('product_id', sqlalchemy.types.INTEGER(), primary_key=True),
Column('product_sku', sqlalchemy.types.NVARCHAR(length=255)),
Column('on_hand_qty', sqlalchemy.dialects.postgresql.DOUBLE_PRECISION()),
Column('available_qty', sqlalchemy.dialects.postgresql.DOUBLE_PRECISION()),
Column('output_qty', sqlalchemy.dialects.postgresql.DOUBLE_PRECISION())
)
stock_today = pandas.read_sql_query(queryStock, odoo_engine)
insert_stmt = sqlalchemy.dialects.postgresql.insert(stock_table).values(stock_today)
upser_stmt = insert_stmt.on_conflict_do_update(
index_elements=['date', 'product_id'],
set_=stock_today.to_dict(orient='dict')
)
I'm getting the following error:
AttributeError: 'StrSQLCompiler' object has no attribute 'visit_on_conflict_do_update'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "pompeiiETL.py", line 15, in <module>
pompeiiJobs.runStockJob(dwh_engine, odoo_prod_engine)
File "/Users/alex/Development/DataLab/pompeii-datalab/pompeiiETL/jobs.py", line 54, in runStockJob
print(upser_stmt)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 446, in __str__
return str(self.compile())
File "<string>", line 1, in <lambda>
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 436, in compile
return self._compiler(dialect, bind=bind, **kw)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 442, in _compiler
return dialect.statement_compiler(dialect, self, **kw)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 435, in __init__
Compiled.__init__(self, dialect, statement, **kwargs)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 216, in __init__
self.string = self.process(self.statement, **compile_kwargs)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 242, in process
return obj._compiler_dispatch(self, **kwargs)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 2041, in visit_insert
insert_stmt._post_values_clause, **kw)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 242, in process
return obj._compiler_dispatch(self, **kwargs)
File "/Users/alex/Development/DataLab/ETLenv/lib/python3.6/site-packages/sqlalchemy/sql/visitors.py", line 79, in _compiler_dispatch
raise exc.UnsupportedCompilationError(visitor, cls)
sqlalchemy.exc.UnsupportedCompilationError: Compiler <sqlalchemy.sql.compiler.StrSQLCompiler object at 0x105b55be0> can't render element of type <class 'sqlalchemy.dialects.postgresql.dml.OnConflictDoUpdate'>
What I'm ding wrong? Is there a better way to do an upsert?
Thanks!
You are trying to get string representation of Insert-object that has no proper bind, if we write
db_uri = make_url('your-postgres-db-uri-here')
engine = create_engine(db_uri)
upser_stmt.bind = engine
print(upser_stmt)
it works
We can also create insert statement with bind specified
insert_stmt = sqlalchemy.dialects.postgresql.insert(stock_table,
bind=engine).values(stock_today)
upser_stmt = insert_stmt.on_conflict_do_update(
index_elements=['date', 'product_id'],
set_=stock_today.to_dict(orient='dict')
)
I'm using django-rq, the django bindings for python-rq, to try generate a PDF asynchronously. The class TemplateProcesser initializes with two arguments and automatically generates the PDF in the __init__ function. This works fine synchronously, outside of django-rq, but with django-rq it fails with this error:
Error:
AttributeError: type object 'TemplateProcesser' has no attribute 'rsplit'
From this call:
django_rq.enqueue(TemplateProcesser, nail_order=serializer.object, user_photo=base64_image)
Any idea on how to correctly include the uninstantiated class in django-rq?
Class:
class TemplateProcesser(object):
def __init__(self, nail_order, user_photo, *args, **kwargs):
self.nail_order = nail_order
self.user_photo = user_photo
...
self.procces_template()
...
StackTrace:
Traceback (most recent call last):
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/worker.py", line 426, in perform_job
rv = job.perform()
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 386, in perform
self._result = self.func(*self.args, **self.kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 154, in func
return import_attribute(self.func_name)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/utils.py", line 168, in import_attribute
module_name, attribute = name.rsplit('.', 1)
AttributeError: type object 'TemplateProcesser' has no attribute 'rsplit'
Traceback after metaperture's answer:
Traceback (most recent call last):
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/core/handlers/base.py", line 114, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/views/generic/base.py", line 69, in view
return self.dispatch(request, *args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/views/decorators/csrf.py", line 57, in wrapped_view
return view_func(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rest_framework/views.py", line 400, in dispatch
response = self.handle_exception(exc)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rest_framework/views.py", line 397, in dispatch
response = handler(request, *args, **kwargs)
File "/Users/admin/dev/ncla-web/ncla/api/views.py", line 91, in post
django_rq.enqueue(self.template_processor_factory, **parameter_dict)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 162, in enqueue
return get_queue().enqueue(func, *args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 213, in enqueue
description=description, depends_on=depends_on)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 42, in enqueue_call
return self.original_enqueue_call(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 37, in original_enqueue_call
return super(DjangoRQ, self).enqueue_call(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 176, in enqueue_call
return self.enqueue_job(job)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 232, in enqueue_job
job.save()
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 360, in save
connection.hmset(key, self.dump())
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 329, in dump
obj['data'] = dumps(self.job_tuple)
File "/Users/admin/dev/ncla-web/env/bin/../lib/python2.7/copy_reg.py", line 70, in _reduce_ex
raise TypeError, "can't pickle %s objects" % base.__name__
TypeError: can't pickle BytesIO objects
Looks like python-rq expects a function as the first argument. I would try:
def template_processor_factory(*args, **kwargs):
return TemplateProcessor(*args, **kwargs)
django_rq.enqueue(template_processer_factory, nail_order=serializer.object,
user_photo=base64_image)
When I use fetch_async() on a query it crashes with AssertionError the first time it is run. If I run it again immediately, it's fine.
Eg.
With model:
class User(ndb.Model):
user = ndb.UserProperty()
name = ndb.StringProperty()
penname = ndb.StringProperty()
first_login = ndb.DateTimeProperty(auto_now_add=True)
contact = ndb.BooleanProperty()
The following works straight away, returning an empty list:
users = User.query(User.penname == "asdf").fetch()
But this crashes:
future = User.query(User.penname == "asdf").fetch_async()
users = future.get_result()
With:
Traceback (most recent call last):
File "/opt/google-appengine-python/google/appengine/ext/admin/__init__.py", line 320, in post
exec(compiled_code, globals())
File "<string>", line 6, in <module>
File "/opt/google-appengine-python/google/appengine/ext/ndb/tasklets.py", line 320, in get_result
self.check_success()
File "/opt/google-appengine-python/google/appengine/ext/ndb/tasklets.py", line 357, in _help_tasklet_along
value = gen.throw(exc.__class__, exc, tb)
File "/opt/google-appengine-python/google/appengine/ext/ndb/query.py", line 887, in _run_to_list
batch = yield rpc
File "/opt/google-appengine-python/google/appengine/ext/ndb/tasklets.py", line 435, in _on_rpc_completion
result = rpc.get_result()
File "/opt/google-appengine-python/google/appengine/api/apiproxy_stub_map.py", line 592, in get_result
return self.__get_result_hook(self)
File "/opt/google-appengine-python/google/appengine/datastore/datastore_query.py", line 2386, in __query_result_hook
self._batch_shared.conn.check_rpc_success(rpc)
File "/opt/google-appengine-python/google/appengine/datastore/datastore_rpc.py", line 1191, in check_rpc_success
rpc.check_success()
File "/opt/google-appengine-python/google/appengine/api/apiproxy_stub_map.py", line 558, in check_success
self.__rpc.CheckSuccess()
File "/opt/google-appengine-python/google/appengine/api/apiproxy_rpc.py", line 156, in _WaitImpl
self.request, self.response)
File "/opt/google-appengine-python/google/appengine/api/datastore_file_stub.py", line 568, in MakeSyncCall
response)
File "/opt/google-appengine-python/google/appengine/api/apiproxy_stub.py", line 87, in MakeSyncCall
method(request, response)
File "/opt/google-appengine-python/google/appengine/datastore/datastore_stub_util.py", line 2367, in UpdateIndexesWrapper
self._UpdateIndexes()
File "/opt/google-appengine-python/google/appengine/datastore/datastore_stub_util.py", line 2656, in _UpdateIndexes
self._index_yaml_updater.UpdateIndexYaml()
File "/opt/google-appengine-python/google/appengine/datastore/datastore_stub_index.py", line 244, in UpdateIndexYaml
all_indexes, manual_indexes)
File "/opt/google-appengine-python/google/appengine/datastore/datastore_stub_index.py", line 85, in GenerateIndexFromHistory
required, kind, ancestor, props, num_eq_filters = datastore_index.CompositeIndexForQuery(query)
File "/opt/google-appengine-python/google/appengine/datastore/datastore_index.py", line 424, in CompositeIndexForQuery
assert filter.property(0).name() == ineq_property
AssertionError
But if I run it again immediately, for example by doing:
future = User.query(User.penname == "asdf").fetch_async()
try:
users = future.get_result()
except:
users = future.get_result()
It works.
This has been cropping up all over the place and I'm struggling to pin down the root cause.
The solution was to upgrade to the 1.7.1 SDK.