I have a program which is eventually deployed on Google Cloud Platform and I use the Logs Explorer with GCP to monitor my logs. For that, I need to use the google.cloud.logging library, which is necessary to show logs with correct severity level on GCP Logs Explorer.
In my unit-tests, I am trying to patch the the call to the google.cloud.logging library, however, they all end up with 403 error on my local, suggesting that the library has not been patched.
cloud_logger.py
import google.cloud.logging
import logging
def get_logger():
client = google.cloud.logging.Client()
client.get_default_handler()
client.setup_logging()
logger = logging.getLogger(__name__)
return logger
rss_crawler.py which utlizes the cloud_logger.get_logger
from cloud_logger import get_logger
logger = get_logger()
def crawl_rss_source(source_crawling_details):
brand_name = source_crawling_details[constants.BRAND_NAME]
source_name = source_crawling_details[constants.SOURCE_NAME]
initial_agent_settings = source_crawling_details[constants.INITIAL_AGENT_SETTINGS]
logger.info(f"Started crawling {brand_name}-{source_name} crawling")
source = source_crawling_details["source"]
entry_points_list = source[constants.ENTRY]
source_crawling_details.update({constants.ENTRY: entry_points_list})
source_crawling_details.update({constants.AGENT: initial_agent_settings})
content = get_content(source_crawling_details)
logger.info("Getting links present in rss feed entry")
entry_points = rss_entry_points(content)
source_crawling_details.update({constants.ENTRY_POINTS: entry_points})
candidate_urls = start_content_tasks(source_crawling_details)
if not candidate_urls:
raise CustomException("There are no links to scrape")
# filtered urls found with crawl rules, next step get scrape candidates based on scrape rules
scrape_rules = source[constants.SCRAPE]
scrape_candidates = get_scrape_candidates(scrape_rules, candidate_urls)
if not scrape_candidates:
raise CustomException(
f"Could not find any links for scraping, please check scrape,crawl rules, or possibly depth level for brand {brand_name} , source {source_name}"
)
return scrape_candidates
test_rss_crawler.py
#patch("start_crawl.fetch_source_crawling_fields")
#patch("rss_crawler.logger")
def test_crawl_rss_source_raises_exception(
self, mocked_logger, mocked_source_fetcher
):
mocked_logger.logger.return_value = logging.getLogger(__name__)
self.test_source[constants.SCRAPE] = {
"white_list": ["https://buffer.com/blog/(\\w|\\d|\\-)+/$"]
}
details = set_content_source_details(
self.brand_name,
self.source_name,
self.agent_args,
self.source,
**self.key_word_argument,
)
# test to see if exception is raised if scrape rule is not matching
self.assertRaises(CustomException, crawl_rss_source, details)
self.test_source[constants.CRAWL] = {"white_list": ["https://buffer.com/blog/"]}
details = set_content_source_details(
self.brand_name,
self.source_name,
self.agent_args,
self.source,
**self.key_word_argument,
)
# test to see if exception is raised if crawl rule is not matching
self.assertRaises(CustomException, crawl_rss_source, details)
However, I get these warning messages, when I run these tests, even after patching:
Traceback (most recent call last):
File "/Users/reydon227/Concured/crawler/env/lib/python3.8/site-packages/google/cloud/logging_v2/handlers/transports/background_thread.py", line 115, in _safely_commit_batch
batch.commit()
File "/Users/reydon227/Concured/crawler/env/lib/python3.8/site-packages/google/cloud/logging_v2/logger.py", line 385, in commit
client.logging_api.write_entries(entries, **kwargs)
File "/Users/reydon227/Concured/crawler/env/lib/python3.8/site-packages/google/cloud/logging_v2/_gapic.py", line 149, in write_entries
self._gapic_api.write_log_entries(request=request)
File "/Users/reydon227/Concured/crawler/env/lib/python3.8/site-packages/google/cloud/logging_v2/services/logging_service_v2/client.py", line 592, in write_log_entries
response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)
File "/Users/reydon227/Concured/crawler/env/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py", line 145, in __call__
return wrapped_func(*args, **kwargs)
File "/Users/reydon227/Concured/crawler/env/lib/python3.8/site-packages/google/api_core/retry.py", line 286, in retry_wrapped_func
return retry_target(
File "/Users/reydon227/Concured/crawler/env/lib/python3.8/site-packages/google/api_core/retry.py", line 189, in retry_target
return target()
File "/Users/reydon227/Concured/crawler/env/lib/python3.8/site-packages/google/api_core/grpc_helpers.py", line 69, in error_remapped_callable
six.raise_from(exceptions.from_grpc_error(exc), exc)
File "<string>", line 3, in raise_from
google.api_core.exceptions.PermissionDenied: 403 The caller does not have permission
not sure about how author handled that, but I had similar symptoms with one of my tests with mocked google things, and I can't see any imports section in test_rss_crawler.py code. In my code I had to import class under test inside test method, because #mock.patch(..) runs before method, but after test file import section, and code in imported module may already initialize class supposed to be patched, so patch has no effect. Mb smbd will find useful that importing modules supposed to be patched inside test method may help.
#mock.patch('some_class')
def test_smth(self, some_class):
import some_class
some_class.return_value = ....
Related
I have a function with #jwt_required decorator.
class Test(Resource):
#jwt_required
def get(self):
return {"test": "ok" }
Which works fine when the correct HTTP header is set, i.e.
Authentication: Bearer [TOKEN]
but when the token is invalid/wrong or messed with, a jwt.exceptions.DecodeError is raised:
File "env/lib/python3.6/site-packages/flask_restplus/resource.py", line 44, in dispatch_request
resp = meth(*args, **kwargs)
File "env/lib/python3.6/site-packages/flask_jwt_extended/view_decorators.py", line 103, in wrapper
verify_jwt_in_request()
File "env/lib/python3.6/site-packages/flask_jwt_extended/view_decorators.py", line 32, in verify_jwt_in_request
jwt_data = _decode_jwt_from_request(request_type='access')
File "env/lib/python3.6/site-packages/flask_jwt_extended/view_decorators.py", line 267, in _decode_jwt_from_request
decoded_token = decode_token(encoded_token, csrf_token)
File "env/lib/python3.6/site-packages/flask_jwt_extended/utils.py", line 80, in decode_token
encoded_token, verify=False, algorithms=config.algorithm
File "env/lib/python3.6/site-packages/jwt/api_jwt.py", line 84, in decode
payload, _, _, _ = self._load(jwt)
File "env/lib/python3.6/site-packages/jwt/api_jws.py", line 183, in _load
raise DecodeError('Not enough segments')
jwt.exceptions.DecodeError: Not enough segments
I cannot rely on clients always using correct tokens all the time.
And I cannot catch the exception because it is raised in the decorator rather than my own function. So the result is a http 500 error. How should I handle the exception more gracefully?
Flask-jwt-extended should be handling those for you gracefully. If not, you are probably using another extension (like flask-restful for example) that is breaking native flask functionality. You can try setting this option to fix it app.config[‘PROPAGATE_EXCEPTIONS’] = True, or take a look at this thread for some advice if you are using a different flask extension that is causing problems https://github.com/vimalloc/flask-jwt-extended/issues/86
I've created a task that adds a value to session so then i can access it through browser , my celery is running perfectly, here are my codes:
from flask import session, g
from ..extensions import celery
CELERYBEAT_SCHEDULE = {
'add-every-10-seconds': {
'task': 'tasks.ruble',
'schedule': timedelta(seconds=20)
}
}
#celery.task(name='tasks.ruble')
def ruble():
url = 'https://finance.google.com/finance/converter?a=1&from=KGS&to=RUB&meta=ei%3DmSr0WeHCCYvBsAH8n6OIBA'
urlHandler = urllib2.urlopen(url)
html = urlHandler.read()
bsoup = BeautifulSoup(html, 'lxml')
num = bsoup.find('span').text.split()[0]
g.get('ruble', float(num))
session['get_ruble'] = float(num)
but when the times come to execute the task, it ends up with this error:
[2017-10-30 19:19:29,372: ERROR/PoolWorker-4] Task tasks.ruble[ff7617ad-764b-455c-b541-96f3ba78a87b] raised unexpected: RuntimeError('Working outside of request context.\n\nThis typically means that you attempted to use functionality that needed\nan active HTTP request. Consult the documentation on testing for\ninformation about how to avoid this problem.',)
Traceback (most recent call last):
File "/home/xakep/Desktop/work/optomvse/venv/local/lib/python2.7/site-packages/celery/app/trace.py", line 367, in trace_task
R = retval = fun(*args, **kwargs)
File "/home/xakep/Desktop/work/optomvse/celery_worker.py", line 21, in __call__
return TaskBase.__call__(self, *args, **kwargs)
File "/home/xakep/Desktop/work/optomvse/venv/local/lib/python2.7/site-packages/celery/app/trace.py", line 622, in __protected_call__
return self.run(*args, **kwargs)
File "/home/xakep/Desktop/work/optomvse/develop/tasks.py", line 34, in ruble
session['get_ruble'] = float(num)
File "/home/xakep/Desktop/work/optomvse/venv/local/lib/python2.7/site-packages/werkzeug/local.py", line 350, in __setitem__
self._get_current_object()[key] = value
File "/home/xakep/Desktop/work/optomvse/venv/local/lib/python2.7/site-packages/werkzeug/local.py", line 306, in _get_current_object
return self.__local()
File "/home/xakep/Desktop/work/optomvse/venv/local/lib/python2.7/site-packages/flask/globals.py", line 37, in _lookup_req_object
raise RuntimeError(_request_ctx_err_msg)
RuntimeError: Working outside of request context.
This typically means that you attempted to use functionality that needed
an active HTTP request. Consult the documentation on testing for
information about how to avoid this problem.
Another thing i forgot to mention is that i also tried to use g instead of session , by just assigning the float(num) value to it so g.ruble = float(num), but inside my template if i typed {{g.ruble}} nothing get printed out .
If the value keeps changing store it somewhere (Redis/ any database) and when the request from the user comes just update the session data in the API call.
session['get_ruble'] = get_stored_value()
You can not update the session data asynchronously since you don't have the request context in the async task.
Just a suggestion:
If the value is so dynamic instead of storing it in the session, you can provide a separate API to fetch the latest data.
Trying to get the GET parameters from the URL. I have it working in my __init__.py file, but in a different file its not working.
I tried to use with app.app_context(): but I am still getting the same issue.
def log_entry(entity, type, entity_id, data, error):
with app.app_context():
zip_id = request.args.get('id')
RuntimeError: working outside of request context
Any suggestions?
Additional Info:
This is using Flask web framework which is setup as a service (API).
Example URL the user would hit http://website.com/api/endpoint?id=1
As mentioned above using `zip_id = request.args.get('id') works fine in the main file but I am in runners.py (just another file with definitions in)
Full traceback:
Debugging middleware caught exception in streamed response at a point where response headers were already sent.
Traceback (most recent call last):
File "/Users/ereeve/.virtualenvs/pi-automation-api/lib/python2.7/site-packages/werkzeug/wsgi.py", line 703, in __next__
return self._next()
File "/Users/ereeve/.virtualenvs/pi-automation-api/lib/python2.7/site-packages/werkzeug/wrappers.py", line 81, in _iter_encoded
for item in iterable:
File "/Users/ereeve/Documents/TechSol/pi-automation-api/automation_api/runners.py", line 341, in create_agencies
log_entry("test", "created", 1, "{'data':'hey'}", "")
File "/Users/ereeve/Documents/TechSol/pi-automation-api/automation_api/runners.py", line 315, in log_entry
zip_id = request.args.get('id')
File "/Users/ereeve/.virtualenvs/pi-automation-api/lib/python2.7/site-packages/werkzeug/local.py", line 343, in __getattr__
return getattr(self._get_current_object(), name)
File "/Users/ereeve/.virtualenvs/pi-automation-api/lib/python2.7/site-packages/werkzeug/local.py", line 302, in _get_current_object
return self.__local()
File "/Users/ereeve/.virtualenvs/pi-automation-api/lib/python2.7/site-packages/flask/globals.py", line 20, in _lookup_req_object
raise RuntimeError('working outside of request context')
RuntimeError: working outside of request context
Def in the same file calling the log_entry def
def create_agencies(country_code, DB, session):
document = DB.find_one({'rb_account_id': RB_COUNTRIES_new[country_code]['rb_account_id']})
t2 = new_t2(session)
log_entry("test", "created", 1, "{'data':'hey'}", "")
I have the following very simple app:
from lib.flask import Flask
from lib.flask_restful import Resource, Api
app = Flask(__name__)
api = Api(app)
class TestResource(Resource):
def get(self):
return {"a":"b","c":"d"}
api.add_resource(TestResource, "/")
When I run this, I get the following Exception:
Traceback (most recent call last):
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/google/appengine/runtime/wsgi.py", line 266, in Handle
result = handler(dict(self._environ), self._StartResponse)
File "/Users/me/dev/project/src/main.py", line 22, in do_in_request
app(*args, **kwargs)
File "/Users/me/dev/project/src/lib/flask/app.py", line 1836, in __call__
return self.wsgi_app(environ, start_response)
File "/Users/me/dev/project/src/lib/flask/app.py", line 1820, in wsgi_app
response = self.make_response(self.handle_exception(e))
File "/Users/me/dev/project/src/lib/flask_restful/__init__.py", line 249, in error_router
if self._has_fr_route():
File "/Users/me/dev/project/src/lib/flask_restful/__init__.py", line 230, in _has_fr_route
if self._should_use_fr_error_handler():
File "/Users/me/dev/project/src/lib/flask_restful/__init__.py", line 211, in _should_use_fr_error_handler
adapter = self.app.create_url_adapter(request)
File "/Users/me/dev/project/src/lib/flask/app.py", line 1601, in create_url_adapter
return self.url_map.bind_to_environ(request.environ,
File "/Users/me/dev/project/src/lib/werkzeug/local.py", line 338, in __getattr__
return getattr(self._get_current_object(), name)
File "/Users/me/dev/project/src/lib/werkzeug/local.py", line 297, in _get_current_object
return self.__local()
File "/Users/me/dev/project/src/lib/flask/globals.py", line 20, in _lookup_req_object
raise RuntimeError('working outside of request context')
RuntimeError: working outside of request context
So I tried to put my entire app into what I thought would be the request context. In the code above, both the Flask object and the Api object are being instantiated once, and can be called by the server multiple times. From the traceback, it looks like the instantiation should happen within a request context, so I wrapped it like this:
def do_in_request(*args, **kwargs):
from lib.flask import Flask
from lib.flask_restful import Resource, Api
app = Flask(__name__)
api = Api(app)
class TestResource(Resource):
def get(self):
return {"a":"b","c":"d"}
api.add_resource(TestResource, "/")
app(*args, **kwargs)
app = do_in_request
That still raises the same error. What's going on, what do they mean by 'request context', and how do I fix this?
I started from scratch using the App Engine Flask Skeleton and added flask-restful as a dependency in requirements.txt and then just added more or less the code you have above and it worked without an issue. I added the repository to my github here if you want to use it - you can see the changes I made to the skeleton in this commit.
I'm not sure why your example isn't working, but one thing I noticed that could be causing issues is that you're doing from lib.flask .... It seems your third-party packages live in lib and you haven't necessarily "lifted" lib into your sys.path. The skeleton includes a vendoring library that makes sure that this is handled properly. It might be a potential source of the problem.
Either way, I hope the forked skeleton can help you get up and running.
I am new to python and Google App Engine. I am currently working on a location based web application, I have used the following sample code http://code.google.com/p/latitudesample/
I was able to run the program in the localhost, however when I deploy the program and try to access the website I get an Internal Server Error Message.
I check the App Engine log file and it is as follows:
argument 2 to map() must support iteration
Traceback (most recent call last):File"/base/python_runtime/python_lib/versions/1/google/appengine/ext/webapp/__init__.py", line 700, in __call__
handler.get(*groups)
File "/base/data/home/apps/s~latitudesocial/1.351807357650160822/main.py", line 57, in get
self.request.host_url + OAUTH_CALLBACK_PATH, parameters)
File "/base/data/home/apps/s~latitudesocial/1.351807357650160822/oauth_webapp.py", line 28, in redirect_to_authorization_page
request_token = helper.GetRequestToken(callback_url, parameters)
File "/base/data/home/apps/s~latitudesocial/1.351807357650160822/oauth_appengine.py", line 78, in GetRequestToken
None)
File "/base/data/home/apps/s~latitudesocial/1.351807357650160822/oauth.py", line 262, in sign_request
self.build_signature(signature_method, consumer, token))
File "/base/data/home/apps/s~latitudesocial/1.351807357650160822/oauth.py", line 266, in build_signature
return signature_method.build_signature(self, consumer, token)
File "/base/data/home/apps/s~latitudesocial/1.351807357650160822/oauth.py", line 632, in build_signature
oauth_request, consumer, token)
File "/base/data/home/apps/s~latitudesocial/1.351807357650160822/oauth.py", line 623, in build_signature_base_string
key = '%s&' % escape(consumer.secret)
File "/base/data/home/apps/s~latitudesocial/1.351807357650160822/oauth.py", line 50, in escape
return urllib.quote(s, safe='~')
File "/base/python_runtime/python_dist/lib/python2.5/urllib.py", line 1214, in quote
res = map(safe_map.__getitem__, s)
TypeError: argument 2 to map() must support iteration
I have been trying to figure this out myself, but I have made no progress so far.
I am guessing there is something wrong with line 623 in oauth.py
If you could, please help me locate the error.
$key = '%s&' % escape(consumer.secret)
The change I've made to the main.py is:
class SetMyOauth(webapp.RequestHandler):
def get(self):
Config.set('oauth_consumer_key', 'myconsumerkey'),
Config.set('oauth_consumer_secret', 'myconsumersecret'),
self.response.out.write ("""key and secret set""")
(Update 17/7/2011)
Upon further inquiry I came across the following error when running the debugger
C:\Python25\lib\threading.py:699: RuntimeWarning: tp_compare didn't return -1 or -2 for exception
return _active[_get_ident()]
Exception exceptions.SystemError: 'error return without exception set' in <generator object at 0x03C41378> ignored
and it highlighted the following piece of code:
def escape(s):
"""Escape a URL including any /."""
return urllib.quote(s, safe='~')
which is the same code that is logged in the log file.
However when I run the code in using the development server the application works fine with no errors.