I'm very new to python programming, I'm trying to expose few terminal commands as a http call. I'm packaging this below code into a tar.gz using setup.py and extracting this on the server to run the code in nginx web server. once I instantiate the app.py code using below, the http calls work fine for few hours. Not sure what happens after that, the http pages show an error and I do see the app is running in the server. Can someone please help me with this
gunicorn --bind 0.0.0.0:8000 cstaradmin:app &
#!/usr/bin/env python
from flask import Flask
from shelljob import proc
from flask import Response
from flask import json,jsonify
def get_pretty_print(json_object):
return json.dumps(json_object, sort_keys=False, indent=4, separators=('\n',' '))
def read_process(g):
while g.is_pending():
lines=g.readlines()
for proc,line in lines:
yield line
app = Flask(__name__)
def has_no_empty_params(rule):
defaults = rule.defaults if rule.defaults is not None else ()
arguments = rule.arguments if rule.arguments is not None else ()
return len(defaults) >= len(arguments)
#app.route('/', methods = ['GET'])
def help():
"""Print available functions."""
func_list = {}
for rule in app.url_map.iter_rules():
if rule.endpoint != 'static':
func_list[rule.rule] = app.view_functions[rule.endpoint].__doc__
return jsonify(func_list)
#app.route("/ps")
def IRR():
IRR = ["ps"]
g=proc.Group()
p=g.run(IRR)
return Response(read_process(g),mimetype= 'text/plain')
def catch_all(path):
return
if __name__ == "__main__":
try:
app.run(host='0.0.0.0',port=8000)
except:
pass
Error:
Internal Server Error The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application
[2016-10-22 15:38:55,505] ERROR in app: Exception on /path/ToMy/command [GET]
Traceback (most recent call last):
File "/usr/lib64/python2.7/site-packages/flask/app.py", line 1988, in wsgi_app
response = self.full_dispatch_request()
File "/usr/lib64/python2.7/site-packages/flask/app.py", line 1641, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/lib64/python2.7/site-packages/flask/app.py", line 1544, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/lib64/python2.7/site-packages/flask/app.py", line 1639, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/lib64/python2.7/site-packages/flask/app.py", line 1625, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/path/ToMyapp/admin/cstaradmin.py", line 42, in listnodes
p=g.run(listnodes)
File "/usr/lib/python2.7/site-packages/shelljob/proc.py", line 53, in run
CommandException( e, "Group.run '{}' failed".format( cmd ) ).do_raise()
File "/usr/lib/python2.7/site-packages/shelljob/proc.py", line 51, in run
return self._run_impl( cmd, shell )
File "/usr/lib/python2.7/site-packages/shelljob/proc.py", line 63, in _run_impl
stdin = subprocess.PIPE, # needed to detach from calling terminal (other wacky things can happen)
File "/usr/lib64/python2.7/subprocess.py", line 711, in __init__
errread, errwrite)
File "/usr/lib64/python2.7/subprocess.py", line 1216, in _execute_child
errpipe_read, errpipe_write = self.pipe_cloexec()
File "/usr/lib64/python2.7/subprocess.py", line 1168, in pipe_cloexec
r, w = os.pipe()
CommandException: ("Group.run '['nodetool', ' status']' failed", OSError(24, 'Too many open files'))
Related
I have a python script (script.py) that when I run it in the console it keeps receiving data in JSON format, now I want to visualize the output of the console on a web page or to be able to execute the script.py and capture the output to view in the browser, from what I have read I understand that with FLASK I could do it, an example or guide to achieve it.
from flask import Flask
app = Flask(__name__)
#app.route("/")
def code():
out = open(r'output.py', 'r').read()
return exec(out)
if __name__ == '__main__':
app.run(host='0.0.0.0')
///////////////////////////////////////////////
builtins.RuntimeError
RuntimeError: There is no current event loop in thread 'Thread-9'.
Traceback (most recent call last)
File "..//python3.6/site-packages/flask/app.py", line 2301, in __call__
return self.wsgi_app(environ, start_response)
File "..//python3.6/site-packages/flask/app.py", line 2287, in wsgi_app
response = self.handle_exception(e)
File "..//python3.6/site-packages/flask/app.py", line 1733, in handle_exception
reraise(exc_type, exc_value, tb)
File "..//python3.6/site-packages/flask/_compat.py", line 35, in reraise
raise value
File "..//python3.6/site-packages/flask/app.py", line 2284, in wsgi_app
response = self.full_dispatch_request()
File "..//python3.6/site-packages/flask/app.py", line 1807, in full_dispatch_request
rv = self.handle_user_exception(e)
File "..//python3.6/site-packages/flask/app.py", line 1710, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "..//python3.6/site-packages/flask/_compat.py", line 35, in reraise
raise value
File "..//python3.6/site-packages/flask/app.py", line 1805, in full_dispatch_request
rv = self.dispatch_request()
File "..//python3.6/site-packages/flask/app.py", line 1791, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/share/server/www/test/index1.py", line 9, in dynamic_page
return exec(file)
File "<string>", line 60, in <module>
///////////////////////////////////////////////////////////////////////////
output.py
#!/usr/bin/env python3
import json
import sys
from phone import Runn, BReporter, EventHandler
class NoOpEventHandler(EventHandler):
def on_event(self, event):
self._reporter.on_event(event)
class VerboseNoOpEventHandler(NoOpEventHandler):
FILTER_EVENTS = False
class JsonReporter(BReporter):
def __init__(self):
self._has_first_line = False
def on_event(self, event):
json_data = json.dumps(dict(event), sort_keys=True)
if self._has_first_line:
sys.stdout.write(',')
else:
sys.stdout.write('[')
self._has_first_line = True
sys.stdout.write('\n {}'.format(json_data))
def close(self):
sys.stdout.write('\n]')
reporter = JsonReporter()
runner = Runn([
'tcp://test123:test123#127.0.0.1:5038',
], reporter, handler=NoOpEventHandler)
runner.run()
just use this; you must set debug = True. You do not have to provide host for testing.
if __name__ == "__main__":
app.run(debug=True)
you should see an output like that on terminal;
* Serving Flask app "my_flask_app" (lazy loading)
* Environment: production
WARNING: This is a development server. Do not use it in a production deployment.
Use a production WSGI server instead.
* Debug mode: on
* Restarting with stat
* Debugger is active!
* Debugger PIN: xxx-xxx-xxx
* Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)
open a web browser and copy paste: http://127.0.0.1:5000/
I saved your code in tmp.py and just ran it as python tmp.py.
from flask import Flask
app = Flask(__name__)
#app.route("/")
def code():
out = open(r'output.py', 'r').read()
return exec(out)
if __name__ == '__main__':
app.run(debug = True)
run it as below:
(flaskvenv) C:...>python tmp.py
Here is the output:
* Serving Flask app "tmp" (lazy loading)
* Environment: production
WARNING: This is a development server. Do not use it in a production deployment.
Use a production WSGI server instead.
* Debug mode: on
* Restarting with stat
* Debugger is active!
* Debugger PIN: XXX-XXX-XXX
* Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)
That means the code runs. You might have another problem!
I think the problem is with the exec(out) . Actually , exec is a dynamic execution of a program which can either be a string or object code, and it does not return anything , it just executes a program and returns None.
If you run your code without trying to return the exec(out) you won't get error.
from google.cloud import tasks_v2
import json
GCP_PROJECT='test'
GCP_LOCATION='europe-west6'
def enqueue_task(queue_name, payload, process_url):
client = tasks_v2.CloudTasksClient()
parent = client.queue_path(GCP_PROJECT, GCP_LOCATION, queue_name)
task = {
'app_engine_http_request': {
'http_method': 'POST',
'relative_uri': process_url
}
}
if payload is None:
return False
payload = json.dumps(payload)
converted_payload = payload.encode()
task['app_engine_http_request']['body'] = converted_payload
return client.create_task(parent, task)
I keep getting the following error when I try to create a Google Cloud Task.
The Google App Engine Runtime used is python38. It was working fine but suddenly after deployment using gcp CLI it does not work now.
Traceback (most recent call last):
File "/layers/google.python.pip/pip/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/layers/google.python.pip/pip/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/layers/google.python.pip/pip/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/layers/google.python.pip/pip/flask/_compat.py", line 39, in reraise
raise value
File "/layers/google.python.pip/pip/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/layers/google.python.pip/pip/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/srv/main.py", line 153, in sync_request
queue_response = queues.enqueue_task(
File "/srv/queues.py", line 28, in enqueue_task
return client.create_task(parent, task)
TypeError: create_task() takes from 1 to 2 positional arguments but 3 were given
The same thing has just happened to me (on live server :/). Turns out, the Cloud Tasks library has changed significantly with version 2.0.0. You can read what you need to do to upgrade here.
Your problem is in this line:
client.create_task(parent, task)
The updated library requires to either use a dictionary as a positional argument or use keyword arguments. So this should fix it:
client.create_task(parent=parent, task=task)
EDIT: Looking at your code now that I've made this work for myself, you will also have to change the following:
# Before
parent = client.queue_path(GCP_PROJECT, GCP_LOCATION, queue_name)
# After
parent = client.queue_path(project=GCP_PROJECT, location=GCP_LOCATION, queue=queue_name)
and
# Before
'http_method': 'POST',
# After
'http_method': tasks_v2.HttpMethod.POST,
Trying to use the google provided python sample code for adding a task to a Google Cloud Tasks queue. The docs from google are here: https://cloud.google.com/tasks/docs/creating-appengine-tasks
When I attempt to execute the endpoint I get the following error:
ImportError: cannot import name 'resource_pb2'
Has anyone seen an similar issue and know how to get past this?
Thanks in advance for your time :)
-nathan
I have tried adding google-cloud-tasks, google-api, google-cloud to my requirements.txt and running pip install google-cloud-tasks in my virtual environment. I also tried using this code:
from google.cloud import tasks_v2beta3
The results were the same
This is the example code provided by google. I have sent a note to them to let them know the example code does not work.
from google.cloud import tasks_v2
from google.protobuf import timestamp_pb2
# Create a client.
client = tasks_v2.CloudTasksClient()
# TODO(developer): Uncomment these lines and replace with your values.
# project = 'my-project-id'
# queue = 'my-appengine-queue'
# location = 'us-central1'
# payload = 'hello'
# Construct the fully qualified queue name.
parent = client.queue_path(project, location, queue)
# Construct the request body.
task = {
'app_engine_http_request': { # Specify the type of request.
'http_method': 'POST',
'relative_uri': '/example_task_handler'
}
}
if payload is not None:
# The API expects a payload of type bytes.
converted_payload = payload.encode()
# Add the payload to the request.
task['app_engine_http_request']['body'] = converted_payload
if in_seconds is not None:
# Convert "seconds from now" into an rfc3339 datetime string.
d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds)
# Create Timestamp protobuf.
timestamp = timestamp_pb2.Timestamp()
timestamp.FromDatetime(d)
# Add the timestamp to the tasks.
task['schedule_time'] = timestamp
# Use the client to build and send the task.
response = client.create_task(parent, task)
print('Created task {}'.format(response.name))
return response
I expected the code to add a task to the Cloud Tasks Queue, instead I get this stack trace:
Traceback (most recent call last)
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/app.py", line 2309, in __call__
return self.wsgi_app(environ, start_response)
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/app.py", line 2295, in wsgi_app
response = self.handle_exception(e)
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/app.py", line 1741, in handle_exception
reraise(exc_type, exc_value, tb)
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/_compat.py", line 35, in reraise
raise value
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/app.py", line 2292, in wsgi_app
response = self.full_dispatch_request()
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/app.py", line 1815, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/app.py", line 1718, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/_compat.py", line 35, in reraise
raise value
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/app.py", line 1813, in full_dispatch_request
rv = self.dispatch_request()
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/flask/app.py", line 1799, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/athena_nsunderman/posttest-get/main.py", line 134, in TestQueueInsert
from google.cloud import tasks_v2
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/google/cloud/tasks_v2/__init__.py", line 19, in <module>
from google.cloud.tasks_v2 import types
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/google/cloud/tasks_v2/types.py", line 22, in <module>
from google.cloud.tasks_v2.proto import cloudtasks_pb2
File "/home/athena_nsunderman/virtualenv/recharge/lib/python3.5/site-packages/google/cloud/tasks_v2/proto/cloudtasks_pb2.py", line 18, in <module>
from google.api import resource_pb2 as google_dot_api_dot_resource__pb2
ImportError: cannot import name 'resource_pb2'
The debugger caught an exception in your WSGI application. You can now look at the traceback which led to the error.
This worked for me:
pip install -U googleapis-common-protos==1.5.10
I'm using older versions for one of my servers (because some of dependencies). I setup my server in docker container and its working fine until today. When i rebuild my image and re-run the service then its getting this error:
Exception on /messages [GET]
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1379, in handle_user_exception
return handler(e)
File "/opt/sync-engine/inbox/api/ns_api.py", line 226, in handle_input_error
request.environ['log_context']['error'] = error.__class__.__name__
KeyError: 'log_context'
{"module": "inbox.api.srv:28", "level": "error", "event": "Uncaught error thrown by Flask/Werkzeug", "timestamp": "2019-03-27T10:36:55.760581Z", "greenlet_id": 140055343249040}
srv.py (inbox.api.srv):
...
def default_json_error(ex):
""" Exception -> flask JSON responder """
logger = get_logger()
logger.error('Uncaught error thrown by Flask/Werkzeug', exc_info=ex)
response = jsonify(message=str(ex), type='api_error')
response.status_code = (ex.code
if isinstance(ex, HTTPException)
else 500)
return response
...
ns_api.py:
...
#app.errorhandler(APIException)
def handle_input_error(error):
# these "errors" are normal, so we don't need to save a traceback
request.environ['log_context']['error'] = error.__class__.__name__
request.environ['log_context']['error_message'] = error.message
response = flask_jsonify(message=error.message,
type='invalid_request_error')
response.status_code = error.status_code
return response
...
The main reason:
there is no log_context key in the request.environ
To fix this just initialize the log_context key
request.environ.setdefault('log_context', dict())
# etc...
request.environ['log_context']['error'] = error.__class__.__name__
Hope this helps.
What I am trying to achieve is pretty simple.
I want to use Flask to create a web app that connects to a remote Server via API calls (specifically ParseServer).
I am using a third-party library to achieve this and everything works perfectly when I am running my code in a stand-alone script. But when I add my code into the Flask I suddenly can't authenticate with the Server
Or to be more precise I get an 'unauthorized' error when executing an API call.
It seems to me that in Flask, the registration method used by the APi library is not remembered.
I tried many things of putting the registration and initialization code in different places in Flask, nothing worked.
I asked a similar question in the Github of the Library with no help.
So I guess I have two questions that could help me solve this
1) Where should I put a registration method and import of the files from this library?
&
2) What can I do to identify the issue specifically, eg. to know precisely what's wrong?
Here's some code
The Flask code is here
#app.route('/parseinsert')
def run_parse_db_insert():
"""The method for testing implementation and design of the Parse Db
"""
pc = ParseCommunication()
print(pc.get_all_names_rating_table())
return 'done'
The ParseCommunication is my Class that deals with Parse. If I run ParseCommunication from that script, with the same code as above in the main part, everything works perfectly.
I run the Flask app with dev_appserver.py from Google App Engine.
My folder structure
/parseTest
/aplication
views.py
app.yaml
run.py
My run.py code
import os
import sys
sys.path.insert(1, os.path.join(os.path.abspath('.'), 'lib'))
sys.path.insert(1, os.path.join(os.path.abspath('.'), 'application'))
import aplication
Let me know what else I could provide to help out.
Thank you in Advance
EDIT:
A stack trace as requested.
It's mostly related to the library (from what I can say?)
ERROR 2016-09-28 06:45:50,271 app.py:1587] Exception on /parseinsert [GET]
Traceback (most recent call last):
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/flask/app.py", line 1988, in wsgi_app
response = self.full_dispatch_request()
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/flask/app.py", line 1641, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/flask/app.py", line 1544, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/flask/app.py", line 1639, in full_dispatch_request
rv = self.dispatch_request()
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/flask/app.py", line 1625, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/theshade/Devel/ParseBabynames/parseTest/aplication/views.py", line 34, in run_parse_db_insert
name = pc.get_user('testuser1')
File "/home/theshade/Devel/ParseBabynames/parseTest/aplication/parseCommunication.py", line 260, in get_user
return User.Query.get(username=uname)
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/parse_rest/query.py", line 58, in get
return self.filter(**kw).get()
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/parse_rest/query.py", line 150, in get
results = self._fetch()
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/parse_rest/query.py", line 117, in _fetch
return self._manager._fetch(**options)
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/parse_rest/query.py", line 41, in _fetch
return [klass(**it) for it in klass.GET(uri, **kw).get('results')]
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/parse_rest/connection.py", line 108, in GET
return cls.execute(uri, 'GET', **kw)
File "/home/theshade/Devel/ParseBabynames/parseTest/lib/parse_rest/connection.py", line 102, in execute
raise exc(e.read())
ResourceRequestLoginRequired: {"error":"unauthorized"}
Parse requires keys and env variables. Check this line:
API_ROOT = os.environ.get('PARSE_API_ROOT') or 'https://api.parse.com/1'
Your error is in line 102 at:
https://github.com/milesrichardson/ParsePy/blob/master/parse_rest/connection.py
Before you can parse, you need to register:
from parse_rest.connection import register
APPLICATION_ID = '...'
REST_API_KEY = '...'
MASTER_KEY = '...'
register(APPLICATION_ID, REST_API_KEY, master_key=MASTER_KEY)