I am tying to do a python code with cherrypy. The objective is to call a script from UI and while the script generates the logs on console print it on the UI (Iframe / div).
I am using CherryPy-3.2.4 and Python 2.7.
Following is the sample code I have written which is refusing to work.
TestProcess.py
from multiprocessing import Process, Queue
import subprocess
import os, os.path
from string import Template
import cherrypy
from multiprocessing import Process, Queue
import HTML
import TestScript
class TestProcess(object):
PID=0
jquery_url = 'C:\CherryPy\TestScripts\jquery\jquery-1.11.1.js'
q=Queue()
#cherrypy.expose
def index(self):
html = """\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<HTML>
<HEAD>
<TITLE> Test Doc </TITLE>
<script type="text/javascript" src="/static/jq/jquery-1.11.1.js"></script>
</HEAD>
<BODY>
<BR/>
<h3>Test utility</h3>
<form id="ping_form" target="console_iframe" method="post" action="/f">
<button id="ping" type="submit">Upgrade Installed Releases</button>
</form>
<BR/>
<iframe name="console_iframe" frameborder="1" width="1200"/>
</BODY>
</HTML>
"""
t = Template(html)
page = t.substitute(jquery_url=self.jquery_url)
return page
def f1():
print "*********** TEST **********************"
q.put([42, None, 'hello'])
#cherrypy.expose
def f(self, **kw):
q=Queue()
ts = TestScript.TestScript()
p = Process(target=ts.testCallFunction, args=(q,))
p.start()
#print q.get() # prints "[42, None, 'hello']"
p.join()
print "Test F"
def run_command():
# The yeilds here are the key to keeping things streaming
yield '<style>body {font-family: monospace;}</style>'
while(p.is_alive()):
while(not q.empty()):
yield q.get_nowait()
while(not q.empty()):
yield q.get_nowait()
return run_command()
if __name__ == '__main__':
conf = {
'/': {
'tools.sessions.on': True,
'tools.staticdir.root': os.path.abspath(os.getcwd())
},
'/static': {
'tools.staticdir.on': True,
'tools.staticdir.dir': './public'
}
}
q = Queue()
cherrypy.server.socket_host = '10.49.69.103'
cherrypy.config.update({
'log.screen':True,
'tools.sessions.on': True,
'checker.on':False
})
cherrypy.tree.mount(TestProcess(), config=None)
cherrypy.engine.start()
cherrypy.engine.block()
#cherrypy.quickstart(TestProcess(), '/', conf)
Class TestScript
class TestScript(object):
q=Queue()
def write_queue(self,data):
print "*********** TEST **********************"
scroll_to_bottom = '<script type="text/javascript">window.scrollBy(0,50);</script>'
if data == '\n':
self.q.put("\n<br />%s" % scroll_to_bottom) # include the iframe scroll fix
else:
self.q.put(data)
def testCallFunction(self, q):
#proc = subprocess.Popen(['python','CreateLog.py'],stdout=subprocess.PIPE)
cmd = subprocess.Popen(['python','CreateLog.py'], shell=True, stdout=subprocess.PIPE)
while True:
data = cmd.stdout.read(1) # Alternatively proc.stdout.read(1024)
if len(data) == 0:
break
self.write_queue(data) # sys.stdout.buffer.write(data) on Python 3.x
#print "*********** TEST **********************"
script CreateLog.py
#filters output
import time
i = 0
while (i<=10):
print hex(i)*512
i += 1
time.sleep(0.5)
Error That I get while running it
[24/Jul/2014:16:59:10] ENGINE Bus STARTING
[24/Jul/2014:16:59:10] ENGINE Started monitor thread 'Autoreloader'.
[24/Jul/2014:16:59:10] ENGINE Started monitor thread '_TimeoutMonitor'.
[24/Jul/2014:16:59:15] ENGINE Error in 'start' listener <bound method Server.sta
rt of <cherrypy._cpserver.Server object at 0x01583390>>
Traceback (most recent call last):
File "c:\Python27\lib\site-packages\cherrypy\process\wspbus.py", line 197, in
publish
output.append(listener(*args, **kwargs))
File "c:\Python27\lib\site-packages\cherrypy\_cpserver.py", line 151, in start
ServerAdapter.start(self)
File "c:\Python27\lib\site-packages\cherrypy\process\servers.py", line 168, in
start
wait_for_free_port(*self.bind_addr)
File "c:\Python27\lib\site-packages\cherrypy\process\servers.py", line 412, in
wait_for_free_port
raise IOError("Port %r not free on %r" % (port, host))
IOError: Port 8080 not free on '10.49.69.103'
[24/Jul/2014:16:59:15] ENGINE Shutting down due to error in start listener:
Traceback (most recent call last):
File "c:\Python27\lib\site-packages\cherrypy\process\wspbus.py", line 235, in
start
self.publish('start')
File "c:\Python27\lib\site-packages\cherrypy\process\wspbus.py", line 215, in
publish
raise exc
ChannelFailures: IOError("Port 8080 not free on '10.49.69.103'",)
What am I doing wrong ?
See 16.6.3.2. Windows:
Make sure that the main module can be safely imported by a new Python interpreter without causing unintended side effects (such a starting a new process).
You did not protect the last lines of TestProcess.py with if __name__ == '__main__':, so the Process attempts to run another server.
Port 8080 is not free, so the HTTP server can't listen on it. Try netstat -anpt to see what's using that port, or try "server.socket_port: 8081" or some other port that is free.
Related
I'm trying to make the following. I want to make a remote control for powering on/off my NAS. See: https://www.youtube.com/watch?v=bDtPBzR6FF0 I didn't want to control it by knobs, but by a HTML button. I want to use a Raspberry Pi to host a website with the buttons.
I tried https://raspberrypi.stackexchange.com/questions/66908/how-to-call-and-run-python-file-on-raspberry-pi-using-web-based-button.
#!/usr/bin/python
# coding: utf-8
import cherrypy
import subprocess
text = """
<html><body>
<form method='get' action='do_it'>
<input type='submit' value='Submit' />
</form></body>
{}
</html>
"""
class PiButton(object):
#cherrypy.expose
def index(self):
return text.format("")
#cherrypy.expose
def do_it(self, *vargs, **kwargs):
#command = "ls /"
command = "python my_other_python.script.py"
result = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout.read().decode('utf-8').replace('\n', '<br>')
result2 = "command: {!r}<br>result:<br>{}".format(command, result)
return text.format(result2)
if __name__ == "__main__":
cherrypy.engine.autoreload.unsubscribe()
cherrypy.config.update({'server.socket_host': "0.0.0.0", 'server.socket_port': 8181})
cherrypy.quickstart(PiButton(), '/', {'':{}})
It gave an error:
Traceback (most recent call last):
File "button.py", line 17, in
class PiButton(object):
File "button.py", line 34, in PiButton
cherrypy.quickstart(PiButton(), '/', {'':{}})
NameError: name 'PiButton' is not defined
I'm quit new to Python?Raspberry and I don't quite understand what goes wrong. Anyone can tell me what the right solution can be?
I'm trying to write a "hello world" of IPC between a node js and a python3 application. At the moment I'm having a parse error when a new message arrives the Python app. The codes are:
Python:
from ipcqueue import posixmq
from time import sleep
q1 = posixmq.Queue('/fila1')
while True:
while q1.qsize() > 0:
print('p1: Recebi na minha fila: ' + str(q1.get()))
sleep(0.5)
Node:
const PosixMQ = require('posix-mq')
var mq = new PosixMQ();
mq.open({
name: '/fila1',
create: true,
mode: '0777',
maxmsgs: 10,
msgsize: 11
});
mq.push("hello world")
mq.close();
When the second app sends the message, the python app fails with:
File "../test-fila1.py", line 9, in
print('p1: Recebi na minha fila: ' + str(q1.get())) File "/usr/lib/python3.7/site-packages/ipcqueue/posixmq.py", line 174, in
get
return self._serializer.loads(data) File "/usr/lib/python3.7/site-packages/ipcqueue/serializers.py", line 14,
in loads
return pickle.loads(data) KeyError: 101
[2]- Exit 1 python3 ../test-fila1.py
[3]+ Done node index.js
EDIT
So, I decided to change the question from "KeyError: 101 when loading ipc message between node and python apps" to "How to IPC between node and python using posix message queue?" because I just noticed that the sample code I posted generates different errors everytime I run them. Now the error that is happening is (however, no code was changed):
Traceback (most recent call last): File "snippets/test-fila1.py",
line 9, in
print('p1: Recebi na minha fila: ' + str(q1.get())) File "/usr/lib/python3.7/site-packages/ipcqueue/posixmq.py", line 174, in
get
return self._serializer.loads(data) File "/usr/lib/python3.7/site-packages/ipcqueue/serializers.py", line 14,
in loads
return pickle.loads(data)
_pickle.UnpicklingError: unpickling stack underflow
If you look the __init__ of the Queue
class Queue(object):
"""
POSIX message queue.
"""
def __init__(self, name, maxsize=10, maxmsgsize=1024, serializer=PickleSerializer):
As you can see the default serializer is PickleSerializer, which means it assumes the data coming on the queue is pickled, while you are sending raw data from nodejs. The fix is simple, use the RawSerializer
from ipcqueue import posixmq
from time import sleep
from ipcqueue.serializers import RawSerializer
q1 = posixmq.Queue('/fila1', serializer=RawSerializer)
while True:
while q1.qsize() > 0:
print('p1: Recebi na minha fila: ' + str(q1.get()))
sleep(0.5)
I Have a Python project which I have built in PyCharm, which works if I run from PyCharm, however now I am ready to run it from command line I get the error.
ImportError: No module named 'integration'
From what I have seen on other similar issues you need to set PYTHONPATH to be the working directory, which I have also done:
[nathan#Linux-Macbook integration]$ export PYTHONPATH=/home/nathan/mobileSubscriberDB/integration/integration
[nathan#Linux-Macbook integration]$ echo $PYTHONPATH
/home/nathan/mobileSubscriberDB/integration/integration
Then when I run the command below (Which I am also within the directory specified on the PYTHONPATH), I also get the error straight after:
[nathan#Linux-Macbook integration]$ /home/nathan/python_msdb/bin/python /home/nathan/mobileSubscriberDB/integration/integration/main.py
Traceback (most recent call last):
File "/home/nathan/mobileSubscriberDB/integration/integration/main.py", line 5, in <module>
from integration import bigchain
ImportError: No module named 'integration'
Below is my project structure:
Below is a Copy of my main.py:
import threading
import sys
import logging
from integration import bigchain
from integration import config
from integration import log
from integration import rabbitMq, file
def do_something_with_exception():
exc_type, exc_value = sys.exc_info()[:2]
print('Handling %s exception with message "%s" in %s' %
(exc_type.__name__, exc_value, threading.current_thread().name))
class ConsumerThread(threading.Thread):
def __init__(self, queue, blockchain_host, blockchain_port, rabbit_props, blockchain_props,*args, **kwargs):
super(ConsumerThread, self).__init__(*args, **kwargs)
logger.debug("Data Type: %s" % queue)
self._queue = queue
logger.debug("Blockchain Host: %s" % blockchain_host)
self._blockchain_host = blockchain_host
logger.debug("Blockchain Port: %s" % blockchain_port)
self._blockchain_port = blockchain_port
self._rabbit_props = rabbit_props
self._blockchain_props = blockchain_props
def run(self):
logger.info("Connecting to BigChain DB for %s Data Type" % self._queue)
bigchaindb = bigchain.BigChain()
bdb = bigchaindb.connect(self._blockchain_host,
self._blockchain_port)
private_key = file.File.read_file(self._blockchain_props['Private Key Path'])
public_key = file.File.read_file(self._blockchain_props['Public Key Path'])
keys = bigchaindb.loadkeys(private_key, public_key)
logger.info('Connecting to RabbitMQ for %s Data Type' % self._queue)
rabbit = rabbitMq.RabbitMq(self._rabbit_props,
self._queue,
bdb,
keys)
channel = rabbit.connect()
rabbit.consume(channel)
if __name__ == "__main__":
log = log.Logger('/etc/msdb/integration/logger.ini')
log.load_config()
logger = logging.getLogger('main')
logger.info("Starting Mobile Subscriber Database Integration...")
threads = []
logger.debug("Created Empty Threads Array")
logger.info("Loading Configuration")
config = config.Config('/etc/msdb/integration/config.ini')
properties = config.load_config()
sections = config.get_sections(properties)
logger.debug("Getting RabbitMQ Properties")
Rabbit_Props = config.loadSection('RabbitMQ', properties)
logger.debug("Getting Blockchain Properties")
Blockchain_Props = config.loadSection('Blockchain', properties)
logger.debug("Loading Data Type Properties")
for section in sections:
props = config.loadSection(section, properties)
if (section != 'RabbitMQ') or (section != 'Blockchain'):
logger.debug("Loading %s Properties" % section)
if props.getboolean('Enabled'):
logger.debug("Section Enabled, Creating Thread")
threads.append(
ConsumerThread(
props['RabbitMQ Queue'],
props['Blockchain Hostname'],
int(props['Blockchain Port']),
Rabbit_Props,
Blockchain_Props
)
)
else:
logger.debug("Section Disabled")
logger.info("Starting Threads")
for thread in threads:
thread.daemon = True
thread.start()
logger.info("Joining Threads")
for thread in threads:
thread.join()
exit(0)
I'm trying to write a web parser script using requests module. Here is my current code:
import requests
import subprocess
import json
import sys
import threading
import time
from Queue import Queue
numberOfViewers = int(sys.argv[1])
builderThreads = int(sys.argv[2])
startTime = time.time()
numberOfSockets = 0
concurrent = 25
urls = []
urlsUsed = []
def getURL(): # Get tokens
output = subprocess.Popen(["livestreamer", "twitch.tv/CHANNEL_NAME", "-j"],
stdout=subprocess.PIPE).communicate()[0]
return json.loads(output)['streams']['worst']['url'] # Parse json and return the URL parameter
def build(): # Builds a set of tokens, aka viewers
global numberOfSockets
global numberOfViewers
while True:
if numberOfSockets < numberOfViewers:
numberOfSockets += 1
print ("Building viewers " + str(numberOfSockets) + "/" + str(numberOfViewers))
urls.append(getURL())
def view(): # Opens connections to send views
global numberOfSockets
while True:
url=q.get()
requests.head(url)
if (url in urlsUsed):
urls.remove(url)
urlsUsed.remove(url)
numberOfSockets -= 1
else:
urlsUsed.append(url)
q.task_done()
if __name__ == '__main__':
for i in range(0, builderThreads):
threading.Thread(target = build).start()
while True:
while (numberOfViewers != numberOfSockets): # Wait until sockets are built
time.sleep(1)
q=Queue(concurrent*2)
for i in range(concurrent):
try:
t=threading.Thread(target=view)
t.daemon=True
t.start()
except:
print ('thread error')
try:
for url in urls:
print (url)
q.put(url.strip())
q.join()
except KeyboardInterrupt:
sys.exit(1)
But when I run the code, it says:
Traceback (most recent call last):
File "C:\Users\flamelier\Desktop\Twitch.py", line 1, in <module>
import requests
ImportError: No module named 'requests'
Why am I getting this error? How do I install this module?
Will this error keep repeating for all the scripts henceforth?
How can I prevent such similar errors in the future?
Requests is a 3rd party module. You should first install it to Python using PIP or easy_install.
You have to run pip3 install requests as requests doesn't come with Python by default, as it is a third party library.
Even after you have pip3-installed requests, the code shown won't do anything. The
if __name__ == "__main__"
test and everything after it is part of an else block in the view function. Back this line and the block that follows out to the left margin.
I'm writing a program to get the domain in same server and it also can scan the web directory.
#!/usr/bin/env python
#encoding = utf-8
import threading
import urllib,urllib2,httplib
from urllib2 import Request, urlopen, URLError
import Queue,sys
import re
concurrent = 5
url = sys.argv[1]
class Scanner(threading.Thread):
def __init__(self, work_q):
threading.Thread.__init__(self)
self.work_q = work_q
def getdomains(self):
doreq = Request('http://www.logontube.com/website/'+ url)
response = urlopen(doreq)
html = response.read()
response.close()
domains = re.findall('<br><a href=\"(.*?)\" target=\"_blank\"',html)
return domains
def run(self):
alldomains = self.getdomains()
pathline = [line.rstrip() for line in open("path.txt")]
while True:
for aim in alldomains:
for path in pathline:
path = self.work_q.get()
req = Request(aim+path)
try:
response = urlopen(req)
except URLError, e:
if hasattr(e, 'reason'):
print aim+path,'Not Found'
elif hasattr(e,'code'):
print aim+path,'Not Found'
else:
try:
logs = open('log.txt',"a+")
except(IOError):
print "[x] Failed to create log file"
print aim+path,"Found"
logs.writelines(aim+path+"\n")
logs.close()
def main():
work_q = Queue.Queue()
paths = [line.rstrip() for line in open("path.txt")]
for i in range(concurrent):
t = Scanner(work_q)
t.setDaemon(True)
t.start()
for path in paths:
work_q.put(path)
work_q.join()
main()
The problem is this program only do the loop of the path, so i only can get the scan result of one website.
I've found the problem,
for path in paths:
work_q.put(path) # The program finishes when it puts all the path
If you want to help me to test this program, you may need some directory of website(save it as path.txt)
/default.asp
/index.asp
/index.htm
/index.html
/index.jsp
/index.php
/admin.asp
/admin.php
/admin.shtml
/admin.txt
/admin_admin.asp
/config.asp
/inc/
/login.asp
/login.jsp
/login.php
/login/
/phpinfo.php
/readme.txt
/robots.txt
/test.asp
/test.html
/test.txt
/test.php
/news/readme.txt
/addmember/
You need a:
while 1:
pass
or something that waits until your threads are completed then it exits.
What is happening is that you are starting the threads but you are terminating the main thread so you never get to see the results of your threads.