Develop Reference

How to start app using sockets Tornado 4.4

I'm a newbie to creating manually sockets. My OS is ubuntu. I've got an proxy server written python using Tornado, everything is fine when I use the "fast version" starting the app, I mean the:
if __name__ == "__main__":
app = make_app()
port = options.port # default 8000
if len(sys.argv) > 1:
port = int(sys.argv[1])
app.listen(port)
print 'tornado working on port %s' % port
tornado.ioloop.IOLoop.current().start()
But when i want to change it to use the 'socket version' it seems that I'm doing something wrong. I get an error saying that the address is already used.
code:
def make_app():
return MyApplication()
def connection_ready(sock, fd, events):
while True:
try:
connection, address = sock.accept()
except socket.error as e:
if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
raise
return
connection.setblocking(0)
app = make_app()
app.listen(8000) # I get here an error: [Errno 98] Address already in use
if __name__ == "__main__":
port = options.port # default port 8000
if len(sys.argv) > 1:
port = int(sys.argv[1])
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(False)
sock.bind(("", port))
sock.listen(128)
io_loop = tornado.ioloop.IOLoop.current()
callback = functools.partial(connection_ready, sock)
io_loop.add_handler(sock.fileno(), callback, io_loop.READ)
io_loop.start()
I'm trying to implement the same way that the documentation says (http://www.tornadoweb.org/en/stable/ioloop.html) but I don't see it starting the app in there.
Could someone tell me what is the proper way to start an app using sockets? I'm trying to accomplish an application that is available when the sever accepts the incoming socket. (So every client that connects to my listining port described in the main function at lines:sock.bind(("", port)) and sock.listen(128) will get a new socket and have access to the application).
Edit: I'm adding my proxy class:
class ProxyHandler(tornado.web.RequestHandler):
SUPPORTED_METHODS = ['GET', 'POST']
def data_received(self, chunk):
pass
def compute_etag(self):
return None # disable tornado Etag
def handle_response(self, response):
if response.error and not isinstance(response.error, tornado.httpclient.HTTPError):
self.set_status(500)
self.write('Internal server error:\n' + str(response.error))
else:
self.set_status(response.code, response.reason)
self._headers = tornado.httputil.HTTPHeaders() # clear tornado default header
for header, v in response.headers.get_all():
if header not in ('Content-Length', 'Transfer-Encoding', 'Content-Encoding', 'Connection'):
self.add_header(header, v) # some header appear multiple times, eg 'Set-Cookie'
secured_page = False
for page in secure_pages:
if page in self.request.uri:
secured_page = True
self.set_header('Content-Length', len(response.body))
self.write(response.body)
break
if response.body and not secured_page:
c.execute('SELECT filter_name FROM filters WHERE filter_type=1')
tags = c.fetchall()
soup = BeautifulSoup(response.body, 'html.parser')
for row in tags:
catched_tags = soup.find_all(str(row[0]))
if catched_tags:
print 'catched: %s of <%s> tags' % (len(catched_tags), str(row[0]))
for tag in catched_tags:
tag.extract()
new_body = str(soup)
self.set_header('Content-Length', len(new_body))
self.write(new_body)
self.finish()
#tornado.web.asynchronous
def get(self):
logger.debug('Handle %s request to %s', self.request.method, self.request.uri)
body = self.request.body
if not body:
body = None
try:
if 'Proxy-Connection' in self.request.headers:
del self.request.headers['Proxy-Connection']
c.execute('SELECT filter_name FROM filters WHERE filter_type=2')
urls = c.fetchall()
for url in urls:
if url[0] in self.request.path:
self.set_status(403)
self.finish()
return
fetch_request(self.request.uri, self.handle_response,
method=self.request.method, body=body, headers=self.request.headers, follow_redirects=False,
allow_nonstandard_methods=True)
except tornado.httpclient.HTTPError as e:
if hasattr(e, 'response') and e.response:
self.handle_response(e.response)
else:
self.set_status(500)
self.write('Internal server error:\n' + str(e))
self.finish()
#tornado.web.asynchronous
def post(self):
return self.get()
And my urls for the application:
urls = [
url(r"/admin/$", mainHandlers.MainHandler),
url(r"/admin/delete_filter/", mainHandlers.DataDeleteHandler),
url(r"/admin/filters/$", mainHandlers.DataGetter),
url(r"/admin/new_filter/$", mainHandlers.FormHandler),
url(r"/admin/stats/$", mainHandlers.StatsTableHandler),
url(r"/admin/stats/query/$", mainHandlers.AjaxStatsGetHandler),
url(r"/static/", StaticFileHandler, dict(path=settings['static_path'])),
url(r'.*', myProxy.ProxyHandler),
]

It says the port is already in use because it is. You're listening on port 8000 at least twice: once in the __main__ block when you call sock.listen, and again in the connection_ready handler when you call app.listen() (which creates another socket and tries to bind it to port 8000). You need to remove the app.listen() line, but I don't understand what you're trying to do well enough to say what you should do instead.

If you start app on Windows, you must wait for the firewall unblock. In windows it is safe to assume that if an application occupies a port it is blocked for use by other processes that might listen to packets not intended for them.

I've rewitten my Proxy to pure Python code on sockets, I'm not using URL's now and I only handle the responses from the remote addresses. I'm not using any framework

Cache Proxy Server Returning 404 with www.google.com

I have a homework assignment which involves implementing a proxy cache server in Python for web pages. Here is my implementation of it
from socket import *
import sys
def main():
#Create a server socket, bind it to a port and start listening
tcpSerSock = socket(AF_INET, SOCK_STREAM) #Initializing socket
tcpSerSock.bind(("", 8030)) #Binding socket to port
tcpSerSock.listen(5) #Listening for page requests
while True:
#Start receiving data from the client
print 'Ready to serve...'
tcpCliSock, addr = tcpSerSock.accept()
print 'Received a connection from:', addr
message = tcpCliSock.recv(1024)
print message
#Extract the filename from the given message
filename = ""
try:
filename = message.split()[1].partition("/")[2].replace("/", "")
except:
continue
fileExist = False
try: #Check whether the file exists in the cache
f = open(filename, "r")
outputdata = f.readlines()
fileExist = True
#ProxyServer finds a cache hit and generates a response message
tcpCliSock.send("HTTP/1.0 200 OK\r\n")
tcpCliSock.send("Content-Type:text/html\r\n")
for data in outputdata:
tcpCliSock.send(data)
print 'Read from cache'
except IOError: #Error handling for file not found in cache
if fileExist == False:
c = socket(AF_INET, SOCK_STREAM) #Create a socket on the proxyserver
try:
srv = getaddrinfo(filename, 80)
c.connect((filename, 80)) #https://docs.python.org/2/library/socket.html
# Create a temporary file on this socket and ask port 80 for
# the file requested by the client
fileobj = c.makefile('r', 0)
fileobj.write("GET " + "http://" + filename + " HTTP/1.0\r\n")
# Read the response into buffer
buffr = fileobj.readlines()
# Create a new file in the cache for the requested file.
# Also send the response in the buffer to client socket and the
# corresponding file in the cache
tmpFile = open(filename,"wb")
for data in buffr:
tmpFile.write(data)
tcpCliSock.send(data)
except:
print "Illegal request"
else: #File not found
print "404: File Not Found"
tcpCliSock.close() #Close the client and the server sockets
main()
I configured my browsers to use my proxy server like so
But my problem when I run it is that no matter what web page I try to access it returns a 404 error with the initial connection and then a connection reset error with subsequent connections. I have no idea why so any help would be greatly appreciated, thanks!

There are quite a number of issues with your code.
Your URL parser is quite cumbersome. Instead of the line
filename = message.split()[1].partition("/")[2].replace("/", "")
I would use
import re
parsed_url = re.match(r'GET\s+http://(([^/]+)(.*))\sHTTP/1.*$', message)
local_path = parsed_url.group(3)
host_name = parsed_url.group(2)
filename = parsed_url.group(1)
If you catch an exception there, you should probably throw an error because it is a request your proxy doesn't understand (e.g. a POST).
When you assemble your request to the destination server, you then use
fileobj.write("GET {object} HTTP/1.0\n".format(object=local_path))
fileobj.write("Host: {host}\n\n".format(host=host_name))
You should also include some of the header lines from the original request because they can make a major difference to the returned content.
Furthermore, you currently cache the entire response with all header lines, so you should not add your own when serving from cache.
What you have doesn't work, anyway, because there is no guarantee that you will get a 200 and text/html content. You should check the response code and only cache if you did indeed get a 200.

python server with images

I'm running an python server that loads a login page. All the html pages will load (they're in the same folder as the html page that's being hosted, so are the images), but the images that are in the same folder as the html don't load.Anybody know what's wrong with the image loading ? Thanks. Here's the code:
#/usr/bin/python
from BaseHTTPServer import BaseHTTPRequestHandler,HTTPServer
from os import curdir, sep
PORT_NUMBER = 8080
#This class will handles any incoming request from
#the browser
class myHandler(BaseHTTPRequestHandler):
#Handler for the GET requests
def do_GET(self):
if self.path=="/":
self.path="login.html"
try:
#Check the file extension required and
#set the right mime type
sendReply = False
if self.path.endswith(".html"):
mimetype='text/html'
sendReply = True
if self.path.endswith(".jpg"):
mimetype='image/jpg'
sendReply = True
if self.path.endswith(".gif"):
mimetype='image/gif'
sendReply = True
if self.path.endswith(".js"):
mimetype='application/javascript'
sendReply = True
if self.path.endswith(".css"):
mimetype='text/css'
sendReply = True
if sendReply == True:
#Open the static file requested and send it
f = open(curdir + sep + self.path)
self.send_response(200)
self.send_header('Content-type',mimetype)
self.end_headers()
self.wfile.write(f.read())
f.close()
return
except IOError:
self.send_error(404,'File Not Found: %s' % self.path)
try:
#Create a web server and define the handler to manage the
#incoming request
server = HTTPServer(('', PORT_NUMBER), myHandler)
print 'Started httpserver on port ' , PORT_NUMBER
#Wait forever for incoming htto requests
server.serve_forever()
except KeyboardInterrupt:
print '^C received, shutting down the web server'
server.socket.close()

You can try the following code for jpg/png files accordingly
if self.path.endswith(".jpg"):
f = open(applicationPath + '/' + self.path, 'rb')
self.send_response(200)
self.send_header('Content-type', 'image/png')
self.end_headers()
self.wfile.write(f.read())
f.close()
return

You have to add mode='rb' to f = open(curdir + sep + self.path) because you are reading a binary file.

Transfer A Text File Via Socket In Python

I have a project in PYTHON that is two machines (A , B) ,
1) request machine A send a request to B to list a directory(in my code i set it to current directory)
2) in second request machine A wants to download the Text file of the directory. (Put a text file in machine B's directory)
3) after that machine A changes the text file and send back to the machine B.
4) and At the end machine A send two number and machine B send back the result of it.
it work till step 2 but nothing happen after that it's like a true while I can't understand why?!
Here is my Code
Machine A (Client):
# -*- coding: UTF-8 -*-
import os
import socket
PORT = 9000
HOST = 'localhost'
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.connect((HOST, PORT))
store=[]
direc = raw_input("Enter The Directory to List : ")
socket.sendall(direc)
len_data = socket.recv(2048)
print int(len_data)
for i in range(int(len_data)):
data = socket.recv(2048)
store.append(data)
print("List of Files:")
for i in store:
print(i)
txt_file = raw_input("Please Choose a TEXT file :")
if store.count(txt_file) is 0:
print("There no such a TXT file!!")
else:
socket.sendall(txt_file)
def write_file(name):
fname = open(name,'ab')
while True:
string = socket.recv(2048)
if string:
fname.write(string)
else:
fname.write("changed")
fname.close()
break
def read_file(name):
fileToSend = open(name, 'rb')
while True:
data = fileToSend.readline()
if data:
socket.send(data)
else:
fileToSend.close()
break
write_file(txt_file)
read_file(txt_file)
x = raw_input("Enter The First Num: ")
socket.send(x)
y = raw_input("Enter The Second Num: ")
socket.send(y)
result = socket.recv(1024)
print result
raw_input()
socket.sendall('')
socket.close()
exit()
and the Machine B (Server):
import os,sys,socket
PORT = 9000
HOST = 'localhost'
tcpsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_address = (HOST, PORT)
print >>sys.stderr, 'starting up on %s port %s' % server_address
socket.bind((HOST,PORT))
socket.listen(1)
conn, addr = socket.accept()
directory = conn.recv(2048)
if os.listdir(os.curdir):
data = os.listdir(os.curdir)
len_data = data.__len__()
print(len_data)
if len_data:
conn.send(str(len_data))
for i in data:
if i:
print >>sys.stderr, 'sending data back to the client'
conn.send(i)
else:
break
txt_file_name = conn.recv(2048)
def write_file(name):
with open(name,'wb') as fname:
while True:
string = conn.recv(2048)
if string:
fname.write(string)
else:
fname.close()
break
def read_file(name):
with open(name, 'rb') as fileToSend:
while True:
data = fileToSend.readline()
if data:
conn.send(data)
else:
fileToSend.close()
break
def add (x,y):
return str(x+y)
read_file(txt_file_name)
write_file(txt_file_name)
x = conn.recv(1024)
y = conn.recv(1024)
conn.send(add(x,y))
conn.sendall('')
conn.close()
exit()

I am fascinated with your problem and looked into it. While we can solve it using socket. I lean toward HTTP protocol for several reasons:
You don't have to make up your own "hand shake". The HTTP protocol has provision for requesting file, uploading a file, and do some processing (your step #4)
You can test your server using a web browser
Web services are very popular now. This is a baby step to learn about web services.
Here is the server code (server.py):
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
import os
class MyHandler(BaseHTTPRequestHandler):
def do_GET(self):
global running
if self.path == '/':
self.list_files()
elif self.path.startswith('/calculation'):
self.send_calculation()
elif self.path.startswith('/quit'):
self.send_response(200)
running = False
else:
self.send_file(self.path[1:])
def do_POST(self):
filename = self.path[1:] # Remove the / from the path
filesize = int(self.headers['Content-Length'])
contents = self.rfile.read(filesize)
with open(filename, 'w') as f:
f.write(contents.decode())
self.send_response(200)
def send_file(self, filename):
# Check to see if file exists and is a file, not directory
if os.path.isfile(filename):
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.end_headers()
# Read and send the contents of the file
with open(filename) as f:
contents = f.read()
self.wfile.write(contents)
else:
self.send_response(404)
self.send_header('Content-Type', 'text/plain')
self.end_headers()
self.wfile.write('Dude! File not found')
def send_calculation(self):
empty, operation, number1, number2 = self.path.split('/')
result = int(number1) + int(number2)
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.end_headers()
self.wfile.write(result)
def list_files(self):
file_list = os.listdir(os.curdir)
if file_list:
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.end_headers()
for filename in file_list:
self.wfile.write('{}\n'.format(filename))
#
# Main
#
running = True
server = HTTPServer(('', 9000), MyHandler)
print 'Server started on host:{}, port:{}'.format(*server.server_address)
while running:
server.handle_request()
And here is the client code (client.py):
import urllib2
import urlparse
def make_url(server, port, path, scheme='http'):
netloc = '{}:{}'.format(server, port)
url = urlparse.urlunsplit((scheme, netloc, path, '', ''))
return url
#
# Main
#
server = '10.0.0.5'
port = 9000
# 1 - Request directory listing
url = make_url(server, port, '/')
file_list = urllib2.urlopen(url).read()
print 'Files from server:'
for filename in file_list.splitlines():
print '- {}'.format(filename)
# 2 - Request contents of a file
filename = raw_input('Type a file name: ')
url = make_url(server, port, filename)
contents = urllib2.urlopen(url).read()
print 'Contents:'
print contents
# 3 - Upload a file to the server
contents = 'hello, world.\nThe End'
filename = 'foo.txt'
url = make_url(server, port, filename)
f = urllib2.urlopen(url, data=contents)
# 4 - Do some calculation
n1 = 19
n2 = 5
path = '/calculation/{}/{}'.format(n1, n2)
url = make_url(server, port, path)
result = int(urllib2.urlopen(url).read())
print '{} + {} = {}'.format(n1, n2, result)
# Send quit signal
url = make_url(server, port, '/quit')
urllib2.urlopen(url).read()
Web Service
The server is really a web service, which provides the following services:
Get a directory listing
GET http://server:port/
This service will return a list of files in the current directory.
Get contents of a file
GET http://server:port/filename
Returns the contents of a file in plain text format.
Upload a file
POST http://server:port/filename
Copy a file from the client to the server. If the file already exists on the server, override it.
Do some calculation
GET http://server:port/calculation/x/y
Returns x + y
Shut down the server
GET http://server:port/quit
Tells the server to quit.
Error Handling
For the sake of brevity and clarity, I did not add and error handling to the code. Here are a few error condition that I can think of:
Retrieve a non-existing file, or a directory (server)
Upload failed because of the lack of file write permission (server)
In the calculation service, the parameters are not numbers (server)
The server has not started, wrong port, wrong server (client)
Other Discussions
In a general, GET means data flow from the server to the client, and POST the opposite direction.
To test GET action from the server, you can use your browser. For example, to retrieve the directory contents from 192.168.1.5, port 9000, point your web browser to:
http://192.168.1.5:900/
Testing POST is trickier, see the client code in the upload section for idea of using POST.
In the server code, the do_GET() function handles all the GET requests, and the do_POST() function handles all the POST requests.

How to serve an mp3 file with built-in python http server

I am currently trying to serve MP3 Files using Python. The problem is that I can only play the MP3 once. Afterwards media controls stop responding and I need to reload entirely the page to be able to listen again to the MP3. (tested in Chrome)
Problem: running the script below, and entering http://127.0.0.1/test.mp3 on my browser will return an MP3 files which can be replayed only if I refresh the page
Notes:
Saving the page as HTML and loading it directly with Chrome (without Python server) would make the problem disappear.
Serving the file with Apache would solve the problem, but this is overkilled: I want to make the script very easy to use and not require installing Apache.
Here is the code I use:
import string
import os
import urllib
import socket
# Setup web server import string,cgi,time
import string,cgi,time
from os import curdir, sep
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import hashlib
class MyHandler(BaseHTTPRequestHandler):
def do_GET(self):
try:
# serve mp3 files
if self.path.endswith(".mp3"):
print curdir + sep + self.path
f = open(curdir + sep + self.path, 'rb')
st = os.fstat( f.fileno() )
length = st.st_size
data = f.read()
md5 = hashlib.md5()
md5.update(data)
md5_key = self.headers.getheader('If-None-Match')
if md5_key:
if md5_key[1:-1] == md5.hexdigest():
self.send_response(304)
self.send_header('ETag', '"{0}"'.format(md5.hexdigest()))
self.send_header('Keep-Alive', 'timeout=5, max=100')
self.end_headers()
return
self.send_response(200)
self.send_header('Content-type', 'audio/mpeg')
self.send_header('Content-Length', length )
self.send_header('ETag', '"{0}"'.format(md5.hexdigest()))
self.send_header('Accept-Ranges', 'bytes')
self.send_header('Last-Modified', time.strftime("%a %d %b %Y %H:%M:%S GMT",time.localtime(os.path.getmtime('test.mp3'))))
self.end_headers()
self.wfile.write(data)
f.close()
return
except IOError:
self.send_error(404,'File Not Found: %s' % self.path)
from SocketServer import ThreadingMixIn
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
pass
if __name__ == "__main__":
try:
server = ThreadedHTTPServer(('', 80), MyHandler)
print 'started httpserver...'
server.serve_forever()
except KeyboardInterrupt:
print '^C received, shutting down server'
server.socket.close()

BaseServer is single-threaded, you should use either ForkingMixIn or ThreadingMixIn to support multiple connections.
For example replace line:
server = HTTPServer(('', 80), MyHandler)
with
from SocketServer import ThreadingMixIn
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
pass
server = ThreadedHTTPServer(('', 80), MyHandler)

EDIT: I wrote much of this before I realized Mapadd only planned to use this in a lab. WSGI probably is not required for his use case.
If you are willing to run this as a wsgi app (which I would recommend over vanilla CGI for any real scalability), you can use the script I have included below.
I took the liberty of modifying your source... this works with the assumptions above.. btw, you should spend some time checking that your html is reasonably compliant... this will help ensure that you get better cross-browser compatibility... the original didn't have <head> or <body> tags... mine (below) is strictly prototype html, and could be improved.
To run this, you just run the python executable in your shell and surf to the ipaddress of the machine on 8080. If you were doing this for a production website, we should be using lighttpd or apache for serving files, but since this is simply for lab use, the embedded wsgi reference server should be fine. Substitute the WSGIServer line at the bottom of the file if you want to run in apache or lighttpd.
Save as mp3.py
from webob import Request
import re
import os
import sys
####
#### Run with:
#### twistd -n web --port 8080 --wsgi mp3.mp3_app
_MP3DIV = """<div id="musicHere"></div>"""
_MP3EMBED = """<embed src="mp3/" loop="true" autoplay="false" width="145" height="60"></embed>"""
_HTML = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"><html><head></head><body> Hello %s %s</body></html> ''' % (_MP3DIV, _MP3EMBED)
def mp3_html(environ, start_response):
"""This function will be mounted on "/" and refer the browser to the mp3 serving URL."""
start_response('200 OK', [('Content-Type', 'text/html')])
return [_HTML]
def mp3_serve(environ, start_response):
"""Serve the MP3, one chunk at a time with a generator"""
file_path = "/file/path/to/test.mp3"
mimetype = "application/x-mplayer2"
size = os.path.getsize(file_path)
headers = [
("Content-type", mimetype),
("Content-length", str(size)),
]
start_response("200 OK", headers)
return send_file(file_path, size)
def send_file(file_path, size):
BLOCK_SIZE = 4096
fh = open(file_path, 'r')
while True:
block = fh.read(BLOCK_SIZE)
if not block:
fh.close()
break
yield block
def _not_found(environ,start_response):
"""Called if no URL matches."""
start_response('404 NOT FOUND', [('Content-Type', 'text/plain')])
return ['Not Found']
def mp3_app(environ,start_response):
"""
The main WSGI application. Dispatch the current request to
the functions andd store the regular expression
captures in the WSGI environment as `mp3app.url_args` so that
the functions from above can access the url placeholders.
If nothing matches call the `not_found` function.
"""
# map urls to functions
urls = [
(r'^$', mp3_html),
(r'mp3/?$', mp3_serve),
]
path = environ.get('PATH_INFO', '').lstrip('/')
for regex, callback in urls:
match = re.search(regex, path)
if match is not None:
# assign http environment variables...
environ['mp3app.url_args'] = match.groups()
return callback(environ, start_response)
return _not_found(environ, start_response)
Run from the bash shell with: twistd -n web --port 8080 --wsgi mp3.mp3_app from the directory where you saved mp3.py (or just put mp3.py somewhere in $PYTHONPATH).
Now surf to the external ip (i.e. http://some.ip.local:8080/) and it will serve the mp3 directly.
I tried running your original app as it was posted, and could not get it to source the mp3, it barked at me with an error in linux...