html5 websocket connecting to python - python

I am trying to connect a html5 websocket to python. I have two files:
1. index.html
<script>
window.onload = function() {
var connection = new WebSocket("ws://localhost:9876/");
connection.onopen = function () {
connection.send('Ping');
};
connection.onerror = function (error) {
console.log('WebSocket Error ' + error);
};
connection.onmessage = function (e) {
console.log('Server: ' + e.data);
};
};
</script>
2. server.py
import socket
HOST = ''
PORT = 9876
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(5)
conn, addr = s.accept()
print 'Connected by', addr
conn.send("Hello")
while 1:
data = conn.recv(4096)
print data
if not data: break
conn.sendall(data)
conn.close()
After starting server.py, it keeps running until I open index.html in my browser. When I do so, I get on the console the following:
WebSocket connection to 'ws://localhost:9876/' failed:
WebSocket Error [object Event]
However, on the other side(server.py) I get:
Connected by('127.0.0.1', 55460)GET / HTTP/1.1
Upgrade: websocket
Connection: Upgrade
Host: localhost:9876
Origin: null
Pragma: no-cache
Cache-Control: no-cache
Sec-WebSocket-Key: WP0GtT0hrdX1bKa0DepOHQ==
Sec-WebSocket-Version: 13
Sec-WebSocket-Extensions: x-webkit-deflate-frame
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36
Q What's wrong with my codes? How should I modify it so as to have biorder communication between them?
I am looking for something like this:
if data == 'kill': conn.close()
important edit
If I change in the index.html WebSocket("ws://localhost:9876/") to WebSocket("ws://html5rocks.websocket.org/echo") than it works. I guess that means that to problem is on the server.py side
clue #2
If I write WebSocket("ws://localhost/:9876") in the html, the python code cannot proceed the line conn, addr = s.accept()

You haven't response headers.
try ,
server.py
import socket, hashlib, base64, threading
class PyWSock:
MAGIC = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
HSHAKE_RESP = "HTTP/1.1 101 Switching Protocols\r\n" + \
"Upgrade: websocket\r\n" + \
"Connection: Upgrade\r\n" + \
"Sec-WebSocket-Accept: %s\r\n" + \
"\r\n"
LOCK = threading.Lock()
clients = []
def recv_data (self, client):
# as a simple server, we expect to receive:
# - all data at one go and one frame
# - one frame at a time
# - text protocol
# - no ping pong messages
data = bytearray(client.recv(512))
if(len(data) < 6):
raise Exception("Error reading data")
# FIN bit must be set to indicate end of frame
assert(0x1 == (0xFF & data[0]) >> 7)
# data must be a text frame
# 0x8 (close connection) is handled with assertion failure
assert(0x1 == (0xF & data[0]))
# assert that data is masked
assert(0x1 == (0xFF & data[1]) >> 7)
datalen = (0x7F & data[1])
#print("received data len %d" %(datalen,))
str_data = ''
if(datalen > 0):
mask_key = data[2:6]
masked_data = data[6:(6+datalen)]
unmasked_data = [masked_data[i] ^ mask_key[i%4] for i in range(len(masked_data))]
str_data = str(bytearray(unmasked_data))
return str_data
def broadcast_resp(self, data):
# 1st byte: fin bit set. text frame bits set.
# 2nd byte: no mask. length set in 1 byte.
resp = bytearray([0b10000001, len(data)])
# append the data bytes
for d in bytearray(data):
resp.append(d)
self.LOCK.acquire()
for client in self.clients:
try:
client.send(resp)
except:
print("error sending to a client")
self.LOCK.release()
def parse_headers (self, data):
headers = {}
lines = data.splitlines()
for l in lines:
parts = l.split(": ", 1)
if len(parts) == 2:
headers[parts[0]] = parts[1]
headers['code'] = lines[len(lines) - 1]
return headers
def handshake (self, client):
print('Handshaking...')
data = client.recv(2048)
headers = self.parse_headers(data)
print('Got headers:')
for k, v in headers.iteritems():
print k, ':', v
key = headers['Sec-WebSocket-Key']
resp_data = self.HSHAKE_RESP % ((base64.b64encode(hashlib.sha1(key+self.MAGIC).digest()),))
print('Response: [%s]' % (resp_data,))
return client.send(resp_data)
def handle_client (self, client, addr):
self.handshake(client)
try:
while 1:
data = self.recv_data(client)
print("received [%s]" % (data,))
self.broadcast_resp(data)
except Exception as e:
print("Exception %s" % (str(e)))
print('Client closed: ' + str(addr))
self.LOCK.acquire()
self.clients.remove(client)
self.LOCK.release()
client.close()
def start_server (self, port):
s = socket.socket()
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('', port))
s.listen(5)
while(1):
print ('Waiting for connection...')
conn, addr = s.accept()
print ('Connection from: ' + str(addr))
threading.Thread(target = self.handle_client, args = (conn, addr)).start()
self.LOCK.acquire()
self.clients.append(conn)
self.LOCK.release()
ws = PyWSock()
ws.start_server(9876)
link
I think minimal server.py
import socket,hashlib,base64
MAGIC = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
HSHAKE_RESP = "HTTP/1.1 101 Switching Protocols\r\n" + \
"Upgrade: websocket\r\n" + \
"Connection: Upgrade\r\n" + \
"Sec-WebSocket-Accept: %s\r\n" + \
"\r\n"
HOST = ''
PORT = 9876
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(5)
conn, addr = s.accept()
print 'Connected by', addr
data = conn.recv(4096)
headers = {}
lines = data.splitlines()
for l in lines:
parts = l.split(": ", 1)
if len(parts) == 2:
headers[parts[0]] = parts[1]
headers['code'] = lines[len(lines) - 1]
key = headers['Sec-WebSocket-Key']
resp_data = HSHAKE_RESP % ((base64.b64encode(hashlib.sha1(key+MAGIC).digest()),))
conn.send(resp_data)
while 1:
data = conn.recv(4096)
if not data:
break
databyte = bytearray(data)
datalen = (0x7F & databyte[1])
str_data = ''
if(datalen > 0):
mask_key = databyte[2:6]
masked_data = databyte[6:(6+datalen)]
unmasked_data = [masked_data[i] ^ mask_key[i%4] for i in range(len(masked_data))]
str_data = str(bytearray(unmasked_data))
print str_data
resp = bytearray([0b10000001, len(str_data)])
for d in bytearray(str_data):
resp.append(d)
conn.sendall(resp)
conn.close()
One time running. Because not threading.

Related

Chat server and chat client

This program has both the ChatServer class and ChatClient class in the same file, and should be called in the terminal by --name=server --port=8800 for the server and --name=client1 --port=8800 for the client. The problem comes from the client class not being able to complete a try:
When running the program with server name, it seems to work fine. When it runs with client name, I get the output Failed to connect to chat server # port 8800.
You can find where this except statement lies.
import select
import socket
import sys
import signal
import _pickle as cPickle
import struct
import argparse
SERVER_HOST = 'localhost'
CHAT_SERVER_NAME = 'server'
# Some utilities
def send(channel, *args):
buffer = cPickle.dumps(args)
value = socket.htonl(len(buffer))
size = struct.pack("L", value)
channel.send(size)
channel.send(buffer)
def receive(channel):
size = struct.calcsize("L")
size = channel.recv(size)
try:
size = socket.ntohl(struct.unpack("L", size)[0])
except struct.error as e:
return ''
buf = ""
while len(buf) < size:
buf = channel.recv(size - len(buf))
return cPickle.loads(buf)[0]
class ChatServer(object):
def __init__(self, port, backlog=5):
self.clients = 0
self.clientmap = {}
self.outputs = []
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Enable re-using socket address
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server.bind((SERVER_HOST, port))
print('Server listening to port: %s...' % port)
self.server.listen(backlog)
# Catch keyboard interrupts
signal.signal(signal.SIGINT, self.sighandler)
def sighandler(self, signum, frame):
# Close the server
print("Shutting down server...")
# Close existing client sockets
for output in self.outputs:
output.close()
self.server.close()
def get_client_name(self,client):
info = self.clientmap[client]
host, name = info[0][0], info[1]
return '#'.join((name, host))
def run(self):
inputs = [self.server, sys.stdin]
self.outputs = []
running = True
while running:
try:
readable, writeable, exceptional = select.select(inputs, self.outputs, [])
except select.error as e:
break
for sock in readable:
if sock == self.server:
# handle the server socket
client, address = self.server.accept()
print("Chat Server: got connection %d from %s" % (client.fileno(), address))
# Read the login name
cname = receive(client).split('NAME: ')[1]
# Compute client name ad send back
self.clients += 1
send(client, 'CLIENT: ' + str(address[0]))
inputs.append(client)
self.clientmap[client] = (address, cname)
# Send joining information to other clients
msg = "\n(Connected: New client (%d) from %s)" % (self.clients, self.get_client_name(client))
for output in self.outputs:
send(output, msg)
self.outputs.append(client)
elif sock == sys.stdin:
# Handle standard input
junk = sys.stdin.readline()
running = False
else:
# Handle all other sockets
try:
data = receive(sock)
if data:
# Send as new client's message..
msg = '\n[' + self.get_client_name(sock) + ']>>' + data
# Send data to all except ourself
for output in self.outputs:
if output != sock:
send(output, msg)
else:
print("Chat server: %d hung up" % sock.fileno())
self.clients -= 1
sock.close()
inputs.remove(sock)
self.outputs.remove(sock)
# Sending client leaving info to others
msg = "\n(Now hung up: Client from %s" % self.get_client_name(sock)
for output in self.outputs:
send(output, msg)
except socket.error as e:
# Remove
inputs.remove(sock)
self.outputs.remove(sock)
self.server.close()
class ChatClient(object):
def __init__(self, name, port, host=SERVER_HOST):
self.name = name
self.connected = False
self.host = host
self.port = port
# Initial Prompt
self.prompt = '[' + '#'.join((name, socket.gethostname().split('.')[0])) + ']> '
# Connect to server at port
try:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((host, self.port))
print("Now connected to chat server# port %d" % self.port)
self.connected = True
# Send by name...
send(self.sock, 'NAME: ' + self.name)
data = receive(self.sock)
# Contains client address, set it
addr = data.split('CLIENT: ')[1]
self.prompt = '[' + '#'.join((self.name, addr)) + ']>'
except socket.error as e:
print("Failed to connect to chat server # port %d" % self.port)
sys.exit(1)
def run(self):
while self.connected:
try:
sys.stdout.write(self.prompt)
sys.stdout.flush()
# Wait for input from stdin and socket
readable, writable, exceptional = select.select([0, self.sock], [], [])
for sock in readable:
if sock == 0:
data = sys.stdin.readline().strip()
if data:
send(self.sock, data)
elif sock == self.sock:
data = receive(self.sock)
if not data:
print('Client shutting down')
self.connected = False
break
else:
sys.stdout.write(data + '\n')
sys.stdout.flush()
except KeyboardInterrupt:
print("Client interrupted")
self.sock.close()
break
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Socket Server Example with Select')
parser.add_argument('--name', action="store", dest="name", required=True)
parser.add_argument('--port', action="store", dest="port", type=int, required=True)
given_args = parser.parse_args()
port = given_args.port
name = given_args.name
if name == CHAT_SERVER_NAME:
server = ChatServer(port)
server.run()
else:
client = ChatClient(name=name, port=port)
client.run()

I got connection refused for python TCP HolePunching

This is the code in serverside:
#!/usr/bin/env python
import sys
import logging
import socket
import struct
import fcntl
import os
from util import *
logger = logging.getLogger()
clients = {}
def main(host='**client public ip**', port=3443):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, port))
s.listen(1)
s.settimeout(30)
while True:
try:
conn, addr = s.accept()
except socket.timeout:
continue
logger.info('connection address: %s', addr)
data = recv_msg(conn)
priv_addr = msg_to_addr(data)
send_msg(conn, addr_to_msg(addr))
data = recv_msg(conn)
data_addr = msg_to_addr(data)
if data_addr == addr:
logger.info('client reply matches')
clients[addr] = Client(conn, addr, priv_addr)
else:
logger.info('client reply did not match')
conn.close()
logger.info('server - received data: %s', data)
if len(clients) == 2:
(addr1, c1), (addr2, c2) = clients.items()
logger.info('server - send client info to: %s', c1.pub)
send_msg(c1.conn, c2.peer_msg())
logger.info('server - send client info to: %s', c2.pub)
send_msg(c2.conn, c1.peer_msg())
clients.pop(addr1)
clients.pop(addr2)
conn.close()
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
main(*addr_from_args(sys.argv))
and this is clientside:
#!/usr/bin/env python
import sys
import logging
import socket
import struct
from threading import Event, Thread
from util import *
logger = logging.getLogger('client')
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
STOP = Event()
def accept(port):
logger.info("accept %s", port)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
s.bind(('**server's public ip**', port))
s.listen(1)
s.settimeout(5)
while not STOP.is_set():
try:
conn, addr = s.accept()
except socket.timeout:
continue
else:
logger.info("Accept %s connected!", port)
# STOP.set()
def connect(local_addr, addr):
logger.info("connect from %s to %s", local_addr, addr)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
s.bind(local_addr)
while not STOP.is_set():
try:
s.connect(addr)
except socket.error:
continue
# except Exception as exc:
# logger.exception("unexpected exception encountered")
# break
else:
logger.info("connected from %s to %s success!", local_addr, addr)
# STOP.set()
def main(host='**client's local ip**', port=3443):
sa = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sa.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sa.connect((host, port))
priv_addr = sa.getsockname()
send_msg(sa, addr_to_msg(priv_addr))
data = recv_msg(sa)
logger.info("client %s %s - received data: %s", priv_addr[0], priv_addr[1], data)
pub_addr = msg_to_addr(data)
send_msg(sa, addr_to_msg(pub_addr))
data = recv_msg(sa)
pubdata, privdata = data.split(b'|')
client_pub_addr = msg_to_addr(pubdata)
client_priv_addr = msg_to_addr(privdata)
logger.info(
"client public is %s and private is %s, peer public is %s private is %s",
pub_addr, priv_addr, client_pub_addr, client_priv_addr,
)
threads = {
'0_accept': Thread(target=accept, args=(priv_addr[1],)),
'1_accept': Thread(target=accept, args=(client_pub_addr[1],)),
'2_connect': Thread(target=connect, args=(priv_addr, client_pub_addr,)),
'3_connect': Thread(target=connect, args=(priv_addr, client_priv_addr,)),
}
for name in sorted(threads.keys()):
logger.info('start thread %s', name)
threads[name].start()
while threads:
keys = list(threads.keys())
for name in keys:
try:
threads[name].join(1)
except TimeoutError:
continue
if not threads[name].is_alive():
threads.pop(name)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, message='%(asctime)s %(message)s')
main(*addr_from_args(sys.argv))
and this is the util.py code:
import struct
from collections import namedtuple
def addr_from_args(args, host='**server's local ip**', port=9999):
if len(args) >= 3:
host, port = args[1], int(args[2])
elif len(args) == 2:
host, port = host, int(args[1])
else:
host, port = host, port
return host, port
def msg_to_addr(data):
ip, port = data.decode('utf-8').strip().split(':')
return (ip, int(port))
def addr_to_msg(addr):
return '{}:{}'.format(addr[0], str(addr[1])).encode('utf-8')
def send_msg(sock, msg):
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = b''
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data += packet
return data
def recv_msg(sock):
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
class Client(namedtuple('Client', 'conn, pub, priv')):
def peer_msg(self):
return addr_to_msg(self.pub) + b'|' + addr_to_msg(self.priv)
and this is the error:
Traceback (most recent call last):
File "tcpclient.py", line 96, in <module>
main(*addr_from_args(sys.argv))
File "tcpclient.py", line 55, in main
sa.connect((host, port))
ConnectionRefusedError: [Errno 111] Connection refused
It's working in same network but in different one not!!
For adding more details I would say that we have one client and on server that wanna communicate to each other, client and server are both behind the nat and the server has webserver and client want to go to the webserver but because of the nat problem it can't. so we are trying to make a hole punching peer to peer connection to make client able to see the webserver.
we can't take any kind of ip static or commercial web service or isp or what ever that everyone say.
what's wrong with this code?
in util.py the ip should be public not private
import struct
from collections import namedtuple
def addr_from_args(args, host='**server's ip public**', port=9999):
if len(args) >= 3:
host, port = args[1], int(args[2])
elif len(args) == 2:
host, port = host, int(args[1])
else:
host, port = host, port
return host, port
def msg_to_addr(data):
ip, port = data.decode('utf-8').strip().split(':')
return (ip, int(port))
def addr_to_msg(addr):
return '{}:{}'.format(addr[0], str(addr[1])).encode('utf-8')
def send_msg(sock, msg):
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = b''
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data += packet
return data
def recv_msg(sock):
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
class Client(namedtuple('Client', 'conn, pub, priv')):
def peer_msg(self):
return addr_to_msg(self.pub) + b'|' + addr_to_msg(self.priv)

How to send files in "chunks" by socket?

I'm trying to send a large file (.avi) over socket by sending the content of the file in chunks (a little bit like torrents). The problem is that the script doesn't send the file. I'm out of ideas here.
Any help or twerking of the script would be very appreciated.
Server:
import socket
HOST = ""
PORT = 8050
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((HOST, PORT))
sock.listen(1)
conn, addr = sock.accept()
print("Connected by ", str(addr))
while 1:
data = conn.recv(1024)
if data.decode("utf-8") == 'GET':
with open(downFile,'rb') as output:
l = output.read(1024)
while (l):
conn.send(l)
l = output.read(1024)
output.close()
conn.close()
Client:
import socket
HOST = "localhost"
PORT = 8050
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST,PORT))
while 1:
message = input()
sock.send(bytes(message,'UTF-8'))
conn.send(str.encode('GET'))
with open(downFile, 'wb+') as output:
while True:
rec = str(sock.recv(1024), "utf-8")
if not rec:
break
output.write(rec)
output.close()
print('Success!')
sock.close()
Here are a working client and server that should demonstrate transferring a file over a socket. I made some assumptions about what your code was supposed to do, for example, I assumed that the initial message the client sent to the server was supposed to be the name of the file to download.
The code also includes some additional functionality for the server to return an error message to the client. Before running the code, make sure the directory specified by DOWNLOAD_DIR exists.
Client:
import socket
import sys
import os
HOST = "localhost"
PORT = 8050
BUF_SIZE = 4096
DOWNLOAD_DIR = "downloads"
def download_file(s, down_file):
s.send(str.encode("GET\n" + down_file))
rec = s.recv(BUF_SIZE)
if not rec:
return "server closed connection"
if rec[:2].decode("utf-8") != 'OK':
return "server error: " + rec.decode("utf-8")
rec = rec[:2]
if DOWNLOAD_DIR:
down_file = os.path.join(DOWNLOAD_DIR, down_file)
with open(down_file, 'wb') as output:
if rec:
output.write(rec)
while True:
rec = s.recv(BUF_SIZE)
if not rec:
break
output.write(rec)
print('Success!')
return None
if DOWNLOAD_DIR and not os.path.isdir(DOWNLOAD_DIR):
print('no such directory "%s"' % (DOWNLOAD_DIR,), file=sys.stderr)
sys.exit(1)
while 1:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except Exception as e:
print("cannot connect to server:", e, file=sys.stderr)
break
file_name = input("\nFile to get: ")
if not file_name:
sock.close()
break
err = download_file(sock, file_name)
if err:
print(err, file=sys.stderr)
sock.close()
Server:
import socket
import sys
import os
HOST = ""
PORT = 8050
BUF_SIZE = 4096
def recv_dl_file(conn):
data = conn.recv(1024)
if not data:
print("Client finished")
return None, None
# Get command and filename
try:
cmd, down_file = data.decode("utf-8").split("\n")
except:
return None, "cannot parse client request"
if cmd != 'GET':
return None, "unknown command: " + cmd
print(cmd, down_file)
if not os.path.isfile(down_file):
return None, 'no such file "%s"'%(down_file,)
return down_file, None
def send_file(conn):
down_file, err = recv_dl_file(conn)
if err:
print(err, file=sys.stderr)
conn.send(bytes(err, 'utf-8'))
return True
if not down_file:
return False # client all done
# Tell client it is OK to receive file
sent = conn.send(bytes('OK', 'utf-8'))
total_sent = 0
with open(down_file,'rb') as output:
while True:
data = output.read(BUF_SIZE)
if not data:
break
conn.sendall(data)
total_sent += len(data)
print("finished sending", total_sent, "bytes")
return True
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((HOST, PORT))
sock.listen(1)
keep_going = 1
while keep_going:
conn, addr = sock.accept()
print("Connected by", str(addr))
keep_going = send_file(conn)
conn.close() # close clien connection
print()
sock.close() # close listener

Python proxy HTTP very slow

I'm trying to create a HTTP proxy with Python.
Although, when I configure my browser with this proxy and try to connect to a http website, the loading is very slow.
This is my proxy code :
#!/usr/bin/env python
import socket
import thread
import sys
def get_line(field, data) :
lines = data.split("\n")
for line in lines :
if line.split(":")[0] == field :
return line
return ""
def get_host(data) :
host_line = get_line("Host", data);
port = 80
if len(host_line.split(" ")) > 1 :
host = host_line.split(" ")[1]
arr = host.split(":")
if len(arr) > 1 :
host = arr[0]
port = arr[1]
else :
host = host[:-1]
else :
host = ""
port = 0
return (host, port)
def http_proxy(conn, data, client_addr) :
(host, port) = get_host(data)
if host != "" :
try :
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((socket.gethostbyname(host), int(port)))
s.send(data)
while 1 :
reply = s.recv(8192)
test = reply
if (len(reply) > 0) :
conn.send(reply)
print "Send %d bytes from %s" % (len(reply), host)
else :
conn.close()
s.close()
break
except socket.error, (value, message) :
s.close()
conn.close()
print value, message
sys.exit(1)
def main():
try :
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 8080))
s.listen(100)
except socket.error, (value, message) :
if s :
s.close()
print "Socket error : ", message
while 1 :
conn, client_addr = s.accept()
data = conn.recv(8192)
thread.start_new_thread(http_proxy, (conn, data, client_addr))
s.close()
if __name__ == "__main__" :
main()
I think these delays are caused by the fact that some HTTP requests are not taken into account by my proxy, so the browser waits a certain time before sending them back.
If that's the problem, I don't understand why that happens.

What is the proper way of sending a large amount of data over sockets in Python?

Recently I wrote some code (client and server) to send an image - the client simply uploads the image to the server, just using the socket module: Sending image over sockets (ONLY) in Python, image can not be open.
However, the image sending part is now what I am concerned with. This is the original image I'm using:
In my server code (which receives the images), I have these lines:
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
data = sock.recv(40960000)
if not data:
myfile.close()
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
But I don't think this is the best way of doing it. I think I should instead implement the server such that it receives the data in chunks:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = 0
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()
But when I do this, the server prints:
got size 54674
4096
8192
12288
16384
20480
24576
28672
32768
36864
40960
45056
49152
50578
And then seems to hang, and the client hangs too. However, at the server's side I can see only a piece of the image the client wanted to send:
It seems like there are some bytes missing. What is the proper way of sending a huge amount of data (images, other type of file) using ONLY sockets?
I'm assuming that you have a particular reason for doing this with naked sockets, such as self-edification, which means that I won't answer by saying "You accidentally forgot to just use HTTP and Twisted", which perhaps you've heard before :-P. But really you should look at higher-level libraries at some point as they're a lot easier!
Define a protocol
If all you want is to send an image, then it can be simple:
Client -> server: 8 bytes: big endian, length of image.
Client -> server: length bytes: all image data.
(Client <- server: 1 byte, value 0: indicate transmission received - optional step you may not care if you're using TCP and just assume that it's reliable.)
Code it
server.py
import os
from socket import *
from struct import unpack
class ServerProtocol:
def __init__(self):
self.socket = None
self.output_dir = '.'
self.file_num = 1
def listen(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.bind((server_ip, server_port))
self.socket.listen(1)
def handle_images(self):
try:
while True:
(connection, addr) = self.socket.accept()
try:
bs = connection.recv(8)
(length,) = unpack('>Q', bs)
data = b''
while len(data) < length:
# doing it in batches is generally better than trying
# to do it all in one go, so I believe.
to_read = length - len(data)
data += connection.recv(
4096 if to_read > 4096 else to_read)
# send our 0 ack
assert len(b'\00') == 1
connection.sendall(b'\00')
finally:
connection.shutdown(SHUT_WR)
connection.close()
with open(os.path.join(
self.output_dir, '%06d.jpg' % self.file_num), 'w'
) as fp:
fp.write(data)
self.file_num += 1
finally:
self.close()
def close(self):
self.socket.close()
self.socket = None
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
sp = ServerProtocol()
sp.listen('127.0.0.1', 55555)
sp.handle_images()
client.py
from socket import *
from struct import pack
class ClientProtocol:
def __init__(self):
self.socket = None
def connect(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.connect((server_ip, server_port))
def close(self):
self.socket.shutdown(SHUT_WR)
self.socket.close()
self.socket = None
def send_image(self, image_data):
# use struct to make sure we have a consistent endianness on the length
length = pack('>Q', len(image_data))
# sendall to make sure it blocks if there's back-pressure on the socket
self.socket.sendall(length)
self.socket.sendall(image_data)
ack = self.socket.recv(1)
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
cp = ClientProtocol()
image_data = None
with open('IMG_0077.jpg', 'r') as fp:
image_data = fp.read()
assert(len(image_data))
cp.connect('127.0.0.1', 55555)
cp.send_image(image_data)
cp.close()
A simple way is to send data size as the first 4 bytes of your data and then read complete data in one shot. Use the below functions on both client and server-side to send and receive data.
def send_data(conn, data):
serialized_data = pickle.dumps(data)
conn.sendall(struct.pack('>I', len(serialized_data)))
conn.sendall(serialized_data)
def receive_data(conn):
data_size = struct.unpack('>I', conn.recv(4))[0]
received_payload = b""
reamining_payload_size = data_size
while reamining_payload_size != 0:
received_payload += conn.recv(reamining_payload_size)
reamining_payload_size = data_size - len(received_payload)
data = pickle.loads(received_payload)
return data
you could find sample program at https://github.com/vijendra1125/Python-Socket-Programming.git
The problem is you are not incrementing amount_received for the first chunk of the data received.
Fix below:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = len(data) # The fix!
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()

Categories

Resources