I'm trying to create a HTTP proxy with Python.
Although, when I configure my browser with this proxy and try to connect to a http website, the loading is very slow.
This is my proxy code :
#!/usr/bin/env python
import socket
import thread
import sys
def get_line(field, data) :
lines = data.split("\n")
for line in lines :
if line.split(":")[0] == field :
return line
return ""
def get_host(data) :
host_line = get_line("Host", data);
port = 80
if len(host_line.split(" ")) > 1 :
host = host_line.split(" ")[1]
arr = host.split(":")
if len(arr) > 1 :
host = arr[0]
port = arr[1]
else :
host = host[:-1]
else :
host = ""
port = 0
return (host, port)
def http_proxy(conn, data, client_addr) :
(host, port) = get_host(data)
if host != "" :
try :
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((socket.gethostbyname(host), int(port)))
s.send(data)
while 1 :
reply = s.recv(8192)
test = reply
if (len(reply) > 0) :
conn.send(reply)
print "Send %d bytes from %s" % (len(reply), host)
else :
conn.close()
s.close()
break
except socket.error, (value, message) :
s.close()
conn.close()
print value, message
sys.exit(1)
def main():
try :
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 8080))
s.listen(100)
except socket.error, (value, message) :
if s :
s.close()
print "Socket error : ", message
while 1 :
conn, client_addr = s.accept()
data = conn.recv(8192)
thread.start_new_thread(http_proxy, (conn, data, client_addr))
s.close()
if __name__ == "__main__" :
main()
I think these delays are caused by the fact that some HTTP requests are not taken into account by my proxy, so the browser waits a certain time before sending them back.
If that's the problem, I don't understand why that happens.
Related
I'm trying to send a large file (.avi) over socket by sending the content of the file in chunks (a little bit like torrents). The problem is that the script doesn't send the file. I'm out of ideas here.
Any help or twerking of the script would be very appreciated.
Server:
import socket
HOST = ""
PORT = 8050
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((HOST, PORT))
sock.listen(1)
conn, addr = sock.accept()
print("Connected by ", str(addr))
while 1:
data = conn.recv(1024)
if data.decode("utf-8") == 'GET':
with open(downFile,'rb') as output:
l = output.read(1024)
while (l):
conn.send(l)
l = output.read(1024)
output.close()
conn.close()
Client:
import socket
HOST = "localhost"
PORT = 8050
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST,PORT))
while 1:
message = input()
sock.send(bytes(message,'UTF-8'))
conn.send(str.encode('GET'))
with open(downFile, 'wb+') as output:
while True:
rec = str(sock.recv(1024), "utf-8")
if not rec:
break
output.write(rec)
output.close()
print('Success!')
sock.close()
Here are a working client and server that should demonstrate transferring a file over a socket. I made some assumptions about what your code was supposed to do, for example, I assumed that the initial message the client sent to the server was supposed to be the name of the file to download.
The code also includes some additional functionality for the server to return an error message to the client. Before running the code, make sure the directory specified by DOWNLOAD_DIR exists.
Client:
import socket
import sys
import os
HOST = "localhost"
PORT = 8050
BUF_SIZE = 4096
DOWNLOAD_DIR = "downloads"
def download_file(s, down_file):
s.send(str.encode("GET\n" + down_file))
rec = s.recv(BUF_SIZE)
if not rec:
return "server closed connection"
if rec[:2].decode("utf-8") != 'OK':
return "server error: " + rec.decode("utf-8")
rec = rec[:2]
if DOWNLOAD_DIR:
down_file = os.path.join(DOWNLOAD_DIR, down_file)
with open(down_file, 'wb') as output:
if rec:
output.write(rec)
while True:
rec = s.recv(BUF_SIZE)
if not rec:
break
output.write(rec)
print('Success!')
return None
if DOWNLOAD_DIR and not os.path.isdir(DOWNLOAD_DIR):
print('no such directory "%s"' % (DOWNLOAD_DIR,), file=sys.stderr)
sys.exit(1)
while 1:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except Exception as e:
print("cannot connect to server:", e, file=sys.stderr)
break
file_name = input("\nFile to get: ")
if not file_name:
sock.close()
break
err = download_file(sock, file_name)
if err:
print(err, file=sys.stderr)
sock.close()
Server:
import socket
import sys
import os
HOST = ""
PORT = 8050
BUF_SIZE = 4096
def recv_dl_file(conn):
data = conn.recv(1024)
if not data:
print("Client finished")
return None, None
# Get command and filename
try:
cmd, down_file = data.decode("utf-8").split("\n")
except:
return None, "cannot parse client request"
if cmd != 'GET':
return None, "unknown command: " + cmd
print(cmd, down_file)
if not os.path.isfile(down_file):
return None, 'no such file "%s"'%(down_file,)
return down_file, None
def send_file(conn):
down_file, err = recv_dl_file(conn)
if err:
print(err, file=sys.stderr)
conn.send(bytes(err, 'utf-8'))
return True
if not down_file:
return False # client all done
# Tell client it is OK to receive file
sent = conn.send(bytes('OK', 'utf-8'))
total_sent = 0
with open(down_file,'rb') as output:
while True:
data = output.read(BUF_SIZE)
if not data:
break
conn.sendall(data)
total_sent += len(data)
print("finished sending", total_sent, "bytes")
return True
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((HOST, PORT))
sock.listen(1)
keep_going = 1
while keep_going:
conn, addr = sock.accept()
print("Connected by", str(addr))
keep_going = send_file(conn)
conn.close() # close clien connection
print()
sock.close() # close listener
I'm new to sockets. I've written a simple 'proxy' server in Python that will just catch the data from the remote server and send it to my client (browser). I was wondering: is there a way to send the response_text without a time.sleep? While i try to delete time.sleep(0.5) I only get one package of data from the remote server so the 'Content-lenght' isn't equal to the length of the package and I get an error (I'm using recv() to get a buffer with size equaled to buffer_size, so if the server data needs more then one package of 4096 bytes I need to catch it in the next package). With the time.sleep i get all the packages of data from the remote server and I can send the data to my browser. Am I doing something wrong? Or I just don't know enough? Can someone help?
The code:
# coding: utf-8
import socket
import sys
import time
from thread import *
max_conn = 5
buffer_size = 4096
def proxy_server(webserver, port, conn, addr, data):
try:
remote_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
remote_sock.connect((webserver, port))
remote_sock.send(data)
response_text = ''
while True:
time.sleep(0.5)
response = remote_sock.recv(buffer_size)
response_text += response
if len(response) < buffer_size:
remote_sock.close()
break
conn.sendall(response_text)
conn.close()
except socket.error, msg:
print 'Proccessing error. Error Code: ' + str(msg[0]) + ', Wiadomość: ' + msg[1]
remote_sock.close()
conn.close()
sys.exit()
def conn_string(conn, data, address):
header = data.split('\r\n')
method, address, protocol = header[0].split(' ')
host_key, host_value = header[1].split(': ')
proxy_server(host_value, 80, conn, address, data)
def start():
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 8001))
sock.listen(max_conn)
print 'Proxy: on'
except socket.error, msg:
print 'Failed creating a socket. Error Code: ' + str(msg[0]) + ', Wiadomość: ' + msg[1]
sys.exit()
while True:
try:
connection, address = sock.accept()
data = connection.recv(buffer_size)
# start_new_thread(conn_string, (connection, data, address))
conn_string(connection, data, address)
except KeyboardInterrupt:
sock.close()
print "Socket closed"
sys.exit()
if __name__ == "__main__":
start()
Don't use time.sleep() , it makes your proxy very slow , and its not efficient .
You need to set your socket in non-blocking mode , with a timeout .
You can do this with socket.settimeout()
I made a few modifications to your proxy_server , it should be much faster now .
def proxy_server(webserver, port, conn, addr, data):
try:
remote_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
remote_sock.connect((webserver, port))
remote_sock.send(data)
remote_sock.settimeout(0.5)
response_text = ''
while True:
try :
response = remote_sock.recv(buffer_size)
if len(response) == 0:
break
except :
break
response_text += response
conn.sendall(response_text)
except socket.error, msg:
if str(msg) != 'timed out' :
print 'Proccessing error. Error Code: ' + str(msg[0]) + ', Wiadomość: ' + msg[1]
remote_sock.close()
conn.close()
The rest of your code is quite ok , but you may want to use multithreading
if you want to handle multiple clients at the same time .
attempting message encryption with a basic client to host connection
client code:
import socket
import datetime
import time
import threading
tLock = threading.Lock()
shutdown = False
def receving(name, sock):
while not shutdown:
try:
tLock.acquire()
while True:
data, addr = socket.recvfrom(1024)
print (str(data))
except:
pass
finally:
tLock.release()
host = '127.0.0.1'
port = 0
server = ('127.0.0.1',5000)
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind((host, port))
s.setblocking(0)
rT = threading.Thread(target=receving, args=("RecvThread",s))
rT.start()
alias = input("Name: ")
IP=int(socket.gethostbyname(socket.gethostname()).replace(".","5"))
time=(int(datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
qw=(int(str((time)+(IP))))
a=int("934")
b=int("346")
c=int("926")
d=int("9522")
e=int("7334")
f=int("5856")
g=int("2432")
h=int("2027")
i=int("7024")
j=int("828")
k=int("798")
m=int("593")
n=int("662")
l=int("5950")
o=int("357")
p=int("506")
q=int("237")
r=int("98")
s=int("372")
t=int("636")
u=int("553")
v=int("255")
w=int("298")
x=int("8822")
y=int("458")
z=int("657")
space=("633")
msg=input("")
msg=msg.replace("a",(str(a)))
msg=msg.replace("b",(str(b)))
msg=msg.replace("c",(str(c)))
msg=msg.replace("d",(str(d)))
msg=msg.replace("e",(str(e)))
msg=msg.replace("f",(str(f)))
msg=msg.replace("g",(str(g)))
msg=msg.replace("h",(str(h)))
msg=msg.replace("i",(str(i)))
msg=msg.replace("j",(str(j)))
msg=msg.replace("k",(str(k)))
msg=msg.replace("m",(str(m)))
msg=msg.replace("n",(str(n)))
msg=msg.replace("l",(str(l)))
msg=msg.replace("o",(str(o)))
msg=msg.replace("p",(str(p)))
msg=msg.replace("q",(str(q)))
msg=msg.replace("r",(str(r)))
msg=msg.replace("s",(str(s)))
msg=msg.replace("t",(str(t)))
msg=msg.replace("u",(str(u)))
msg=msg.replace("v",(str(v)))
msg=msg.replace("w",(str(w)))
msg=msg.replace("x",(str(x)))
msg=msg.replace("y",(str(y)))
msg=msg.replace("z",(str(z)))
msg=msg.replace(" ",(str(space)))
print(msg)
msg=int(msg)
msg=int(msg)*(qw)
print(msg)
fileb=open("key.txt","w")
filec=fileb.write(str(qw))
fileb.close()
file=open("msg decrypt.txt","w")
filea=file.write(str(msg))
file.close()
msg=(str(e)(msg))
print(IP)
print(qw)
if msg != 'q':
if msg != '':
s.sendto(alias.encode() + ": ".encode() + (str(msg).encode)(), server)
tLock.acquire()
msg = input(alias + "-> ")
tLock.release()
shudown = True
rT.join()
s.close()
host code:
import socket
import time
host = '127.0.0.1'
port = 5000
clients = []
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind((host,port))
s.setblocking(0)
quitting = False
print ("Server Started.")
while not quitting:
try:
data, addr = s.recvfrom(1024)
if "Quit" in str(data):
quitting = True
if addr not in clients:
clients.append(addr)
print (time.ctime(time.time()) + str(addr) + ": :" + str(data))
for client in clients:
s.sendto(data, client)
except:
pass
s.close()
Im struggling as my poor excuse of a encryption is mostly numbers so therefore when im sending using the sendto function only uses str`s or so I think?
either way I get the error:
Traceback (most recent call last):
File "H:\client 2.py", line 103, in <module>
msg=(str(e)(msg))
TypeError: 'str' object is not callable
If msg is an index you should write :
msg = str(e)[msg]
Here is the code for server -
import socket, select,re
def getSocket( idd):
return CONNECTION_LIST[idd]
def broadcast_data (sock, message):
for socket in CONNECTION_LIST:
if socket != server_socket and socket != sock :
try :
socket.send(message)
except :
socket.close()
CONNECTION_LIST.remove(socket)
def single_client (sock , message , idd):
socket = getSocket ( idd )
if socket :
socket.send(message)
else:
print "chudap"
if __name__ == "__main__":
CONNECTION_LIST = []
RECV_BUFFER = 4096
PORT = 5000
PORTC = 2225
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(("0.0.0.0", PORT))
server_socket.listen(10)
listen = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
listen.bind(("0.0.0.0" , PORTC))
#listen.listen(10)
CONNECTION_LIST.append(server_socket)
CONNECTION_LIST.append(listen)
print "Chat server started on port " + str(PORT)
idd = 1
while 1:
# Get the list sockets which are ready to be read through select
read_sockets,write_sockets,error_sockets = select.select(CONNECTION_LIST,[],[])
for sock in read_sockets:
if sock == server_socket:
sockfd, addr = server_socket.accept()
CONNECTION_LIST.append(sockfd)
#name = sockfd.recv(RECV_BUFFER)
print "connected from ip %s, id assigned is %d" % (addr[0] , idd)
broadcast_data(sockfd, "client with IP %s has entered with id = %d\n" % (addr[0] , idd))
idd += 1
elif sock == listen:
print "debugging"
data,addr = listen.recvfrom(RECV_BUFFER)
print "Received server probe request from [%s:%s]"%addr
listen.sendto("iam" , addr)#(addr[0] , 2624))
listen.close()
CONNECTION_LIST.remove(listen)
listen = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
listen.bind(("0.0.0.0" , PORTC))
CONNECTION_LIST.append(listen)
else:
try:
data = sock.recv(RECV_BUFFER)
if re.findall(r'.*/msg\d+' , data):
#print "got single client message request" + data
name = "private message from " + re.findall('([^:]+): /msg(\d+)([^"]+)' , data)[0][0] + ": "
#print name
eid = int(re.findall('([^:]+): /msg(\d+)([^"]+)' , data)[0][1])
#print eid
data = re.findall('([^:]+): /msg(\d+)([^"]+)' , data)[0][2]
#print data
data = name + data
#print "single client message sent with id = %d" %eid
single_client( sock , data , int(eid))
elif data:
broadcast_data(sock, data)
except:
broadcast_data(sock, "Client (%s, %s) is offline" % addr)
print "Client (%s, %s) is offline" % addr
sock.close()
CONNECTION_LIST.remove(sock)
continue
server_socket.close()
Here is the code for client -
import socket, select, string, sys
def prompt() :
sys.stdout.write('<You>: ')
sys.stdout.flush()
def exit(sock):
print "\n Thank you for using chat application\nBye"
sock.close()
sys.exit()
def printUsage():
print "1. By default your message will be sent to all clients sitting on the chat server"
print "2. You can send a private message to a person by starting your message as \"/msg{id}{Your message}\" for example /msg2Hi will send \"hi\" to client with id 2"
print "3. For quitting simply type \"/q\" or \"/quit\""
prompt()
PORTS = 2225
PORTC = 2624
if __name__ == "__main__":
broad = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
broad.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
broad.bind(( '0.0.0.0' , 2624) )
broad.sendto(b'whoisserver', 0, ("255.255.255.255", PORTS))
broad.settimeout(10)
print 15*"-" + "WELCOME TO CHATVILLE" + 15*"-" + "\nFinding the server"
try:
data , addr = broad.recvfrom(10)
except:
print "Can't find server ! Please ensure that server is up"
broad.close()
sys.exit()
broad.close()
if data <> "iam":
print "Can't find a valid server !"
sys.exit()
host = addr[0]
port = 5000
print addr
# host = sys.argv[1]
# port = int(sys.argv[2])
# print host,port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
name = raw_input("Please Enter your name: ")
try :
s.connect((host, port))
s.send(name)
except :
print 'Unable to connect'
sys.exit()
print 'Connected to remote host. Enjoy...............................'
name = "<" + name + ">" + ": "
print " - type /h to see usage instructions any time :) - "
prompt()
while 1:
socket_list = [sys.stdin, s]
read_sockets, write_sockets, error_sockets = select.select(socket_list , [], [])
for sock in read_sockets:
if sock == s:
data = sock.recv(4096)
if not data :
print '\nDisconnected from chat server'
sys.exit()
else :
print ""
sys.stdout.write(data)
prompt()
else :
msg = sys.stdin.readline()
if str.startswith(msg, "/h") or str.startswith(msg,"/help"):
printUsage()
elif str.startswith(msg, "/quit") or str.startswith(msg,"/q"):
exit(s)
else:
msg = name + msg
s.send(msg)
prompt()
Main problem is that only one client is able to connect as soon as the first client connects are to server no other client is able to discover the server.
I tried by looking at the client's code by tcpdump and I can see the packet going at port number 2225, but the socket listen is not responding at all after the first connection.
PS - earlier I was not making instance of listen socket again and again but I tried this also and it didn't work out.
In the sever broadcast_data() does not exclude the UDP socket (listen) from the sockets to write to, and so it calls send() on that socket. This fails with exception
socket.error: [Errno 89] Destination address required
because no address is supplied (and can't be with socket.send()). The exception handler then closes the UDP socket and no further messages from new clients can be received. That's why additional clients can not connect to the server.
This is a perfect example of why it is not a good idea to use a bare except, i.e. an except statement that handles all exceptions. In this case the handler closes the UDP socket without even logging the fact. There are other instances of bare except statements in your code. I suggest that you handle specific exceptions to avoid this sort of bug. You can fix it by adding the UDP socket to the list of sockets to ignore:
def broadcast_data(sock, message):
for socket in CONNECTION_LIST:
if socket not in (server_socket, sock, listen):
try :
socket.send(message)
except socket.error as exc:
print '!!! An error occurred while writing to client. !!!'
print exc
socket.close()
CONNECTION_LIST.remove(socket)
Now no attempt will be made to send messages to the UDP listen socket, and the socket won't be closed due to error.
P.S. the code in your main loop that closes and reopens the listen socket is not necessary.
So I found this chat program on http://www.bogotobogo.com/python/python_network_programming_tcp_server_client_chat_server_chat_client_select.php
And I want to use this at my school for me and my friends, put I don't want to chat to be ex: [192.168.1.3] "Message". I want to be able to add another argument so it shows there name they input instead of Ip. It would be a little hard to have everyone using ips. Any suggestions?
Server Code:
# chat_server.py
import sys
import socket
import select
HOST = ''
SOCKET_LIST = []
RECV_BUFFER = 4096
PORT = 9009
def chat_server():
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
# add server socket object to the list of readable connections
SOCKET_LIST.append(server_socket)
print "Chat server started on port " + str(PORT)
while 1:
# get the list sockets which are ready to be read through select
# 4th arg, time_out = 0 : poll and never block
ready_to_read,ready_to_write,in_error = select.select(SOCKET_LIST,[],[],0)
for sock in ready_to_read:
# a new connection request recieved
if sock == server_socket:
sockfd, addr = server_socket.accept()
SOCKET_LIST.append(sockfd)
print "Client (%s, %s) connected" % addr
broadcast(server_socket, sockfd, "[%s:%s] entered our chatting room\n" % addr)
# a message from a client, not a new connection
else:
# process data recieved from client,
try:
# receiving data from the socket.
data = sock.recv(RECV_BUFFER)
if data:
# there is something in the socket
broadcast(server_socket, sock, "\r" + '[' + str(sock.getpeername()) + '] ' + data)
else:
# remove the socket that's broken
if sock in SOCKET_LIST:
SOCKET_LIST.remove(sock)
# at this stage, no data means probably the connection has been broken
broadcast(server_socket, sock, "Client (%s, %s) is offline\n" % addr)
# exception
except:
broadcast(server_socket, sock, "Client (%s, %s) is offline\n" % addr)
continue
server_socket.close()
# broadcast chat messages to all connected clients
def broadcast (server_socket, sock, message):
for socket in SOCKET_LIST:
# send the message only to peer
if socket != server_socket and socket != sock :
try :
socket.send(message)
except :
# broken socket connection
socket.close()
# broken socket, remove it
if socket in SOCKET_LIST:
SOCKET_LIST.remove(socket)
if __name__ == "__main__":
sys.exit(chat_server())
Client Code:
# chat_client.py
import sys
import socket
import select
def chat_client():
if(len(sys.argv) < 3) :
print 'Usage : python chat_client.py hostname port'
sys.exit()
host = sys.argv[1]
port = int(sys.argv[2])
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
# connect to remote host
try :
s.connect((host, port))
except :
print 'Unable to connect'
sys.exit()
print 'Connected to remote host. You can start sending messages'
sys.stdout.write('[Me] '); sys.stdout.flush()
while 1:
socket_list = [sys.stdin, s]
# Get the list sockets which are readable
ready_to_read,ready_to_write,in_error = select.select(socket_list , [], [])
for sock in ready_to_read:
if sock == s:
# incoming message from remote server, s
data = sock.recv(4096)
if not data :
print '\nDisconnected from chat server'
sys.exit()
else :
#print data
sys.stdout.write(data)
sys.stdout.write('[Me] '); sys.stdout.flush()
else :
# user entered a message
msg = sys.stdin.readline()
s.send(msg)
sys.stdout.write('[Me] '); sys.stdout.flush()
if __name__ == "__main__":
sys.exit(chat_client())