I've been trying to make a proxy server that allows UA spoofing but, it doesn't seem to work and I don't know why.
config.py
# if you want to spoof the useragent then,
# replace "$MyAgent" with the desired agent.
# For example,
# agent="Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:44.0) Gecko/20100101 Firefox/44.0"
host="127.0.0.1"
port=8888
byte=4096
listeners=100
delay=00000.1
agent="$MyAgent"
Proxy.py
import socket, select, time, config, sys
def Print(val):
sys.stdin.flush()
sys.stdout.write(val)
class ProxyServer(object):
def __init__(self):
self._host = (config.host, config.port)
self._socket = socket.socket()
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.setblocking(False)
self._agent = config.agent
self._bytes = config.byte
self._delay = config.delay
self._listeners = config.listeners
self._cons = list()
self._log = Print
def _bind(self):
self._socket.bind(self._host)
self._socket.listen(self._listeners)
def main(self):
self._cons.append(self._socket)
while True:
time.sleep(self._delay)
rl, xl, wl = select.select(self._cons, [], [])
for sock in rl:
if sock == self._socket:
sock, ip = self._socket.accept()
self._on_connect()
self._cons.append(sock)
elif sock == None:
self._socket.close()
data = sock.recv(self._bytes)
agent = self._get_agent_header(data)
if not agent == "NO_AGENT":
agent_new = self._agent.replace("$MyAgent", agent)
data = data.replace(agent, agent_new)
sock.send(data)
else:
sock.send(data)
if not data:
self._on_close(sock)
def _on_close(self, sock):
self._cons.remove(sock)
self._log("client dc {0} left".format(self._count()))
sock.close()
def _on_connect(self):
self._log("connection made, {0} clients connected".format(self._count()))
def _count(self):
c = len(self._cons) - 1
return c
def _get_agent_header(self, data):
pat = "User-Agent: (.+)"
m = re.search(pat, data)
if m:
return m.group(0)
else:
return "NO_AGENT"
if __name__ == '__main__':
server = ProxyServer()
server._bind()
server.main()
If you run this the request will not complete. But, some of the headers will show in plain text. Is there something I'm not doing right?
Related
i have an university project with python which i have to write a proxy server that waits for a request from a client and then connects the client to the server. i searched the net and found an already-written code from this site:
https://www.geeksforgeeks.org/creating-a-proxy-webserver-in-python-set-1/
so i used it and made some changes in the code and add the public server and port to it
but when i run it i get this error:
line 33, in main
request = conn.recv(4096)
NameError: name 'conn' is not defined
so i'm not very familiar with sockets and python so if there are obvious mistakes in the code i would be happy if u guys could explain them in a very basic way so my amateur butt would understand it lol
this is the code:
import signal
import socket
import threading
class Proxy:
def __init__(self):
# creating a tcp socket
self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# reuse the socket
self.serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.ip = 'localhost'
self.port = 8080
self.serverSocket.bind((self.ip, self.port))
self.serverSocket.listen(10)
self.__clients = {}
def shutdown(self):
# shutdown on cntrl c
signal.signal(signal.SIGINT, self.shutdown)
def multirequest (self):
while True:
# establish the connection
(clientSocket, client_address) = self.serverSocket.accept()
d = threading.Thread(name=self._getclientname(client_address),
target=self.proxy_thread,
args=(clientSocket, client_address))
d.setDaemon(True)
d.start()
def main(self, conn):
# get the request from browser
request = conn.recv(4096)
# parse the first line
first_line = request.split('\n')[0]
# get url
url = first_line.split(' ')[1]
http_pos = url.find("://")
if http_pos == -1:
temp = url
else:
temp = url[(http_pos + 3):]
webserver = ""
port = -1
port_pos = temp.find(":")
# find end of web server
webserver_pos = temp.find("/")
if webserver_pos == -1:
webserver_pos = len(temp)
if port_pos == -1 or webserver_pos < port_pos:
# default port
port = 80
webserver = temp[:webserver_pos]
else: # specific port
port = int((temp[(port_pos + 1):])[:webserver_pos - port_pos - 1])
webserver = temp[:port_pos]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1)
s.connect((webserver, port))
s.sendall(request)
while 1:
# receive data from web server
data = s.recv(4096)
if len(data) > 0:
conn.send(data) # send to browser/client
else:
break
p = Proxy()
p.main()
In python, the common pattern for executables is like this:
def main(): # Or whatever name you want to use
"""Your code here"""
# If you are importing the code, the condition will evaluate to false.
if __name__ == "__main__":
main()
Have in mind that you can use whatever function name you want.
You need to put your code inside of a class like this. Any variables you plan to use through your code, define inside of the __init__ method, this could be for example your self.serverSocket, ip, port number, etc.
The init method is deigned to run once, once you create an instance of the class, it usually stores variables. The main method or any other methods you define, would be where you put the rest of your code.
import signal
import socket
import threading
class proxy():
def __init__(self):
# creating a tcp socket
self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# reuse the socket
self.serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.your_ip = "127.0.0.1" # loop back address
self.your_port = 80 # use a port thats open like port 80
def shutdown(self):
# add your signal shutdown code here
pass
def main(self):
# shutdown on cntrl c
signal.signal(signal.SIGINT, self.shutdown)
self.serverSocket.bind((self.your_ip, self.your_port))
self.serverSocket.listen(10)
self.__clients = {}
while True:
# establish the connection
(clientSocket, client_address) = self.serverSocket.accept()
d = threading.Thread(name=self._getClientName(client_address),
target=self.proxy_thread,
args=(clientSocket, client_address))
d.setDaemon(True)
d.start()
# get the request from browser
request = conn.recv(4096)
# parse the first line
first_line = request.split('\n')[0]
# get url
url = first_line.split(' ')[1]
http_pos = url.find("://")
if http_pos == -1:
temp = url
else:
temp = url[(http_pos + 3):]
port_pos = temp.find(":")
# find end of web server
webserver_pos = temp.find("/")
if webserver_pos == -1:
webserver_pos = len(temp)
webserver = ""
port = -1
if port_pos == -1 or webserver_pos < port_pos:
# default port
port = 80
webserver = temp[:webserver_pos]
else: # specific port
port = int((temp[(port_pos + 1):])[:webserver_pos - port_pos - 1])
webserver = temp[:port_pos]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1)
s.connect((webserver, port))
s.sendall(request)
while 1:
# receive data from web server
data = s.recv(4096)
if len(data) > 0:
conn.send(data) # send to browser/client
else:
break
p = proxy()
p.main()
I successful make a local python proxy via socket module: https://github.com/wayne931121/Python_Proxy_Server/blob/main/Proxy.py
It can run http and https request.
Code:
#!/usr/bin/env python
# coding: utf-8
# In[ ]:
# 參考資料
# https://docs.python.org/3/library/socket.html
# https://stackoverflow.com/questions/24218058/python-https-proxy-tunnelling
# https://stackoverflow.com/questions/68008233/proxy-server-with-python/73851150#73851150
import sys
#import ssl
import time
import signal
import socket
#import certifi
import threading
with open("log.txt", "w") as f:
f.write("")
def signal_handler(sig, frame):
print('Proxy is Stopped.')
sys.exit(0)
def write(*content, prt=False):
if prt :
if len(content[0])<100:
print(*content)
else:
print("This message is too long not print in cmd but will store at log.txt.")
if type(content[0])==bytes:
content = b" ".join(content)
else:
content = bytes(" ".join(content), encoding="utf-8")
with open("log.txt", "ab") as f:
f.write(content+b"\n")
class Proxy:
def __init__(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # creating a tcp socket
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # reuse the socket
self.ip = "127.0.0.1"
self.port = 8080
# self.host = socket.gethostbyname(socket.gethostname())+":%s"%self.port
self.sock.bind((self.ip, self.port))
self.sock.listen(10)
print("Proxy Server Is Start, See log.txt get log.")
print("Press Ctrl+C to Stop.")
start_multirequest = threading.Thread(target=self.multirequest)
start_multirequest.setDaemon(True)
start_multirequest.start()
while 1:
time.sleep(0.01)
signal.signal(signal.SIGINT, signal_handler)
def multirequest(self):
while True:
(clientSocket, client_address) = self.sock.accept() # establish the connection
client_process = threading.Thread(target=self.main, args=(clientSocket, client_address))
client_process.setDaemon(True)
client_process.start()
def main(self, client_conn, client_addr): # client_conn is the connection by proxy client like browser.
origin_request = client_conn.recv(4096)
request = origin_request.decode(encoding="utf-8") # get the request from browser
first_line = request.split("\r\n")[0] # parse the first line
url = first_line.split(" ")[1] # get url
http_pos = url.find("://")
if http_pos == -1:
temp = url
else:
temp = url[(http_pos + 3):]
webserver = ""
port = -1
port_pos = temp.find(":")
webserver_pos = temp.find("/") # find end of web server
if webserver_pos == -1:
webserver_pos = len(temp)
if port_pos == -1 or webserver_pos < port_pos: # default port
port = 80
webserver = temp[:webserver_pos]
else: # specific port
port = int(temp[(port_pos + 1):])
webserver = temp[:port_pos]
write("Connected by", str(client_addr))
write("ClientSocket", str(client_conn))
write("Browser Request:")
write(request)
server_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_conn.settimeout(1000)
try:
server_conn.connect((webserver, port)) # "server_conn" connect to public web server, like www.google.com:443.
except: # socket.gaierror: [Errno 11001] getaddrinfo failed
client_conn.close()
server_conn.close()
return
if port==443:
client_conn.send(b"HTTP/1.1 200 Connection established\r\n\r\n")
client_conn.setblocking(0)
server_conn.setblocking(0)
write("Connection established")
# now = time.time()
client_browser_message = b""
website_server_message = b""
error = ""
while 1:
# if time.time()-now>1: # SET TIMEOUT
# server_conn.close()
# client_conn.close()
# break
try:
reply = client_conn.recv(1024)
if not reply: break
server_conn.send(reply)
client_browser_message += reply
except Exception as e:
pass
# error += str(e)
try:
reply = server_conn.recv(1024)
if not reply: break
client_conn.send(reply)
website_server_message += reply
except Exception as e:
pass
# error += str(e)
write("Client Browser Message:")
write(client_browser_message+b"\n")
write("Website Server Message:")
write(website_server_message+b"\n")
# write("Error:")
# write(error+"\n")
server_conn.shutdown(socket.SHUT_RDWR)
server_conn.close()
client_conn.close()
return
server_conn.sendall(origin_request)
write("Website Host Result:")
while 1:
# receive data from web server
data = server_conn.recv(4096)
try:
write(data.decode(encoding="utf-8"))
except:
write(data)
if len(data) > 0:
client_conn.send(data) # send to browser/client
else:
break
server_conn.shutdown(socket.SHUT_RDWR)
server_conn.close()
client_conn.close()
Proxy()
I made an app to send to python socket the message "Oleft" when i tilt the phone but the result on console is like:
connection from('192.168.0.101', 33313)
b'Oleft'
b'OleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'Oleft'
b'OrightOrightOrightOright'
b'OleftOleftOleftOleftOleftOrightOrightOrightOrightOrightOrightOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'OleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'OleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'OleftOleftOleftOleftOleftOleftOleftOleftOleftOleftOleft'
b'OleftOleft'
It has no way to receive only an b'Oleft' ?
wgremote.py
import socket
class WGRemote:
def __init__(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.host = socket.gethostname()
self.port = 10000
self.received = None
def connect(self):
global c, addr
self.sock.bind((self.host, self.port))
self.sock.listen(5)
c, addr = self.sock.accept()
print('connection from' + str(addr))
def setMode(self,mode):
sent = c.send(mode.encode("utf-8"))
def receive(self):
self.received = c.recv(1024)
return self.received
def close(self):
c.close
testApp.py
import socket
import sys
from wgremote import WGRemote
remote = WGRemote()
remote.connect()
remote.setMode('orient')
while True:
data = remote.receive()
print(data)
Because, You wrote print(data) in the while loop :)
while True:
data = remote.receive()
print(data)
If you want to receive only an b'Oleft you can make a filter :
count = 0
while True:
data = remote.receive()
if data == "b'Oleft":
if count < 1:
count = count + 1
print(data)
Or you must send the only an b'Oleft :)
I'd like to know how to write TCP tunnel/relay/bridge/proxy (you name it) using twisted.
I did some research in google, twisted doc/forum etc. etc but couldn't find anwser.
I already done it in pure python using socket, threading and select.
Here is code:
#!/usr/bin/env python
import socket
import sys
import select
import threading
import logging
import time
class Client(threading.Thread):
def __init__(self, client, address, id_number, dst_ip, dst_port):
self.log = logging.getLogger(__name__+'.client-%s' % id_number)
self.running = False
self.cl_soc = client
self.cl_adr = address
self.my_id = id_number
self.dst_ip = dst_ip
self.dst_port = dst_port
threading.Thread.__init__(self)
def stop(self):
self.running = 0
def run(self):
try:
self.dst_soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.dst_soc.connect((self.dst_ip,self.dst_port))
except:
self.log.error('Can\'t connect to %s:%s' % (self.dst_ip, self.dst_port))
else:
self.running = True
self.log.info('Bridge %s <-> %s created' % (
'%s:%s' % self.dst_soc.getpeername(), '%s:%s' % self.cl_adr))
try:
while self.running:
iRdy = select.select([self.cl_soc, self.dst_soc],[],[], 1)[0]
if self.cl_soc in iRdy:
buf = self.cl_soc.recv(4096)
if not buf:
info = 'Ended connection: client'
self.running = False
else:
self.dst_soc.send(buf)
if self.dst_soc in iRdy:
buf = self.dst_soc.recv(4096)
if not buf:
info = 'Ended connection: destination'
self.running = False
else:
self.cl_soc.send(buf)
except:
self.log.error('Sth bad happend', exc_info=True)
self.running = False
self.log.debug('Closing sockets')
try:
self.dst_soc.close()
except:
pass
try:
self.cl_soc.close()
except:
pass
class Server(threading.Thread):
def __init__(self, l_port=25565, d_ip='127.0.0.1', d_port=None):
self.log = logging.getLogger(__name__+'.server-%s:%s' % (d_ip,d_port))
threading.Thread.__init__(self)
self.d_ip = d_ip
if d_port == None:
self.d_port = l_port
else:
self.d_port = d_port
self.port = l_port
binding = 1
wait = 30
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
while binding:
try:
self.s.bind(('',self.port))
except:
self.log.warning('Cant bind. Will wait %s sec' % wait)
time.sleep(wait)
else:
binding = 0
self.log.info('Server ready for connections - port %s' % d_port)
def run(self):
self.s.listen(5)
input = [self.s, sys.stdin]
running = 1
self.cl_threads = []
id_nr = 0
while running:
iRdy = select.select(input, [], [],1)[0]
if self.s in iRdy:
c_soc, c_adr = self.s.accept()
c = Client(c_soc, c_adr, id_nr, self.d_ip, self.d_port)
c.start()
self.cl_threads.append(c)
id_nr += 1
if sys.stdin in iRdy:
junk = sys.stdin.readline()
print junk
running = 0
try:
self.s.close()
except:
pass
for c in self.cl_threads:
c.stop()
c.join(5)
del c
self.cl_threads = None
self.log.info('Closing server')
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
s = Server(1424, '192.168.1.6', 1424)
s.run()
this is actually, already built into twisted, from the command line you can type:
$ twistd --nodaemon portforward --port 1424 --host 192.168.1.6
to get the exact behavior you seem to be looking for.
If you'd like to roll your own, you can still use all of the bits, in twisted.protocols.portforward
I'm trying to use a variable defined in one function in a different function. I know I have to use Global var but I'm still not getting the output I was expecting.
def proxycall():
global ip
global port
ip = "192.168.0.1"
port = "8080"
def web1():
class YWebPage():
def something():
var = 1
def somethingelse():
varr = 2
class Browser():
def someting():
varrr = 3
def sometingelse():
varrrr = 4
print (ip)
print (port)
web1()
This minimal version of my program is giving the traceback - NameError: global name 'ip' is not defined.
I see the problem with the minimal version now, thanks for pointing it out. However I don't think thats my issue now because I call proxycall() in the full version below, any ideas?
import sys
from PyQt4 import QtCore, QtGui, QtWebKit
from PyQt4.QtWebKit import QWebSettings
from PyQt4.QtNetwork import QNetworkAccessManager
from PyQt4.QtNetwork import *
import re
import requests
UA_STRING = """Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36""" # String for User-Agent
#vidurl = "http://www.youtube.com/watch?v=hqDT6G5_1tQ" # 94 views
vidurl = "http://httpbin.org"
def proxycall():
global ip
global port
# Proxy Stuff
count = 0
while (count < 5):
#Call mashape proxy API and setup
try:
headers = {'X-Mashape-Authorization':'xxxxxxxxxxxxx'}
p = requests.get('https://webknox-proxies.p.mashape.com/proxies/new?maxResponseTime=5', headers=headers, timeout=5.000)
prox = p.text
ip = prox.split(":")[1] #split response string
port = prox.split(":")[2]
ip = re.sub('["}]', '', ip) #sanitize output
port = re.sub('["}]', '', port)
proxy = str(ip) + ':' + str(port) #reformat proxy for Requests
print ('Retrieved proxy ' + str(proxy))
proxies = {"http": proxy}
print ('Configured proxy')
except:
print('Mashape API error, FAIL')
#Get real IP address
try:
r = requests.get('http://httpbin.org/ip', timeout=5.000)
s = r.text
realip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', s )
print ('Real IP address is ' + str(realip))
except:
print ("Connection Error (IP), FAIL")
#Test proxy
try:
x = requests.get('http://bot.whatismyipaddress.com/', proxies=proxies, timeout=5.000)
y = x.text
proxyip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', y )
print ('Masked IP address is ' + str(proxyip))
if proxyip != realip:
print ('Proxy test OK')
count = 100
else:
print ('Proxy test FAIL, GET new proxy...')
except:
print ("Connection Error (Proxy), FAIL")
def web1(parent):
class YWebPage(QtWebKit.QWebPage):
def __init__(self):
super(QtWebKit.QWebPage, self).__init__()
def userAgentForUrl(self, url):
return UA_STRING
class Browser(QtGui.QMainWindow): # "Browser" window
def __init__(self, parent):
QtGui.QMainWindow.__init__(self, parent)
self.resize(800,600) # Viewport size
self.centralwidget = QtGui.QWidget(self)
self.html = QtWebKit.QWebView()
def browse(self):
self.webView = QtWebKit.QWebView()
self.yPage = YWebPage()
self.webView.setPage(self.yPage)
self.webView.load(QtCore.QUrl(vidurl)) # Video URL
self.webView.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled,True) # Enables flash player
self.webView.settings().setAttribute(QtWebKit.QWebSettings.AutoLoadImages, False) # No images for speed
QWebSettings.clearMemoryCaches ()
self.webView.show()
proxycall()
x = Browser(parent)
QNetworkProxy.setApplicationProxy(QNetworkProxy(QNetworkProxy.HttpProxy, ip, port))
x.browse()
You didn't call proxycall() at all, so the variables were never set.
Call it first, then call web1():
proxycall()
web1()
Names don't just pop into existence just because you marked them as global in a function; Python names require assignment (binding) to materialize.
You need to call proxycall() before you can access ip globally:
proxycall()
web1()
Your updated code has the same problem as the original code. You are using ip in this statement:
QNetworkProxy.setApplicationProxy(QNetworkProxy(QNetworkProxy.HttpProxy, ip, port))
...but you don't call proxycall until you call x.browse() in the statement after that.
You need to call proxycall() to initialize ip
proxycall()
web1()
You need to initialize/define ip and port before using them. Call proxycall() and then web1()
def proxycall():
global ip
global port
ip = "192.168.0.1"
port = "8080"
def web1():
class YWebPage():
def something():
var = 1
def somethingelse():
varr = 2
class Browser():
def someting():
varrr = 3
def sometingelse():
varrrr = 4
print (ip)
print (port)
proxycall()
web1()
ans:
192.168.0.1
8080
I wrote a simple Python script for a proxy functionality. It works fine, however, if the requested webpage has many other HTTP requests, e.g. Google maps, the page is rendered quite slow.
Any hints as to what might be the bottleneck in my code, and how I can improve?
#!/usr/bin/python
import socket,select,re
from threading import Thread
class ProxyServer():
def __init__(self, host, port):
self.host=host
self.port=port
self.sk1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def startServer(self):
self.sk1.bind((self.host,self.port))
self.sk1.listen(256)
print "proxy is ready for connections..."
while(1):
conn,clientAddr = self.sk1.accept()
# print "new request coming in from " + str(clientAddr)
handler = RequestHandler(conn)
handler.start()
class RequestHandler(Thread):
def __init__(self, sk1):
Thread.__init__(self)
self.clientSK = sk1
self.buffer = ''
self.header = {}
def run(self):
sk1 = self.clientSK
sk2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
while 1:
self.buffer += sk1.recv(8192)
if self.buffer.find('\n') != -1:
break;
self.header = self.processHeader(self.buffer)
if len(self.header)>0: #header got processed
hostString = self.header['Host']
host=port=''
if hostString.__contains__(':'): # with port number
host,port = hostString.split(':')
else:
host,port = hostString,"80"
sk2.connect((host,int(port)))
else:
sk1.send('bad request')
sk1.close();
return
inputs=[sk1,sk2]
sk2.send(self.buffer)
#counter
count = 0
while 1:
count+=1
rl, wl, xl = select.select(inputs, [], [], 3)
if xl:
break
if rl:
for x in rl:
data = x.recv(8192)
if x is sk1:
output = sk2
else:
output = sk1
if data:
output.send(data)
count = 0
if count == 20:
break
sk1.close()
sk2.close()
def processHeader(self,header):
header = header.replace("\r\n","\n")
lines = header.split('\n')
result = {}
uLine = lines[0] # url line
if len(uLine) == 0: return result # if url line empty return empty dict
vl = uLine.split(' ')
result['method'] = vl[0]
result['url'] = vl[1]
result['protocol'] = vl[2]
for line in lines[1: - 1]:
if len(line)>3: # if line is not empty
exp = re.compile(': ')
nvp = exp.split(line, 1)
if(len(nvp)>1):
result[nvp[0]] = nvp[1]
return result
if __name__ == "__main__":
HOST, PORT = "0.0.0.0", 8088
proxy = ProxyServer(HOST,PORT)
proxy.startServer()
I'm not sure what your speed problems are, but here are some other nits I found to pick:
result['protocal'] = vl[2]
should be
result['protocol'] = vl[2]
This code is indented one level too deep:
sk2.connect((host,int(port)))
You can use this decorator to profile your individual methods by line.