I tried to make a web server on port 4460 but when i type "http://127.0.0.1:4460/" in the browser
address bar the browser says ERR_INVALID_HTTP_RESONSE.(Google Chrome).Browser is latest version.
The code did not raise any errors and did not send any bad_gateway requests.it did not access the .ico file.
Python ver:3.8.10
my code:
import socket
from socket import AF_INET,SOCK_STREAM
from threading import Lock
from pprint import pprint
from threadtools import threaded
from email.utils import format_datetime as fmd
import datetime
from deflate import gzip_compress
ms = (lambda x:x/1000)
socket.setdefaulttimeout(ms(700))
ol = Lock()
plok = Lock()
ENCODINGS = "utf-8 utf-16 cp936 latin-1".split()
response_header = b"""\
200 OK
Content-Type: text/html
Content-Length: $$CTXLN$$
Content-Encoding: gzip
Connection: close
Date: $$DATE$$
Keep-Alive: timeout=2, max=2
$$CTX$$"""
bad_gateway = b"""\
502 Bad Gateway
Content-type:text/html
Content-legth:0"""
def decode(x,verbose=False):
for enc in ENCODINGS:
flag = False
try:
return x.decode(enc)
except:
flag = True
finally:
print("Decoded in:"+enc) if(not flag)and verbose else None
return ""
def startswithany(a,lis):
for x in lis:
if a.startswith(x):
return True
return False
def is_newline(x):
return x in ("\r\n","\n")
def load_rsrc(acpt):
if "text/html" in acpt or "text/*" in acpt or "*/*" in acpt:
return open("response.html","rb").read()
elif "image/ico" in acpt or "image/*" in acpt:
return open("response.ico","rb").read()
else:
return b""
def handle_connection(cnct,addr):
global pending
with plok:
pending += 1
try:
if pending > 20:#Too many connections!!!
cnct.send(bad_gateway)
with ol:
print(f"----------------\nConnection from:{addr}\n")
has_ln = True
data = b""
ctx = ""
headers = {"Unknown-Lines":[]}
while True:
data = b""
while not decode(data).endswith("\n"):#\r\n ends with \n
try:
data += cnct.recv(1)
except:#timeout
has_ln = False
break
if not has_ln:
break
assert len(data)
data = decode(data).strip(" ")
assert not data.startswith(" ")
if is_newline(data):#empty line
continue
if startswithany(data,("GET","POST","PUT")):
headers["Request-Type"] = data.strip()
else:
dsp = data.split(":",1)
if len(dsp)!=2:
print(f"Unknown header:{data}")
headers["Unknown-Lines"].append(data)
else:
a,b = data.split(":",1)
b = b.strip()
headers[a] = b
with ol:
print(f"Headers:")
for k,v in headers.items():
print(f"{k}:{v}")
accept = headers.get("Accept","text/html")
accept = accept.split(",")
q = []
for i,x in enumerate(accept):
if ";q=" in x:
a,b = x.split(";q=")
b = float(b)
accept[i] = a
q.append(b)
rt = tuple(map(str.strip,headers.get("Request-Type","GET/HTTP/1.0").split("/")))
req = rt[0]#GET/POST/PUT
protocol = rt[1]#HTTP;NO SECURE SERVER FOR NOW
ver = rt[2]#version
assert ver in ("1.0","1.1")
now = datetime.datetime.now(datetime.timezone.utc)
datestr = fmd(now,True).encode()
ctx = load_rsrc(accept)
ln = str(len(ctx)+1).encode()
response = response_header.replace(b"$$CTXLN$$",ln)\
.replace(b"$$CTX$$",ctx)\
.replace(b"$$DATE$$",datestr)
response_cmpr = gzip_compress(response)
cnct.send(response_cmpr)
print("Sent:")
print(response.decode())
if headers.get("Connection","Keep-alive") == "Keep-alive":
import time
time.sleep(2)
finally:
cnct.close()
with plok:
pending -= 1
skt = socket.socket(AF_INET,SOCK_STREAM)
skt.bind(("",4460))
skt.listen(3)
skt.settimeout(None)
pending = 0
while True:
cn,ad = skt.accept()
handle_connection(cn,ad)
You are close to your goal. Making slight adjustments I got your snippet working. The main issue is in the HTTP response formatting, it should be defined as follow:
HTTP/1.1 200 OK <--- Missing HTTP/1.1 prefix
Content-Type: text/html
...
Keep-Alive: timeout=2, max=2
<--- Mind the extra newline here which is mandatory
$$CTX$$ <--- Browser will expect HTML here
I have adapted the MCVE your provided, please find below a working version for latest Edge and Firefox browsers.
import socket
from socket import AF_INET,SOCK_STREAM
from threading import Lock
from pprint import pprint
#from threadtools import threaded
from email.utils import format_datetime as fmd
import datetime
#from deflate import gzip_compress
ms = (lambda x:x/1000)
socket.setdefaulttimeout(ms(700))
ol = Lock()
plok = Lock()
ENCODINGS = "utf-8 utf-16 cp936 latin-1".split()
response_header = b"""\
HTTP/1.1 200 OK
Content-Type: text/html
Content-Length: $$CTXLN$$
Connection: close
Date: $$DATE$$
Keep-Alive: timeout=2, max=2
$$CTX$$"""
# Missing HTTP/1.1 Prefix
# The extra new line is required
bad_gateway = b"""\
HTTP/1.1 502 Bad Gateway
Content-type:text/html
Content-legth:0
"""
def decode(x,verbose=False):
for enc in ENCODINGS:
flag = False
try:
return x.decode(enc)
except:
flag = True
finally:
print("Decoded in:"+enc) if(not flag)and verbose else None
return ""
def startswithany(a,lis):
for x in lis:
if a.startswith(x):
return True
return False
def is_newline(x):
return x in ("\r\n","\n")
def load_rsrc(acpt):
if "text/html" in acpt or "text/*" in acpt or "*/*" in acpt:
#return open("response.html","rb").read()
return b"hello"
elif "image/ico" in acpt or "image/*" in acpt:
return b"icon"
else:
return b""
def handle_connection(cnct,addr):
global pending
with plok:
pending += 1
try:
if pending > 20:#Too many connections!!!
cnct.send(bad_gateway)
with ol:
print(f"----------------\nConnection from:{addr}\n")
has_ln = True
data = b""
ctx = ""
headers = {"Unknown-Lines":[]}
while True:
data = b""
while not decode(data).endswith("\n"):#\r\n ends with \n
try:
data += cnct.recv(1)
except:#timeout
has_ln = False
break
if not has_ln:
break
assert len(data)
data = decode(data).strip(" ")
assert not data.startswith(" ")
if is_newline(data):#empty line
continue
if startswithany(data,("GET","POST","PUT")):
headers["Request-Type"] = data.strip()
else:
dsp = data.split(":",1)
if len(dsp)!=2:
print(f"Unknown header:{data}")
headers["Unknown-Lines"].append(data)
else:
a,b = data.split(":",1)
b = b.strip()
headers[a] = b
with ol:
print(f"Headers:")
for k,v in headers.items():
print(f"{k}:{v}")
accept = headers.get("Accept","text/html")
accept = accept.split(",")
q = []
for i,x in enumerate(accept):
if ";q=" in x:
a,b = x.split(";q=")
b = float(b)
accept[i] = a
q.append(b)
rt = tuple(map(str.strip,headers.get("Request-Type","GET/HTTP/1.0").split("/")))
req = rt[0]#GET/POST/PUT
protocol = rt[1]#HTTP;NO SECURE SERVER FOR NOW
ver = rt[2]#version
assert ver in ("1.0","1.1")
now = datetime.datetime.now(datetime.timezone.utc)
datestr = fmd(now,True).encode()
ctx = load_rsrc(accept)
ln = str(len(ctx)+1).encode()
response = response_header.replace(b"$$CTXLN$$",ln)\
.replace(b"$$CTX$$",ctx)\
.replace(b"$$DATE$$",datestr)
#response_cmpr = gzip_compress(response)
cnct.send(response)
print("Sent:")
print(response.decode())
if headers.get("Connection","Keep-alive") == "Keep-alive":
import time
time.sleep(2)
finally:
cnct.close()
with plok:
pending -= 1
skt = socket.socket(AF_INET,SOCK_STREAM)
skt.bind(("",8080))
skt.listen(3)
skt.settimeout(None)
pending = 0
while True:
cn,ad = skt.accept()
handle_connection(cn,ad)
Related
I am having some troubles with my program as when it reaches the end of the third() function, it continues to try to execute transactions. I tried having it return None to break out of the seemly infinite loop that it is in with no success. I am sure that I am missing something very simple here and am guessing it has something to do with the recursion that I used. Thanks for any help that you can provide.
import asyncio
import base64
import json
import os
import os.path
import time
import httpcore
import requests
from typing import Awaitable
import solana
import httpx
from rich import print
from solana.keypair import Keypair
from solana.publickey import PublicKey
from solana.rpc.api import Client
from solana.rpc.async_api import AsyncClient
from solana.rpc.commitment import Confirmed
from solana.rpc.types import TxOpts
from solana.transaction import Transaction
# Notes
# This is meant as a bare bones hello world and as such does not have :
#
# - error handling on http calls
# - checks / retries to ensure solana transactions go through
# - logging - just your basic print statement here. But at least you get the Rich pretty printing variant :)
#
# Libraries used
# - https://www.python-httpx.org/ - cause it's shinier and better than requests
# - https://michaelhly.github.io/solana-py/
# - https://github.com/Textualize/rich for pretty printing - because it rocks.
# I use poetry to manage dependencies but am not including the project file here for brevity.
# Mint constants
USDC_MINT = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v", 6
SOL_MINT = "So11111111111111111111111111111111111111112", 9
FAB_MINT = "EdAhkbj5nF9sRM7XN7ewuW8C9XEUMs8P7cnoQ57SYE96", 9
FUSD_MINT = "B7mXkkZgn7abwz1A3HnKkb18Y6y18WcbeSkh1DuLMkee", 8
# This works ok - most of the time
rpc_host = "https://api.mainnet-beta.solana.com"
filename = r"C:\Users\myname\.config\solana\burner.json"
def get_wallet_keypair(filename: str) -> Keypair:
"""Load a keypair from a filesystem wallet."""
if not os.path.isfile(filename):
raise Exception(f"Wallet file '{filename}' is not present.")
with open(filename) as json_file:
data = json.load(json_file)
mid = len(data) // 2
secret_key = data[:mid]
secret_bytes = bytes(secret_key)
keypair = Keypair.from_secret_key(secret_bytes)
print(f"Public Key is: {keypair.public_key}")
return keypair
async def get_quote(
input_mint: str, output_mint: str, amount: int, slippage: int = 0.2):
url_query = f"https://quote-api.jup.ag/v1/quote?outputMint={output_mint}&inputMint={input_mint}&amount={amount}&slippage={slippage}"
print(url_query)
async with httpx.AsyncClient() as client:
r = await client.get(url_query)
return r.json()
async def get_transaction(route: dict, user_key: str) -> dict:
swap_url = "https://quote-api.jup.ag/v1/swap"
input = {"route": route, "userPublicKey": user_key, "wrapUnwrapSOL": True}
print(json.dumps(input, indent=2))
async with httpx.AsyncClient() as client:
r = await client.post(swap_url, json=input,
timeout=6.0) # slightly longer timout as the free rpc server can be a bit laggy
return r.json()
def send_transaction(payer: Keypair, cc: Client, swap_transaction: str, opts: TxOpts) -> str:
""" Send a serialized transaction to the RPC node """
trans = Transaction.deserialize(base64.b64decode(swap_transaction))
result = cc.send_transaction(trans, payer, opts=opts)
txid = result["result"]
print(f"transaction details :https://solscan.io/tx/{txid}")
return txid
async def async_main(from_mint, from_decimals, to_mint, quantity):
cc = Client(rpc_host)
print(f" Converting {quantity} {from_mint} to {to_mint} with {from_decimals} Decimals")
quote_quantity = quantity * (10 ** from_decimals)
r = await get_quote(str(from_mint), str(to_mint), quote_quantity, slippage=2)
quote, outAmount = r["data"][0], int(r['data'][0]['outAmountWithSlippage']) / (10 ** from_decimals)
print("Out Amount =", outAmount)
if quote := r["data"][0]:
print(quote)
# get the relevant transaction details
trans = await get_transaction(quote, str(pubkey))
setup_transaction = trans["setupTransaction"] if "setupTransaction" in trans else None
swap_transaction = trans["swapTransaction"] if "swapTransaction" in trans else None
cleanup_transaction = trans["cleanupTransaction"] if "cleanupTransaction" in trans else None
opts = TxOpts(skip_preflight=True)
# Setup transaction. Will create any missing accounts if required.
if setup_transaction:
print("Sending setup transaction")
#print(setup_transaction)
send_transaction(payer, cc, setup_transaction, opts)
# This one actually does the business
if swap_transaction:
print("Sending swap transaction")
txid = send_transaction(payer, cc, swap_transaction, opts)
# Wait for the transaction to complete before looking it up on chain.
# Clearly this is *not* the right way to do this. Retry in a loop or something fancy.
await asyncio.sleep(20)
result = cc.get_transaction(txid, commitment=Confirmed)
print(result)
# Haven't seen one of these needed yet. Hopefully the jup.ag devs can explain when it's required.
if cleanup_transaction:
print("Sending send transaction")
send_transaction(payer, cc, cleanup_transaction, opts)
print("Swap Complete !")
return outAmount
def get_balance(input_mint):
url = "https://api.mainnet-beta.solana.com"
headers = {'Content-type': 'application/json'}
if input_mint == "So11111111111111111111111111111111111111112":
data = {"jsonrpc": "2.0", "id": 1, "method": "getBalance", "params": [f"{pubkey}"]}
response = requests.post(url, data=json.dumps(data), headers=headers)
response = response.text
parsed = json.loads(response)
# print(json.dumps(parsed, indent=4, sort_keys=True))
accountBal = (parsed['result']['value']) / 10 ** SOL_MINT[1]
print(accountBal)
else:
data = {"jsonrpc": "2.0", "id": 1, "method": "getTokenAccountsByOwner",
"params": [f"{pubkey}",
{"mint": f"{input_mint}"}, {"encoding": "jsonParsed"}]}
response = requests.post(url, data=json.dumps(data), headers=headers)
response = response.text
parsed = json.loads(response)
# print(json.dumps(parsed, indent=4, sort_keys=True))
accountBal = parsed['result']['value'][0]['account']['data']['parsed']['info']['tokenAmount']['uiAmount']
print(accountBal)
return accountBal
# usdc buys fusd fusd is sold for sol sol is sold for usdc
# (from_mint, from_decimals, to_mint, quantity):
class swaps:
def __init__(self, input_mint, decimals, output_mint, amount):
self.input_mint = input_mint
self.decimals = decimals
self.output_mint = output_mint
self.amount = amount
def swap(self):
asyncio.run(async_main(self.input_mint, self.decimals, self.output_mint, self.amount))
def first(count, previous = 0):
try:
if get_balance(USDC_MINT[0]) <= 1:
time.sleep(1)
count += 1
if count >= 60:
third(0)
first(count)
except TypeError:
first(0)
step1 = swaps(USDC_MINT[0], USDC_MINT[1], FUSD_MINT[0], get_balance(USDC_MINT[0]) if previous == 0 else previous)
try:
step1.swap()
except httpx.ReadTimeout:
print("Retrying")
time.sleep(10)
first(0)
second(0)
def second(count, previous = 0):
try:
if get_balance(FUSD_MINT[0]) <= 1:
time.sleep(1)
count += 1
if count >= 60:
first(0)
second(count)
except TypeError:
second(0)
step2 = swaps(FUSD_MINT[0], FUSD_MINT[1], SOL_MINT[0], get_balance(FUSD_MINT[0]) if previous == 0 else previous)
try:
step2.swap()
except:
print("Retrying")
time.sleep(10)
second(0)
count = 0
third(0)
def third(count, previous = 0):
if get_balance(SOL_MINT[0]) < .6:
time.sleep(1)
count += 1
if count >= 60:
second(0)
third(count)
step3 = swaps(SOL_MINT[0], SOL_MINT[1], USDC_MINT[0], get_balance(SOL_MINT[0]) - 0.5 if previous == 0 else previous)
try:
step3.swap()
except:
print("Retrying")
time.sleep(10)
third(previous)
print("All Swaps Completed")
return None
payer = get_wallet_keypair(filename)
pubkey = payer.public_key
loops = 0
if __name__ == "__main__":
previousBalence = get_balance(USDC_MINT[0])
print(f"Starting Balence: {previousBalence}")
#for loops in range(5):
first(0)
loops += 1
endBalance = get_balance((USDC_MINT[0]))
print(f"End balence is {endBalance}")
totalProfit = endBalance-previousBalence
print(f"Total Profit is: {totalProfit}")
Edit: The output when the code continues is it keeps trying to swap fUSD for SOL and SOL for USDC over and over again.
Solution: https://pastebin.com/8id7gfe4
I tried the GitHub code to make a USB connection with an iOS device and Python on a PC.
The above code was created in Python2, so I ran 2to3 -w usbmux.py to convert it to Python3 code.
The Python3 code is as follows.
import socket, struct, select, sys
try:
import plistlib
haveplist = True
except:
haveplist = False
class MuxError(Exception):
pass
class MuxVersionError(MuxError):
pass
class SafeStreamSocket:
def __init__(self, address, family):
self.sock = socket.socket(family, socket.SOCK_STREAM)
self.sock.connect(address)
def send(self, msg):
totalsent = 0
while totalsent < len(msg):
sent = self.sock.send(msg[totalsent:])
if sent == 0:
raise MuxError("socket connection broken")
totalsent = totalsent + sent
def recv(self, size):
msg = ''
while len(msg) < size:
chunk = self.sock.recv(size-len(msg))
if chunk == '':
raise MuxError("socket connection broken")
msg = msg + chunk
return msg
class MuxDevice(object):
def __init__(self, devid, usbprod, serial, location):
self.devid = devid
self.usbprod = usbprod
self.serial = serial
self.location = location
def __str__(self):
return "<MuxDevice: ID %d ProdID 0x%04x Serial '%s' Location 0x%x>"%(self.devid, self.usbprod, self.serial, self.location)
class BinaryProtocol(object):
TYPE_RESULT = 1
TYPE_CONNECT = 2
TYPE_LISTEN = 3
TYPE_DEVICE_ADD = 4
TYPE_DEVICE_REMOVE = 5
VERSION = 0
def __init__(self, socket):
self.socket = socket
self.connected = False
def _pack(self, req, payload):
if req == self.TYPE_CONNECT:
return struct.pack("IH", payload['DeviceID'], payload['PortNumber']) + "\x00\x00"
elif req == self.TYPE_LISTEN:
return ""
else:
raise ValueError("Invalid outgoing request type %d"%req)
def _unpack(self, resp, payload):
if resp == self.TYPE_RESULT:
return {'Number':struct.unpack("I", payload)[0]}
elif resp == self.TYPE_DEVICE_ADD:
devid, usbpid, serial, pad, location = struct.unpack("IH256sHI", payload)
serial = serial.split("\0")[0]
return {'DeviceID': devid, 'Properties': {'LocationID': location, 'SerialNumber': serial, 'ProductID': usbpid}}
elif resp == self.TYPE_DEVICE_REMOVE:
devid = struct.unpack("I", payload)[0]
return {'DeviceID': devid}
else:
raise MuxError("Invalid incoming request type %d"%req)
def sendpacket(self, req, tag, payload={}):
payload = self._pack(req, payload)
if self.connected:
raise MuxError("Mux is connected, cannot issue control packets")
length = 16 + len(payload)
data = struct.pack("IIII", length, self.VERSION, req, tag) + payload
self.socket.send(data)
def getpacket(self):
if self.connected:
raise MuxError("Mux is connected, cannot issue control packets")
dlen = self.socket.recv(4)
dlen = struct.unpack("I", dlen)[0]
body = self.socket.recv(dlen - 4)
version, resp, tag = struct.unpack("III",body[:0xc])
if version != self.VERSION:
raise MuxVersionError("Version mismatch: expected %d, got %d"%(self.VERSION,version))
payload = self._unpack(resp, body[0xc:])
return (resp, tag, payload)
class PlistProtocol(BinaryProtocol):
TYPE_RESULT = "Result"
TYPE_CONNECT = "Connect"
TYPE_LISTEN = "Listen"
TYPE_DEVICE_ADD = "Attached"
TYPE_DEVICE_REMOVE = "Detached" #???
TYPE_PLIST = 8
VERSION = 1
def __init__(self, socket):
if not haveplist:
raise Exception("You need the plistlib module")
BinaryProtocol.__init__(self, socket)
def _pack(self, req, payload):
return payload
def _unpack(self, resp, payload):
return payload
def sendpacket(self, req, tag, payload={}):
payload['ClientVersionString'] = 'usbmux.py by marcan'
if isinstance(req, int):
req = [self.TYPE_CONNECT, self.TYPE_LISTEN][req-2]
payload['MessageType'] = req
payload['ProgName'] = 'tcprelay'
BinaryProtocol.sendpacket(self, self.TYPE_PLIST, tag, plistlib.writePlistToString(payload))
def getpacket(self):
resp, tag, payload = BinaryProtocol.getpacket(self)
if resp != self.TYPE_PLIST:
raise MuxError("Received non-plist type %d"%resp)
payload = plistlib.readPlistFromString(payload)
return payload['MessageType'], tag, payload
class MuxConnection(object):
def __init__(self, socketpath, protoclass):
self.socketpath = socketpath
if sys.platform in ['win32', 'cygwin']:
family = socket.AF_INET
address = ('127.0.0.1', 27015)
else:
family = socket.AF_UNIX
address = self.socketpath
self.socket = SafeStreamSocket(address, family)
self.proto = protoclass(self.socket)
self.pkttag = 1
self.devices = []
def _getreply(self):
while True:
resp, tag, data = self.proto.getpacket()
if resp == self.proto.TYPE_RESULT:
return tag, data
else:
raise MuxError("Invalid packet type received: %d"%resp)
def _processpacket(self):
resp, tag, data = self.proto.getpacket()
if resp == self.proto.TYPE_DEVICE_ADD:
self.devices.append(MuxDevice(data['DeviceID'], data['Properties']['ProductID'], data['Properties']['SerialNumber'], data['Properties']['LocationID']))
elif resp == self.proto.TYPE_DEVICE_REMOVE:
for dev in self.devices:
if dev.devid == data['DeviceID']:
self.devices.remove(dev)
elif resp == self.proto.TYPE_RESULT:
raise MuxError("Unexpected result: %d"%resp)
else:
raise MuxError("Invalid packet type received: %d"%resp)
def _exchange(self, req, payload={}):
mytag = self.pkttag
self.pkttag += 1
self.proto.sendpacket(req, mytag, payload)
recvtag, data = self._getreply()
if recvtag != mytag:
raise MuxError("Reply tag mismatch: expected %d, got %d"%(mytag, recvtag))
return data['Number']
def listen(self):
ret = self._exchange(self.proto.TYPE_LISTEN)
if ret != 0:
raise MuxError("Listen failed: error %d"%ret)
def process(self, timeout=None):
if self.proto.connected:
raise MuxError("Socket is connected, cannot process listener events")
rlo, wlo, xlo = select.select([self.socket.sock], [], [self.socket.sock], timeout)
if xlo:
self.socket.sock.close()
raise MuxError("Exception in listener socket")
if rlo:
self._processpacket()
def connect(self, device, port):
ret = self._exchange(self.proto.TYPE_CONNECT, {'DeviceID':device.devid, 'PortNumber':((port<<8) & 0xFF00) | (port>>8)})
if ret != 0:
raise MuxError("Connect failed: error %d"%ret)
self.proto.connected = True
return self.socket.sock
def close(self):
self.socket.sock.close()
class USBMux(object):
def __init__(self, socketpath=None):
if socketpath is None:
if sys.platform == 'darwin':
socketpath = "/var/run/usbmuxd"
else:
socketpath = "/var/run/usbmuxd"
self.socketpath = socketpath
self.listener = MuxConnection(socketpath, BinaryProtocol)
try:
self.listener.listen()
self.version = 0
self.protoclass = BinaryProtocol
except MuxVersionError:
self.listener = MuxConnection(socketpath, PlistProtocol)
self.listener.listen()
self.protoclass = PlistProtocol
self.version = 1
self.devices = self.listener.devices
def process(self, timeout=None):
self.listener.process(timeout)
def connect(self, device, port):
connector = MuxConnection(self.socketpath, self.protoclass)
return connector.connect(device, port)
if __name__ == "__main__":
mux = USBMux()
print("Waiting for devices...")
if not mux.devices:
mux.process(0.1)
while True:
print("Devices:")
for dev in mux.devices:
print(dev)
mux.process()
I ran the above code in Python 3.7 and got the error.
The details of the error are as follows.
File "C:\Users\taichi\Documents\usbmux.py", line 81, in sendpacket
data = struct.pack("IIII", length, self.VERSION, req, tag) + payload
TypeError: can't concat str to bytes
How can I rewrite this code to get rid of the error?
struct.pack returns bytes in python 3, not a string as it did in python 2. Have a look at this answer to see one way to emulate the python 2 behavior.
Edit: actually you may need to do something slightly different, since it looks like socket.send wants bytes as input. So you'd need to convert the combined payload to bytes.
You could try changing the following code that seems to be populating the payload to return bytes in both cases. Notice the "b" added in the return values to specify bytes instead of str. You are hitting the TYPE_LISTEN case but should test the other case as well, as it looks like it had the same bug trying to concat str and bytes:
def _pack(self, req, payload):
if req == self.TYPE_CONNECT:
return struct.pack("IH", payload['DeviceID'], payload['PortNumber']) + b"\x00\x00"
elif req == self.TYPE_LISTEN:
return b""
else:
raise ValueError("Invalid outgoing request type %d"%req)
You'll likely need to edit _pack in PlistProtocol as well.
The line:
data = struct.pack("IIII", length, self.VERSION, req, tag) + payload
is causing the trouble. According to the struct.pack() docs, the method returns bytes. You need to convert your payload also into bytes so that the two can be concatenated, making the error go away.
Not sure what your payload is, but encoding the payload using something like 'utf-8' might do the trick. I believe a simple payload.encode() will work, since 'utf-8' is the default encoding. So, try:
data = struct.pack("IIII", length, self.VERSION, req, tag) + payload.encode()
# add this ^^^^^^^^^
I successfuly managed to get connected to tracker. After connection establishes, I request for scraping. Tracker returns a response with right byte order but (Seeder, Leecher, Completed) infos of torrent is always zero which seems silly.
I suspect hash info encoding problem but I couldn't come up with a solution. I use following python code block for scraping.
import os
import bencode
import struct
import socket
import urlparse
import binascii
def loadFile(f_name):
user_home = os.path.expanduser('~')
path = user_home+'/completetorrent/data/'+ f_name
return open(path,'rb')
def getSocket():
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
sock.settimeout(8)
return sock
def getConnection(url):
parsed = urlparse.urlparse(url)
hostname = socket.gethostbyname(parsed.hostname)
port = parsed.port
return (hostname,port)
def main():
f = loadFile('test1.torrent')
decoded = bencode.bdecode(f.read())
#Connect request starts
buff = ''
buff += struct.pack('!q',0x41727101980)
buff += struct.pack('!i',0)
buff += struct.pack('!i',123456789)
sock = getSocket()
conn = getConnection("udp://tracker.coppersurfer.tk:6969/announce")
sock.sendto(buff,conn)
#Connect Response starts
response = sock.recvfrom(4096)[0]
assert len(response) == 16, 'response len must be bigger than 16 but %d' % len(response)
if struct.unpack('!i',response[0:4])[0] != 0:
raise ValueError('Bad response')
struct.unpack('!i',response[4:8]) #ignore transaction id
#Scrape request starts
con_id = struct.unpack('!q',response[8:16])[0]
buff = ''
buff += struct.pack('!q',con_id)
buff += struct.pack('!i',2)
buff += struct.pack('!i',123456)
hash_ = decoded['info']['pieces'][0:20]
buff += struct.pack('!20s',hash_)
sock.sendto(buff,conn)
#Scrape response starts
response = sock.recvfrom(4096)[0]
if struct.unpack('!i',response[0:4])[0] != 2:
raise ValueError('Bad scrape response')
struct.unpack('!i',response[4:8]) #ignore transaction id
seedrs = struct.unpack('!i',response[8:12])
completed = struct.unpack('!i',response[12:16])
leechers = struct.unpack('!i',response[16:20])
print seedrs,completed,leechers
Convert the info_hash to hex value.
While I was writing simple message-based fileserver and client, I got the idea about checking fileserver status, but don't know how to realize this: just try to connect and disconnect from server (and how disconnect immediately, when server is not running, if using this way?) or maybe twisted/autobahn have some things, which help to get server status without creating "full connection"?
a) fileserver.py
import os
import sys
import json
from twisted.internet import reactor
from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol, listenWS
CONFIG_TEMPLATE = ''
CONFIG_DATA = {}
class MessageBasedServerProtocol(WebSocketServerProtocol):
"""
Message-based WebSockets server
Template contains some parts as string:
[USER_ID:OPERATION_NAME:FILE_ID] - 15 symbols for USER_ID,
10 symbols for OPERATION_NAME,
25 symbols for FILE_ID
other - some data
"""
def __init__(self):
path = CONFIG_DATA['path']
base_dir = CONFIG_DATA['base_dir']
# prepare to working with files...
if os.path.exists(path) and os.path.isdir(path):
os.chdir(path)
if not os.path.exists(base_dir) or not os.path.isdir(base_dir):
os.mkdir(base_dir)
os.chdir(base_dir)
else:
os.makedir(path)
os.chdir(path)
os.mkdir(base_dir)
os.chdir(base_dir)
# init some things
self.fullpath = path + '/' + base_dir
def __checkUserCatalog(self, user_id):
# prepare to working with files...
os.chdir(self.fullpath)
if not os.path.exists(user_id) or not os.path.isdir(user_id):
os.mkdir(user_id)
os.chdir(user_id)
else:
os.chdir(self.fullpath + '/' + user_id)
def onOpen(self):
print "[USER] User with %s connected" % (self.transport.getPeer())
def connectionLost(self, reason):
print '[USER] Lost connection from %s' % (self.transport.getPeer())
def onMessage(self, payload, isBinary):
"""
Processing request from user and send response
"""
user_id, cmd, file_id = payload[:54].replace('[', '').replace(']','').split(':')
data = payload[54:]
operation = "UNK" # WRT - Write, REA -> Read, DEL -> Delete, UNK -> Unknown
status = "C" # C -> Complete, E -> Error in operation
commentary = 'Succesfull!'
# write file into user storage
if cmd == 'WRITE_FILE':
self.__checkUserCatalog(user_id)
operation = "WRT"
try:
f = open(file_id, "wb")
f.write(data)
except IOError, argument:
status = "E"
commentary = argument
except Exception, argument:
status = "E"
commentary = argument
raise Exception(argument)
finally:
f.close()
# read some file
elif cmd == 'READU_FILE':
self.__checkUserCatalog(user_id)
operation = "REA"
try:
f = open(file_id, "rb")
commentary = f.read()
except IOError, argument:
status = "E"
commentary = argument
except Exception, argument:
status = "E"
commentary = argument
raise Exception(argument)
finally:
f.close()
# delete file from storage (and in main server, in parallel delete from DB)
elif cmd == 'DELET_FILE':
self.__checkUserCatalog(user_id)
operation = "DEL"
try:
os.remove(file_id)
except IOError, argument:
status = "E"
commentary = argument
except Exception, argument:
status = "E"
commentary = argument
raise Exception(argument)
self.sendMessage('[%s][%s]%s' % (operation, status, commentary), isBinary=True)
if __name__ == '__main__':
if len(sys.argv) < 2:
print "using python fileserver_client.py [PATH_TO_config.json_FILE]"
else:
# read config file
CONFIG_TEMPLATE = sys.argv[1]
with open(CONFIG_TEMPLATE, "r") as f:
CONFIG_DATA = json.load(f)
# create server
factory = WebSocketServerFactory("ws://localhost:9000")
factory.protocol = MessageBasedServerProtocol
listenWS(factory)
reactor.run()
b) client.py
import json
import sys
import commands
from twisted.internet import reactor
from autobahn.twisted.websocket import WebSocketClientFactory, WebSocketClientProtocol, connectWS
CONFIG_TEMPLATE = ''
CONFIG_DATA = {}
class MessageBasedClientProtocol(WebSocketClientProtocol):
"""
Message-based WebSockets client
Template contains some parts as string:
[USER_ID:OPERATION_NAME:FILE_ID] - 15 symbols for USER_ID,
10 symbols for OPERATION_NAME,
25 symbols for FILE_ID
other - some data
"""
def onOpen(self):
user_id = CONFIG_DATA['user']
operation_name = CONFIG_DATA['cmd']
file_id = CONFIG_DATA['file_id']
src_file = CONFIG_DATA['src_file']
data = '[' + str(user_id) + ':' + str(operation_name) + ':' + str(file_id) + ']'
if operation_name == 'WRITE_FILE':
with open(src_file, "r") as f:
info = f.read()
data += str(info)
self.sendMessage(data, isBinary=True)
def onMessage(self, payload, isBinary):
cmd = payload[1:4]
result_cmd = payload[6]
if cmd in ('WRT', 'DEL'):
print payload
elif cmd == 'REA':
if result_cmd == 'C':
try:
data = payload[8:]
f = open(CONFIG_DATA['src_file'], "wb")
f.write(data)
except IOError, e:
print e
except Exception, e:
raise Exception(e)
finally:
print payload[:8] + "Successfully!"
f.close()
else:
print payload
reactor.stop()
if __name__ == '__main__':
if len(sys.argv) < 2:
print "using python fileserver_client.py [PATH_TO_config.json_FILE]"
else:
# read config file
CONFIG_TEMPLATE = sys.argv[1]
with open(CONFIG_TEMPLATE, "r") as f:
CONFIG_DATA = json.load(f)
# connection to server
factory = WebSocketClientFactory("ws://localhost:9000")
factory.protocol = MessageBasedClientProtocol
connectWS(factory)
reactor.run()
Find solution this issue: using callLater or deferLater for disconnect from server, if can't connect, but when all was 'OK', just take server status, which he says.
import sys
from twisted.internet.task import deferLater
from twisted.internet import reactor
from autobahn.twisted.websocket import WebSocketClientFactory, WebSocketClientProtocol, connectWS
CONFIG_IP = ''
CONFIG_PORT = 9000
def isOffline(status):
print status
class StatusCheckerProtocol(WebSocketClientProtocol):
def __init__(self):
self.operation_name = "STATUS_SRV"
self.user_id = 'u00000000000000'
self.file_id = "000000000000000000000.log"
def onOpen(self):
data = '[' + str(self.user_id) + ':' + str(self.operation_name) + ':' + str(self.file_id) + ']'
self.sendMessage(data, isBinary=True)
def onMessage(self, payload, isBinary):
cmd = payload[1:4]
result_cmd = payload[6]
data = payload[8:]
print data
reactor.stop()
if __name__ == '__main__':
if len(sys.argv) < 3:
print "using python statuschecker.py [IP] [PORT]"
else:
# read preferences
CONFIG_IP = sys.argv[1]
CONFIG_PORT = int(sys.argv[2])
server_addr = "ws://%s:%d" % (CONFIG_IP, CONFIG_PORT)
# connection to server
factory = WebSocketClientFactory(server_addr)
factory.protocol = StatusCheckerProtocol
connectWS(factory)
# create special Deffered, which disconnect us from some server, if can't connect within 3 seconds
d = deferLater(reactor, 3, isOffline, 'OFFLINE')
d.addCallback(lambda ignored: reactor.stop())
# run all system...
reactor.run()
I wrote a simple Python script for a proxy functionality. It works fine, however, if the requested webpage has many other HTTP requests, e.g. Google maps, the page is rendered quite slow.
Any hints as to what might be the bottleneck in my code, and how I can improve?
#!/usr/bin/python
import socket,select,re
from threading import Thread
class ProxyServer():
def __init__(self, host, port):
self.host=host
self.port=port
self.sk1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def startServer(self):
self.sk1.bind((self.host,self.port))
self.sk1.listen(256)
print "proxy is ready for connections..."
while(1):
conn,clientAddr = self.sk1.accept()
# print "new request coming in from " + str(clientAddr)
handler = RequestHandler(conn)
handler.start()
class RequestHandler(Thread):
def __init__(self, sk1):
Thread.__init__(self)
self.clientSK = sk1
self.buffer = ''
self.header = {}
def run(self):
sk1 = self.clientSK
sk2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
while 1:
self.buffer += sk1.recv(8192)
if self.buffer.find('\n') != -1:
break;
self.header = self.processHeader(self.buffer)
if len(self.header)>0: #header got processed
hostString = self.header['Host']
host=port=''
if hostString.__contains__(':'): # with port number
host,port = hostString.split(':')
else:
host,port = hostString,"80"
sk2.connect((host,int(port)))
else:
sk1.send('bad request')
sk1.close();
return
inputs=[sk1,sk2]
sk2.send(self.buffer)
#counter
count = 0
while 1:
count+=1
rl, wl, xl = select.select(inputs, [], [], 3)
if xl:
break
if rl:
for x in rl:
data = x.recv(8192)
if x is sk1:
output = sk2
else:
output = sk1
if data:
output.send(data)
count = 0
if count == 20:
break
sk1.close()
sk2.close()
def processHeader(self,header):
header = header.replace("\r\n","\n")
lines = header.split('\n')
result = {}
uLine = lines[0] # url line
if len(uLine) == 0: return result # if url line empty return empty dict
vl = uLine.split(' ')
result['method'] = vl[0]
result['url'] = vl[1]
result['protocol'] = vl[2]
for line in lines[1: - 1]:
if len(line)>3: # if line is not empty
exp = re.compile(': ')
nvp = exp.split(line, 1)
if(len(nvp)>1):
result[nvp[0]] = nvp[1]
return result
if __name__ == "__main__":
HOST, PORT = "0.0.0.0", 8088
proxy = ProxyServer(HOST,PORT)
proxy.startServer()
I'm not sure what your speed problems are, but here are some other nits I found to pick:
result['protocal'] = vl[2]
should be
result['protocol'] = vl[2]
This code is indented one level too deep:
sk2.connect((host,int(port)))
You can use this decorator to profile your individual methods by line.