Develop Reference

Flashbots "X-Flashbots-Signature" header not working correctly with web3.py

Recently I've been having some trouble with the X-Flashbots-Signature header when sending a request to the flashbots goerli endpoint.
My python code looks like this:
import requests
import json
import secrets
from eth_account import Account, messages
from web3 import Web3
from math import ceil
rpcUrl = GOERLI_RPC_NODE_PROVIDER
web3 = Web3(Web3.HTTPProvider(rpcUrl))
publicKey = ETH_PUBLIC_KEY
privateKey = ETH_PRIVATE_KEY
contractAddress = GOERLI_TEST_CONTRACT # Goerli test contract
data = CONTRACT_DATA # Contract data to execute
signed = []
for _ in range(2):
nonce = web3.eth.getTransactionCount(publicKey, 'pending')
checksumAddress = Web3.toChecksumAddress(contractAddress)
checksumPublic = Web3.toChecksumAddress(publicKey)
tx = {
'nonce': nonce,
'to': checksumAddress,
'from': checksumPublic,
'value': 0,
'gasPrice': web3.toWei(200, 'gwei'),
'data': data
}
gas = web3.eth.estimateGas(tx)
tx['gas'] = ceil(gas + gas * .1)
signed_tx = web3.eth.account.signTransaction(tx, privateKey)
signed.append(Web3.toHex(signed_tx.rawTransaction))
dt = {
'jsonrpc': '2.0',
'method': 'eth_sendBundle',
'params': [
{
'txs': [
signed[0], signed[1] # Signed txs with web3.eth.account.signTransaction
],
'blockNumber': web3.eth.block_number + 1,
'minTimestamp': '0x0',
'maxTimestamp': '0x0',
'revertingTxHashes': []
}
],
'id': 1337
}
pvk = secrets.token_hex(32)
pbk = Account.from_key(pvk).address
body = json.dumps(dt)
message = messages.encode_defunct(text=Web3.keccak(text=body).hex())
signature = pbk + ':' + Account.sign_message(message, pvk).signature.hex()
hd = {
'Content-Type': 'application/json',
'X-Flashbots-Signature': signature,
}
res = requests.post('https://relay-goerli.flashbots.net/', headers=hd, data=body)
print(res.text)
This code is a modified version of code taken straight from the flashbots docs: https://docs.flashbots.net/flashbots-auction/searchers/advanced/rpc-endpoint/#authentication
Upon running this code I get an internal server error error response. At first, I thought the problem might be fixed by replacing text=Web3.keccak(text=body).hex() to hexstr=Web3.keccak(text=body).hex() or primative=Web3.keccak(text=body), as per the definition of messages.encode_defunct: https://eth-account.readthedocs.io/en/stable/eth_account.html#eth_account.messages.encode_defunct. But after making this replacement, I got the error signer address does not equal expected. This is very confusing, especially because I have resolved the message
with the signature myself and the public key does match. But whenever I send it to the flashbots endpoint, I am left with this error.
Any ideas would be greatly appreciated.

python function return none while data is printed as desired

I have a python script as below that gather some stores' informations using google API :
import json
import requests
import time
from config import API_KEY, PLACES_ENDPOINT
def getPlaces(lat, lng, radius, keyword, token=None, places=None):
# Building my API call
params = {
'location': str(lat) + ',' + str(lng),
'radius': radius,
'keyword': keyword,
'key': API_KEY
}
if token is not None:
params.update({'pagetoken': token})
r = requests.get(PLACES_ENDPOINT, params=params)
# If its the first time the function is called, then we create the list variable
if places is None:
places = []
for item in r.json()['results']:
# adding element to my list 'places'
places.append(item['name'])
# if there is more results to gather from the next page, call the func again
if 'next_page_token' in r.json():
token = r.json()['next_page_token']
time.sleep(2)
getPlaces(lat, lng, radius, keyword, token, places)
else:
print(json.dumps(places)) # print all the data collected
return json.dumps(places) # return None
Why the returned json places is None whereas the data appears as it should be in the final print ?

To fix my issues I just had to get rid of the else statement.
As the function is recursive, it won't proceed to the return until it ends looping :
import json
import requests
import time
from config import API_KEY, PLACES_ENDPOINT
def getPlaces(lat, lng, radius, keyword, token=None, places=None):
params = {
'location': str(lat) + ',' + str(lng),
'radius': radius,
'keyword': keyword,
'key': API_KEY
}
if token is not None:
params.update({'pagetoken': token})
r = requests.get(PLACES_ENDPOINT, params=params)
# parsing json
if places is None:
places = []
for item in r.json()['results']:
places.append(item['name'])
if 'next_page_token' in r.json():
token = r.json()['next_page_token']
time.sleep(2)
getPlaces(lat, lng, radius, keyword, token, places)
return json.dumps(places)
Thanks Michael Butscher for the hint !

Bitmex Api for authorization

This is what i have try:
import time
import json
import hashlib
import hmac
from urllib.parse import urlparse
import requests
from datetime import datetime, timedelta
from pprint import pprint
def generate_signature(secret, verb, url, expires, data):
"""Generate a request signature compatible with BitMEX."""
# Parse the url so we can remove the base and extract just the path.
parsedURL = urlparse(url)
path = parsedURL.path
if parsedURL.query:
path = path + '?' + parsedURL.query
if isinstance(data, (bytes, bytearray)):
data = data.decode('utf8')
print("Computing HMAC: %s" % verb + path + str(expires) + data)
message = verb + path + str(expires) + data
signature = hmac.new(bytes(secret, 'utf8'), bytes(message, 'utf8'), digestmod=hashlib.sha256).hexdigest()
return signature
# Or you might generate it like so:
# expires = int(round(time.time()) + 5)
data = {
'filter': {"symbol": "XBTUSD"},
'columns': [],
'count': 100
'start': 1,
'reverse': "true",
'startTime': str(datetime.now() - timedelta(minutes=50)),
'endTime': str(datetime.now())
}
verb = 'GET'
path = '/api/v1/position'
time_in_epoch = int(datetime.timestamp(datetime.now() + timedelta(hours=.5)))
print("========",time_in_epoch, int(round(time.time()) + 1800))
expires = time_in_epoch
gen_signat = generate_signature('api-sceret', verb, path, expires, json.dumps(data))
URL_get_data = "http://www.bitmex.com/api/v1/position"
headers = {
'api-expires': str(expires),
'api-key': "api-key",
'api-signature': str(gen_signat),
}
response = requests.get(URL_get_data, headers=headers, data= data).json()
Still showing error:
{'error': {'message': 'Signature not valid.', 'name': 'HTTPError'}}
If I pass this gen_signat = generate_signature('api-sceret', verb, path, expires, data) the showing reponse<200> but returns empty list.

Looks like you're basing on this Bitmex sample. Please note that data must be JSON without whitespace between keys. The default separators value for json.dumps is (', ', ': '), so you need to change your code to use (',', ':'). It should be gen_signat = generate_signature('api-sceret', verb, path, expires, json.dumps(data,separators=(',', ':'))).

Signed request with python to binance future

I have been struggling to send a signed request to binance future using signature.
I found that example code on StackOverflow ("Binance API call with SHA56 and Python requests") and an answer has been given to it mentioning to use hmac
as below: but unfortunately i still don't see how to write this example. Could anyone show how the code of this example should look like? i am really uncomfortable with signed request. Thanks a lot for your understanding and your help advice given:
params = urlencode({
"signature" : hashedsig,
"timestamp" : servertimeint,
})
hashedsig = hmac.new(secret.encode('utf-8'), params.encode('utf-8'), hashlib.sha256).hexdigest()
Original example:
import requests, json, time, hashlib
apikey = "myactualapikey"
secret = "myrealsecret"
test = requests.get("https://api.binance.com/api/v1/ping")
servertime = requests.get("https://api.binance.com/api/v1/time")
servertimeobject = json.loads(servertime.text)
servertimeint = servertimeobject['serverTime']
hashedsig = hashlib.sha256(secret)
userdata = requests.get("https://api.binance.com/api/v3/account",
params = {
"signature" : hashedsig,
"timestamp" : servertimeint,
},
headers = {
"X-MBX-APIKEY" : apikey,
}
)
print(userdata)

The proper way would be:
apikey = "myKey"
secret = "mySecret"
servertime = requests.get("https://api.binance.com/api/v1/time")
servertimeobject = json.loads(servertime.text)
servertimeint = servertimeobject['serverTime']
params = urlencode({
"timestamp" : servertimeint,
})
hashedsig = hmac.new(secret.encode('utf-8'), params.encode('utf-8'),
hashlib.sha256).hexdigest()
userdata = requests.get("https://api.binance.com/api/v3/account",
params = {
"timestamp" : servertimeint,
"signature" : hashedsig,
},
headers = {
"X-MBX-APIKEY" : apikey,
}
)
print(userdata)
print(userdata.text)
Make sure to put the signature as the last parameter or the request will return [400]...
Incorrect:
params = {
"signature" : hashedsig,
"timestamp" : servertimeint,
}
Correct:
params = {
"timestamp" : servertimeint,
"signature" : hashedsig,
}

At the time of writing, Binance themselves are mainting a repo with some examples*, using the requests library. Here is a sample in case the link goes down or is moved:
import hmac
import time
import hashlib
import requests
from urllib.parse import urlencode
KEY = ''
SECRET = ''
# BASE_URL = 'https://fapi.binance.com' # production base url
BASE_URL = 'https://testnet.binancefuture.com' # testnet base url
''' ====== begin of functions, you don't need to touch ====== '''
def hashing(query_string):
return hmac.new(SECRET.encode('utf-8'), query_string.encode('utf-8'), hashlib.sha256).hexdigest()
def get_timestamp():
return int(time.time() * 1000)
def dispatch_request(http_method):
session = requests.Session()
session.headers.update({
'Content-Type': 'application/json;charset=utf-8',
'X-MBX-APIKEY': KEY
})
return {
'GET': session.get,
'DELETE': session.delete,
'PUT': session.put,
'POST': session.post,
}.get(http_method, 'GET')
# used for sending request requires the signature
def send_signed_request(http_method, url_path, payload={}):
query_string = urlencode(payload)
# replace single quote to double quote
query_string = query_string.replace('%27', '%22')
if query_string:
query_string = "{}&timestamp={}".format(query_string, get_timestamp())
else:
query_string = 'timestamp={}'.format(get_timestamp())
url = BASE_URL + url_path + '?' + query_string + '&signature=' + hashing(query_string)
print("{} {}".format(http_method, url))
params = {'url': url, 'params': {}}
response = dispatch_request(http_method)(**params)
return response.json()
# used for sending public data request
def send_public_request(url_path, payload={}):
query_string = urlencode(payload, True)
url = BASE_URL + url_path
if query_string:
url = url + '?' + query_string
print("{}".format(url))
response = dispatch_request('GET')(url=url)
return response.json()
response = send_signed_request('POST', '/fapi/v1/order', params)
print(response)
Some additional thoughts from myself:
You can also use a new library also from Binance called Binance connector. It is a bit new, it has some issues, but it can do the basic operations without you worrying about signed requests.
I wouldn't use serverTime because that means you need to make an additional request and networks can be slow, I'd follow this example and use the int(time.time() * 1000) you may not even need the function.
I purposedly used the POST example, because this is more complicated as you need to also encode and hash your custom parameters
At the time of writing, v3 is the latest version
Hope it helps.
* https://github.com/binance/binance-signature-examples/blob/master/python/futures.py

Convert a column of json strings into columns of data (API Result / Output)

I have the exact same request as below:
Convert a column of json strings into columns of data
Following the suggested solutions and getting the following error: What am I doing incorrectly? I'm learning to use python and make API requests.
***df['json'] = df['json'].map(address',': dict(eval(address)))***
^
`**SyntaxError: invalid syntax**
Below is my code
import requests
import json
import pandas as pd
import dictionary as dict
Base_url = 'MY_URL'
TOKEN_EndPoint = Base_url + 'token'
Account_EndPoint = Base_url + 'MY_URL'
data = {
'username': 'MY_USERNAME',
'password': 'MY_PASSWORD',
'grant_type': 'MY_GRANT_TYPE'
}
def main():
results = requests.post(url=TOKEN_EndPoint, data=data)
MyToken = results.json()['access_token']
print(MyToken)
MyInputs = GetSourceAddress()
callData = {
'ClientKey': 'MY_CLIENTKEY',
'StreetName': MyInputs['StreetName'],
'CityName' : MyInputs['CityName'],
# 'StateCode' : MyInputs['StateCode'],
'PostalCode' : MyInputs['PostalCode'],
'ManagerVersion': '2'
}
PostFields = json.dumps(callData)
MyHeader = {'Authorization': 'Bearer ' + MyToken,
'content-type': 'application/json'}
results = requests.post(url = Account_EndPoint, data = PostFields,
headers = MyHeader)
address = results.json()
# results.json() = pd.DataFrame(address)
df = pd.DataFrame(['address'], columns=['json'])
**df['json'] = df['json'].map(address',': dict(eval(address)))**
address = df['json'].apply(pd.Series)
for address in address:
print(address)
def GetSourceAddress():
MyInputs = {
'StreetName': 'MY_STREETNAME',
'CityName': 'MY_CITYNAME',
# 'StateCode': 'MY_STATE',
'PostalCode': 'MY_ZIPCODE',
# 'Unit': 'UNIT #'
}
return MyInputs
def GetAddressFrom****(result):
inputs = {
'StreetName': result['ShippingStreet'],
'CityName': result['BillingCity'],
'StateCode': result['BillingState'],
'PostalCode': result['BillingPostalCode'],
# 'Unit': ''
}
return inputs
main()
Thanks