Is it possible to use python requests with ssh tunnel as proxy ? How can I achieve that ? Tried already this but no success :
from sshtunnel import SSHTunnelForwarder
import requests as r
from bs4 import BeautifulSoup as soup
server = SSHTunnelForwarder(
'sship',
ssh_username="XXXX",
ssh_password="XXXX",
remote_bind_address=('127.0.0.1', 8080)
)
server.start()
print(server.local_bind_port)
proxies = {
"http": "http://127.0.0.1:8080",
}
url = 'http://www.google.com'
headers = {
'User-Agent': 'My User Agent 1.0',
'From': 'youremail#domain.com' # This is another valid field
}
data = r.get(url, headers=headers , proxies = proxies)
page_data = soup(data.text , 'html.parser')
print page_data
this is the error that I get:
37657
Traceback (most recent call last):
File "ssh.py", line 27, in
data = r.get(url, headers=headers , proxies = proxies)
File "/usr/lib/python2.7/site-packages/requests-2.19.1-py2.7.egg/requests/api.py", line 72, in get
return request('get', url, params=params, **kwargs)
File "/usr/lib/python2.7/site-packages/requests-2.19.1-py2.7.egg/requests/api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python2.7/site-packages/requests-2.19.1-py2.7.egg/requests/sessions.py", line 512, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python2.7/site-packages/requests-2.19.1-py2.7.egg/requests/sessions.py", line 622, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python2.7/site-packages/requests-2.19.1-py2.7.egg/requests/adapters.py", line 507, in send
raise ProxyError(e, request=request)
requests.exceptions.ProxyError: HTTPConnectionPool(host='127.0.0.1', port=80): Max retries exceeded with url: http://google.com/ (Caused by ProxyError('Cannot connect to proxy.', NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused',)))
I think you need to either explicitly set the local bind address in SSHTunnel, or set up the port for your proxy like this:
`
https_tunnel.start()
proxies = {
#'http': 'http://localhost:{}'.format(str(http_tunnel.local_bind_port)),
'https': 'https://localhost:{}'.format(str(https_tunnel.local_bind_port))
}
`
where https_tunnel is an instance of SSHTunnelForwarder. I've done that, however I get errors around actually sending the GET request:
python2.7/site-packages/requests/adapters.py", line 490, in send
raise ConnectionError(err, request=request) requests.exceptions.ConnectionError: ('Connection aborted.',
BadStatusLine("''",))
Related
I have an API, where I need to get the bearer token. When I use 'Postman' application, I get the bearer token correctly. I have written below python code for the same but I get below errors. Please help. I need to send username and password in the body as a form data.
import requests
url = "https://322.286.24.01/ach/ach_api/login"
payload={'username': 'test',
'password': 'test12'}
response = requests.post( url,data=payload)
print(response.text)
ERROR:
Traceback (most recent call last):
File "/tmp/test.py", line 13, in <module>
response = requests.post(url,data=payload)
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 119, in post
return request('post', url, data=data, json=json, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 514, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='322.286.24.01', port=443): Max retries exceeded with url: /ach/ach_api/login (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate (_ssl.c:1123)')))
The error is because requests is trying to check for the ca cert, try the following:
response = requests.post(url, data=payload, verify=False)
Or, if you have a ca.crt somewhere, usually pem format, you can try:
response = requests.post(url, data=payload, verify='/path/to/pem')
Also the IP address looks funny, although since the client connects I suspect you just changed that to anonymise your post?
I am trying to get this to work:
Practitioner notes -> Python scripts for automation in NA -> Python client to invoke NA SOAP APIs
Here is my code (sanitized a bit):
#! /usr/bin/env python3
from requests import Session
from zeep import Client
from zeep.transports import Transport
session = Session()
session.verify = False
transport = Transport(session=session)
client = Client( 'https://SERVER_FQDN/soap?wsdl=api.wsdl.wsdl2py', transport=transport)
# I added this for the network proxy
client.transport.session.proxies = {
'http': '10.0.0.1:80',
'https': '10.0.0.1:80',
}
# Then found I needed this because "localhost" is hard-coded in the WSDL
client.service._binding_options['address'] = 'https://SERVER_FQDN/soap'
login_params = {
'username':'user',
'password':'PASSWORD',
}
loginResult = client.service.login(parameters=login_params )
sesnhdr_type = client.get_element('ns0:list_deviceInputParms')
sesnhdr = sesnhdr_type(sessionid=loginResult.Text)
devices = client.service.list_device(_soapheaders=[sesnhdr], parameters=sesnhdr)
print('\n\n ----------------------------- \n')
for i in devices.ResultSet.Row:
print(i.hostName + ' ---> '+i.primaryIPAddress)
params = {
"ip":i.primaryIPAddress,
"sessionid": loginResult.Text
}
device = client.service.show_deviceinfo(parameters=params)
print(device.Text)
print('\n\n ----------------------------- \n')
And here is my output:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 667, in urlopen
self._prepare_proxy(conn)
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 932, in _prepare_proxy
conn.connect()
File "/usr/local/lib/python3.6/site-packages/urllib3/connection.py", line 317, in connect
self._tunnel()
File "/usr/lib64/python3.6/http/client.py", line 929, in _tunnel
message.strip()))
OSError: Tunnel connection failed: 503 Service Unavailable
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 727, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/usr/local/lib/python3.6/site-packages/urllib3/util/retry.py", line 439, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='SERVER_FQDN', port=443): Max retries exceeded with url: /soap (Caused by ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 503 Service Unavailable',)))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./na-1.py", line XX, in <module>
loginResult = client.service.login(parameters=login_params )
File "/usr/local/lib/python3.6/site-packages/zeep/proxy.py", line 51, in __call__
kwargs,
File "/usr/local/lib/python3.6/site-packages/zeep/wsdl/bindings/soap.py", line 127, in send
response = client.transport.post_xml(options["address"], envelope, http_headers)
File "/usr/local/lib/python3.6/site-packages/zeep/transports.py", line 107, in post_xml
return self.post(address, message, headers)
File "/usr/local/lib/python3.6/site-packages/zeep/transports.py", line 74, in post
address, data=message, headers=headers, timeout=self.operation_timeout
File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 578, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.6/site-packages/requests/adapters.py", line 510, in send
raise ProxyError(e, request=request)
requests.exceptions.ProxyError: HTTPSConnectionPool(host='SERVER_FQDN', port=443): Max retries exceeded with url: /soap (Caused by ProxyError('Cannot connect to proxy.', OSError('Tunnel connection failed: 503 Service Unavailable',)))
I get the same errors if I use "localhost" and run the script on the server-in-question.
The system has proxy environment values set.
There are proper forward and reverse DNS entries for the server.
The name and IP for the server are also in /etc/hosts
Here is the problem:
If I use an IP address instead of the server's FQDN, the code runs.
Vendor support says the problem is not in their application that provides the endpoint:
The 503 error means that the service is not available, there are 3 situations that invoke this behavior: 1. The server is under maintenance, 2. The server is overloaded, 3. In rare cases, the DNS configuration is faulty. If we see, this problem is not related to NA because the request is working fine with the IP.
Any ideas on this ?
Why does only the IP work and NOT the FQDN or localhost ?
Most of the documentation I see for using proxies with Zeep start with client = Client(url) but that doesn't work if url is behind a firewall and can't be accessed except through a proxy! My attempt to do it according to the documentation did nothing but time out (of course).
The key is in understanding that Zeep is built on requests and requests can use proxies for initiating the session. So you need to build a proxied Session, then pass that session into the Transport, and initialize the Zeep client with that transport. This worked for me:
session = requests.Session()
session.auth = requests.auth.HTTPBasicAuth(soap_username, soap_password)
session.proxies = {"https": f"socks5://{settings.STATIC_PROXY}"}
transport = zeep.transports.Transport(session=session, timeout=(5, 30))
client = zeep.Client(url, transport=transport)
My problem lay in the fact that the initialization of Client wants to go ahead and make the connection, but I need the proxy setting at the start. So I cobbled together two examples from the official docs to set the proxy at the time the connection is made.
from zeep import Client
from zeep.transports import Transport
from requests import Session
session = Session()
session.proxies = {
'http': 'http://username:password#proxy.example.com:8080',
'https': 'http://username:password#proxy.example.com:8080'
}
transport=Transport(session=session)
client = Client(URL,transport=transport)
What am I missing?
HINT: I've also tried using urllib module
import requests
import sys
import time
import random
headers = {"User-Agent": "Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/25.0"}
url = "HTTP LINK TO YOUTUBE VIDEO"
views = 10
videoMins = 3
videoSec = 33
refreshRate = videoMins * 60 + videoSec
proxy_list = [
{"http":"49.156.37.30:65309"}, {"http":"160.202.42.106:8080"},
{"http":"218.248.73.193:808"}, {"http":"195.246.57.154:8080"},
{"http":"80.161.30.156:80"}, {"http":"122.228.25.97:8101"},
{"http":"165.84.167.54:8080"},{"https":"178.140.216.229:8080"},
{"https":"46.37.193.74:3128"},{"https":"5.1.27.124:53281"},
{"https":"196.202.194.127:62225"},{"https":"194.243.194.51:8080"},
{"https":"206.132.165.246:8080"},{"https":"92.247.127.177:3128"}]
proxies = random.choice(proxy_list)
while True:
for view in range(views): # to loop in the number of allocated views
s = requests.Session()
s.get(url, headers=headers, proxies=proxies, stream=True, timeout=refreshRate)
s.close()
time.sleep(60) # time between loops so we appear real
sys.exit()
Here's the traceback error I got:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "pytest.py", line 24, in <module>
s.get(url, headers=headers, proxies=proxies, stream=True,
timeout=refreshRate)
File "C:\Python\lib\site-packages\requests\sessions.py", line 521, in get
return self.request('GET', url, **kwargs)
File "C:\Python\lib\site-packages\requests\sessions.py", line 508, in
request
resp = self.send(prep, **send_kwargs)
File "C:\Python\lib\site-packages\requests\sessions.py", line 640, in send
history = [resp for resp in gen] if allow_redirects else []
File "C:\Python\lib\site-packages\requests\sessions.py", line 640, in
<listcomp>
history = [resp for resp in gen] if allow_redirects else []
File "C:\Python\lib\site-packages\requests\sessions.py", line 218, in
resolve_redirects
**adapter_kwargs
File "C:\Python\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Python\lib\site-packages\requests\adapters.py", line 506, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.youtube.com',
port=443): Max retries exceed
ch?v=dHUP25DkKWo (Caused by SSLError(SSLError("bad handshake:
SysCallError(-1, 'Unexpected EOF')",),))
I suspect max retries from youtube. But its confusing because I'm connecting via random proxies. If that's the case, maybe the proxies aren't working...or no https connection was made.
I am trying to develop a small tiny application using flask and requests module. I tried to post some data to a flask web application. But I stuck at this error.
flask.py
#app.route('/add/', methods=['POST'])
def add_paths():
paths = request.form['paths']
tags = 'others'
for path in paths:
g.db.execute('insert into entries (path, tags) values(?, ?)',
[path, tags])
g.db.commit()
message = 'New paths are posted'
return jsonify(result = message)
command line file for posting the data
import json
import glob2
import requests
list = glob2.glob('/home/sheeshmohsin/*/**')
post_data = {'paths' : list }
headers = {'content-type': 'application/json'}
post_response = requests.post(url='http://localhost:9696/add/', data=json.dumps(post_data), headers=headers)
print post_response
print post_response.text
And the error i am getting is:-
File "commandline.py", line 8, in <module>
post_response = requests.post(url='http://127.0.0.1:9696/add/', data=json.dumps(post_data), headers=headers)
File "/usr/lib/python2.7/site-packages/requests/api.py", line 88, in post
return request('post', url, data=data, **kwargs)
File "/usr/lib/python2.7/site-packages/requests/api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python2.7/site-packages/requests/sessions.py", line 335, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python2.7/site-packages/requests/sessions.py", line 438, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python2.7/site-packages/requests/adapters.py", line 327, in send
raise ConnectionError(e)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='127.0.0.1', port=9696): Max retries exceeded with url: /add/ (Caused by <class 'socket.error'>: [Errno 111] Connection refused)
The connection refused error message indicates it could be your firewall, or loopback:
Check that firewall isn't blocking port 9696.
I've had the loopback (127.0.0.1) cause similar issues, so replace localhost by actual IP address so that Python doesn't use loopback interface (requests.post(url='http://localhost:9696/add/',...). You might have to do same with Flask (app..run(host='192.168...', port=9696)).
I have my backend developed in java which does all kind of processing. And my frontend is developed using python's flask framework. I am using requests to send a request and get a response from the apis present in java.
Following is the line in my code which does that:
req = requests.post(buildApiUrl.getUrl('user') + "/login", data=payload)
My problem is, sometimes when the tomcat instance is not running or there is some issue with java apis, I always get an error from requests as follows:
ERROR:root:HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /MYAPP/V1.0/user/login (Caused by <class 'socket.error'>: [Errno 111] Connection refused)
Traceback (most recent call last):
File "/home/rahul/git/myapp/webapp/views/utils.py", line 31, in decorated_view
return_value = func(*args, **kwargs)
File "/home/rahul/git/myapp/webapp/views/public.py", line 37, in login
req = requests.post(buildApiUrl.getUrl('user') + "/login", data=payload)
File "/home/rahul/git/myapp/venv/local/lib/python2.7/site-packages/requests/api.py", line 88, in post
return request('post', url, data=data, **kwargs)
File "/home/rahul/git/myapp/venv/local/lib/python2.7/site-packages/requests/api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "/home/rahul/git/myapp/venv/local/lib/python2.7/site-packages/requests/sessions.py", line 335, in request
resp = self.send(prep, **send_kwargs)
File "/home/rahul/git/myapp/venv/local/lib/python2.7/site-packages/requests/sessions.py", line 438, in send
r = adapter.send(request, **kwargs)
File "/home/rahul/git/myapp/venv/local/lib/python2.7/site-packages/requests/adapters.py", line 327, in send
raise ConnectionError(e)
ConnectionError: HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /MYAPP/V1.0/user/login (Caused by <class 'socket.error'>: [Errno 111] Connection refused)
I want to handle any such errors that I receive in my flask app so that I can give the necessary response on the web page instead of showing blank screen. So how can I achieve this?
Catch the exception request.post raises using try-except:
try:
req = requests.post(buildApiUrl.getUrl('user') + "/login", data=payload)
except requests.exceptions.RequestException:
# Handle exception ..