#!/usr/local/bin/python3.7
#Producer.py
from kafka import KafkaProducer
kafkaBrokers='kafka1.xyz.com:443,kafka2.xyz.com:443,kafka3.xyz.com:443'
caRootLocation='CARoot.pem'
certLocation='certificate.pem'
keyLocation='key.pem'
topic='demo_topic'
password='something'
producer = KafkaProducer(bootstrap_servers=kafkaBrokers,
security_protocol='SSL',
ssl_check_hostname=True,
ssl_cafile=caRootLocation,
ssl_certfile=certLocation,
ssl_keyfile=keyLocation,
ssl_password=password)
# Send to a particular partition
producer.send(topic, bytes('Hello Kafka!','utf-8'),partition=0)
producer.flush()
Followed this approach below:
https://dev.to/adityakanekar/connecting-to-kafka-cluster-using-ssl-with-python-k2e
enter code here
-------------------- Error --------------------------------- Traceback (most recent call last): File "kafka_with_pem.py", line 20, in
ssl_password=password) File "/home/vdina01/.local/lib/python3.7/site-packages/kafka/producer/kafka.py",
line 383, in init
**self.config) File "/home/vdina01/.local/lib/python3.7/site-packages/kafka/client_async.py",
line 244, in init
self.config['api_version'] = self.check_version(timeout=check_timeout) File
"/home/vdina01/.local/lib/python3.7/site-packages/kafka/client_async.py",
line 909, in check_version
version = conn.check_version(timeout=remaining, strict=strict, topics=list(self.config['bootstrap_topics_filter'])) File
"/home/vdina01/.local/lib/python3.7/site-packages/kafka/conn.py", line
1238, in check_version
if not self.connect_blocking(timeout_at - time.time()): File "/home/vdina01/.local/lib/python3.7/site-packages/kafka/conn.py", line
340, in connect_blocking
self.connect() File "/home/vdina01/.local/lib/python3.7/site-packages/kafka/conn.py", line
401, in connect
self._wrap_ssl() File "/home/vdina01/.local/lib/python3.7/site-packages/kafka/conn.py", line
484, in _wrap_ssl
password=self.config['ssl_password']) ssl.SSLError: [SSL] PEM lib (_ssl.c:3845)
Related
I am getting this "Forbidden for url" error when using zeep. Does someone knows what the root cause is?
I have tried to use some options from the Client object of zeep but to no avail.
Or is it something wrong in my WSDL file?
The strange part is that this code used to work fine a few months ago, and suddenly started getting this error.
This is the trace back from the error:
Traceback (most recent call last):
File "/usr/src/app/attpcdaq/daq/tasks.py", line 33, in eccserver_refresh_state_task
ecc_server.refresh_state()
File "/usr/src/app/attpcdaq/daq/models.py", line 428, in refresh_state
client = self._get_soap_client()
File "/usr/src/app/attpcdaq/daq/models.py", line 308, in _get_soap_client
return EccClient(self.ecc_url)
File "/usr/src/app/attpcdaq/daq/models.py", line 65, in __init__
client = SoapClient(wsdl_url) # Loads the service definition from ecc.wsdl
File "/usr/local/lib/python3.7/site-packages/zeep/client.py", line 73, in __init__
self.wsdl = Document(wsdl, self.transport, settings=self.settings)
File "/usr/local/lib/python3.7/site-packages/zeep/wsdl/wsdl.py", line 92, in __init__
self.load(location)
File "/usr/local/lib/python3.7/site-packages/zeep/wsdl/wsdl.py", line 97, in load
root_definitions = Definition(self, document, self.location)
File "/usr/local/lib/python3.7/site-packages/zeep/wsdl/wsdl.py", line 193, in __init__
self._load(doc)
File "/usr/local/lib/python3.7/site-packages/zeep/wsdl/wsdl.py", line 198, in _load
self.parse_types(doc)
File "/usr/local/lib/python3.7/site-packages/zeep/wsdl/wsdl.py", line 330, in parse_types
self.types.add_documents(schema_nodes, self.location)
File "/usr/local/lib/python3.7/site-packages/zeep/xsd/schema.py", line 111, in add_documents
document = self.create_new_document(node, location)
File "/usr/local/lib/python3.7/site-packages/zeep/xsd/schema.py", line 195, in create_new_document
schema.load(self, node)
File "/usr/local/lib/python3.7/site-packages/zeep/xsd/schema.py", line 421, in load
visitor.visit_schema(node)
File "/usr/local/lib/python3.7/site-packages/zeep/xsd/visitor.py", line 165, in visit_schema
self.process(child, parent=node)
File "/usr/local/lib/python3.7/site-packages/zeep/xsd/visitor.py", line 93, in process
result = visit_func(self, node, parent)
File "/usr/local/lib/python3.7/site-packages/zeep/xsd/visitor.py", line 238, in visit_import
schema_node = self._retrieve_data(location, base_url=self.document._location)
File "/usr/local/lib/python3.7/site-packages/zeep/xsd/visitor.py", line 1199, in _retrieve_data
url, self.schema._transport, base_url, settings=self.schema.settings
File "/usr/local/lib/python3.7/site-packages/zeep/loader.py", line 87, in load_external
content = transport.load(url)
File "/usr/local/lib/python3.7/site-packages/zeep/transports.py", line 122, in load
content = self._load_remote_data(url)
File "/usr/local/lib/python3.7/site-packages/zeep/transports.py", line 135, in _load_remote_data
response.raise_for_status()
File "/usr/local/lib/python3.7/site-packages/requests/models.py", line 960, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://schemas.xmlsoap.org/soap/encoding/
You need to cache the WSDL by enabling transport caching:
from zeep.cache import InMemoryCache
Client(soap_url, transport=Transport(
cache=InMemoryCache(),
))
The official documentation recommends using the SqliteCache instead of the InMemoryCache:
from zeep import Client
from zeep.cache import SqliteCache
from zeep.transports import Transport
transport = Transport(cache=SqliteCache())
client = Client(
'http://www.webservicex.net/ConvertSpeed.asmx?WSDL',
transport=transport)
I have a computer that has a C and D drive, where PyCharm, Python and the source code are installed on the D drive. I'm using Pipenv with PIPENV_VENV_IN_PROJECT set to enabled so that also ends up in the D drive. Despite all this, when running with code coverage enabled, I get this error:
Destroying test database for alias 'default'...
Traceback (most recent call last):
File "D:\Development\PyCharm\PyCharm 2019.1.1\helpers\coverage_runner\run_coverage.py", line 54, in <module>
main()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 770, in main
status = CoverageScript().command_line(argv)
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 489, in command_line
return self.do_run(options, args)
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 657, in do_run
self.coverage.save()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\control.py", line 529, in save
data = self.get_data()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\control.py", line 583, in get_data
if self._collector and self._collector.flush_data():
File "D:\Business\projectx\.venv\lib\site-packages\coverage\collector.py", line 425, in flush_data
self.covdata.add_lines(abs_file_dict(self.data))
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 236, in add_lines
self._choose_lines_or_arcs(lines=True)
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 279, in _choose_lines_or_arcs
with self._connect() as con:
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 160, in _connect
self._create_db()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 117, in _create_db
with self._db:
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 525, in __enter__
self.connect()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 510, in connect
filename = os.path.relpath(self.filename)
File "D:\Business\projectx\.venv\lib\ntpath.py", line 562, in relpath
path_drive, start_drive))
ValueError: path is on mount 'C:', start on mount 'D:'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Development\PyCharm\PyCharm 2019.1.1\helpers\coverage_runner\run_coverage.py", line 58, in <module>
main(["xml", "-o", coverage_file + ".xml", "--ignore-errors"])
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 770, in main
status = CoverageScript().command_line(argv)
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 511, in command_line
self.coverage.load()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\control.py", line 336, in load
self._data.read()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 409, in read
with self._connect(): # TODO: doesn't look right
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 160, in _connect
self._create_db()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 117, in _create_db
with self._db:
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 525, in __enter__
self.connect()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 510, in connect
filename = os.path.relpath(self.filename)
File "D:\Business\projectx\.venv\lib\ntpath.py", line 562, in relpath
path_drive, start_drive))
ValueError: path is on mount 'C:', start on mount 'D:'
Any ideas why?
I have two python scripts, currentdataupload.py and productioncount.py on my Ubuntu server. currentdataupload.py uploads dummy current data and productioncount.py accesses that data to do some calculations. The code runs perfectly in other scenarios without disrupting the MongoDB, but when I run both of these codes simultaneously using nohup after a few seconds it exits the mongoDB service and is no longer able to upload or retrieve data. I would like to know if I am doing the entire process the right way or I am messing up somewhere big time.
Error log
Traceback (most recent call last):
File "/python_codes/currentdataupload.py", line 38, in <module>
result = posts.insert_one(post_data)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 630, in insert_one
bypass_doc_val=bypass_document_validation),
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 535, in _insert
check_keys, manipulate, write_concern, op_id, bypass_doc_val)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 516, in _insert_one
check_keys=check_keys)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 244, in command
self._raise_connection_failure(error)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 372, in _raise_connection_failure
raise error
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 239, in command
read_concern)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 96, in command
response = receive_message(sock, 1, request_id)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 123, in receive_message
header = _receive_data_on_socket(sock, 16)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 161, in _receive_data_on_socket
raise AutoReconnect("connection closed")
pymongo.errors.AutoReconnect: connection closed
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 543, in connect
sock = _configured_socket(self.address, self.opts)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 452, in _configured_socket
sock = _create_connection(address, options)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 436, in _create_connection
raise err
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 429, in _create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/python_codes/productioncount.py", line 267, in <module>
result = col2.find_one({"date": date, "machine": machine})
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 1014, in find_one
for result in cursor.limit(-1):
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 1090, in next
if len(self.__data) or self._refresh():
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 1012, in _refresh
self.__read_concern))
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 850, in __send_message
**kwargs)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/mongo_client.py", line 844, in _send_message_with_response
exhaust)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/mongo_client.py", line 855, in _reset_on_error
return func(*args, **kwargs)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/server.py", line 99, in send_message_with_response
with self.get_socket(all_credentials, exhaust) as sock_info:
File "/root/anaconda3/lib/python3.5/contextlib.py", line 59, in __enter__
return next(self.gen)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/server.py", line 163, in get_socket
with self.pool.get_socket(all_credentials, checkout) as sock_info:
File "/root/anaconda3/lib/python3.5/contextlib.py", line 59, in __enter__
return next(self.gen)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 582, in get_socket
sock_info = self._get_socket_no_auth()
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 628, in _get_socket_no_auth
sock_info = self._check(sock_info)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 682, in _check
return self.connect()
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 555, in connect
_raise_connection_failure(self.address, error)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 65, in _raise_connection_failure
raise AutoReconnect(msg)
pymongo.errors.AutoReconnect: localhost:27017: [Errno 111] Connection refused
MongoDB shouldn't crash. Please file a bug report, including the Python script you run and a complete MongoDB logfile, at jira.mongodb.org.
I am using boto for uploading and downloading data from S3 bucket. I first check for key presence and then call download to file api of boto as follows.
#retry_decorator
def _get_data_to_file(self, src_key_object, file_name):
try:
s_key.get_contents_to_filename(file_name)
except Exception, fault:
logger.traceback(fault)
raise
retry_decorator is decorator functions which helps to retry the operation if call fails. I observed following error in my traceback.
[2016-02-02 10:23:09,427] [ERROR] Error <class 'socket.error'>:[Errno 104] Connection reset by peer. Traceback -Traceback (most recent call last):
File "roboClientLib/boto/awsDRLib.py", line 1173, in _get_data_to_file
File "boto/s3/key.py", line 1712, in get_contents_to_filename
File "boto/s3/key.py", line 1650, in get_contents_to_file
File "boto/s3/key.py", line 1482, in get_file
File "boto/s3/key.py", line 1535, in _get_file_internal
File "boto/s3/key.py", line 386, in next
File "boto/connection.py", line 413, in read
File "httplib.py", line 542, in read
File "socket.py", line 377, in read
File "ssl.py", line 215, in recv
File "ssl.py", line 136, in read
error: [Errno 104] Connection reset by peer
[2016-02-02 10:23:39,459] [ERROR] Error <class 'httplib.IncompleteRead'>:IncompleteRead(0 bytes read, 112795648 more expected). Traceback -Traceback (most recent call last):
File "roboClientLib/boto/awsDRLib.py", line 1173, in _get_data_to_file
File "boto/s3/key.py", line 1712, in get_contents_to_filename
File "boto/s3/key.py", line 1650, in get_contents_to_file
File "boto/s3/key.py", line 1482, in get_file
File "boto/s3/key.py", line 1535, in _get_file_internal
File "boto/s3/key.py", line 388, in next
File "boto/s3/key.py", line 370, in close
File "boto/connection.py", line 410, in read
File "httplib.py", line 529, in read
File "httplib.py", line 621, in _safe_read
IncompleteRead: IncompleteRead(0 bytes read, 112795648 more expected)
and similar error continue for further retry. What i observed is first traceback show socket error while subsequent traceback of same call show something else..
Can anybody help to understand why i am able to execute the call after retry..?
i try to access web using python suds,the code is as follows.
#!/usr/bin/python
from suds.client import Client
url = 'http://192.168.80.1xx:8079/rpc/soap/jirasoapservice-v2?wsdl'
client = Client(url)
print client
on machine A with below character:ubuntu systemip adds is 192.168.1.xxpython version is 2.6.5can connect internet.have installed easy_install 0.6.10 and suds 0.3.7.another machine B:ubuntu systemip adds is 192.168.80.xxpython version is 2.6.5is offline.have installed setuptools-0.6c11-py2.6.egg and suds 0.3.7.on machine A the code above is ok while on machine B,there is some errors:
Traceback (most recent call last):
File "soaptest.py", line 7, in <module>
client = Client(url)
File "build/bdist.linux-x86_64/egg/suds/client.py", line 109, in __init__
File "build/bdist.linux-x86_64/egg/suds/wsdl.py", line 194, in __init__
File "build/bdist.linux-x86_64/egg/suds/wsdl.py", line 255, in build_schema
File "build/bdist.linux-x86_64/egg/suds/xsd/schema.py", line 90, in load
File "build/bdist.linux-x86_64/egg/suds/xsd/schema.py", line 280, in open_imports
File "build/bdist.linux-x86_64/egg/suds/xsd/schema.py", line 280, in open_imports
File "build/bdist.linux-x86_64/egg/suds/xsd/schema.py", line 280, in open_imports
File "build/bdist.linux-x86_64/egg/suds/xsd/schema.py", line 280, in open_imports
File "build/bdist.linux-x86_64/egg/suds/xsd/schema.py", line 277, in open_imports
File "build/bdist.linux-x86_64/egg/suds/xsd/sxbasic.py", line 608, in open
File "build/bdist.linux-x86_64/egg/suds/xsd/sxbasic.py", line 626, in download
File "build/bdist.linux-x86_64/egg/suds/sax/parser.py", line 134, in parse
File "/usr/lib/python2.6/xml/sax/expatreader.py", line 107, in parse
xmlreader.IncrementalParser.parse(self, source)
File "/usr/lib/python2.6/xml/sax/xmlreader.py", line 123, in parse
self.feed(buffer)
File "/usr/lib/python2.6/xml/sax/expatreader.py", line 211, in feed
self._err_handler.fatalError(exc)
File "/usr/lib/python2.6/xml/sax/handler.py", line 38, in fatalError
raise exception
xml.sax._exceptions.SAXParseException: /tmp/suds/suds-2582214468910359336.http:1:156: mismatched tag