Develop Reference

OSError after SFTP successfully started with pysftp

I am using put from pysftp to upload some files to a SFTP server.
I have a Python list filesToUpload_bordet and I am uploading each file x using a for loop within a context manager, as shown below:
# Enter the sftp server and perform the operations
with pysftp.Connection(host=hostname,
username=username,
password=password) as sftp:
# Create and cd to the directory where the files will be uploaded
try:
sftp.mkdir(f'{lastRun}')
except OSError:
append(line=f"{timeStamp} run {lastRun}: {panel} WARNING: sftp directory /ngs/{lastRun} already exists, so not creating\n",logFile=logFile)
with sftp.cd(f'/ngs/{lastRun}'):
for x in filesToUpload_bordet:
# put the vcf files
sftp.put(x)
I know that the snipped above works because the upload successfully started. Though, after sometime I am getting the following error message on the Python console:
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
/nexus/databases/ngsRunStats_FK/postPipeline/scripts/newStrategy/STEPS_postPipeline_prod.py in <module>
----> 1 sftpUpload(panel)
/nexus/databases/ngsRunStats_FK/postPipeline/scripts/newStrategy/STEPS_postPipeline_prod.py in sftpUpload(panel)
1212 for x in filesToUpload_bordet:
1213 # put the vcf files
---> 1214 sftp.put(x)
1215 sftp.close() # this is probably not necessary as I am working on a context manager
1216 # this message should be sent independently from the coverage check email
~/miniconda3/lib/python3.7/site-packages/pysftp/__init__.py in put(self, localpath, remotepath, callback, confirm, preserve_mtime)
362
363 sftpattrs = self._sftp.put(localpath, remotepath, callback=callback,
--> 364 confirm=confirm)
365 if preserve_mtime:
366 self._sftp.utime(remotepath, times)
~/miniconda3/lib/python3.7/site-packages/paramiko/sftp_client.py in put(self, localpath, remotepath, callback, confirm)
757 file_size = os.stat(localpath).st_size
758 with open(localpath, "rb") as fl:
--> 759 return self.putfo(fl, remotepath, file_size, callback, confirm)
760
761 def getfo(self, remotepath, fl, callback=None):
~/miniconda3/lib/python3.7/site-packages/paramiko/sftp_client.py in putfo(self, fl, remotepath, file_size, callback, confirm)
715 fr.set_pipelined(True)
716 size = self._transfer_with_callback(
--> 717 reader=fl, writer=fr, file_size=file_size, callback=callback
718 )
719 if confirm:
~/miniconda3/lib/python3.7/site-packages/paramiko/sftp_client.py in _transfer_with_callback(self, reader, writer, file_size, callback)
677 while True:
678 data = reader.read(32768)
--> 679 writer.write(data)
680 size += len(data)
681 if len(data) == 0:
~/miniconda3/lib/python3.7/site-packages/paramiko/file.py in write(self, data)
403 raise IOError("File not open for writing")
404 if not (self._flags & self.FLAG_BUFFERED):
--> 405 self._write_all(data)
406 return
407 self._wbuffer.write(data)
~/miniconda3/lib/python3.7/site-packages/paramiko/file.py in _write_all(self, data)
520 # a socket).
521 while len(data) > 0:
--> 522 count = self._write(data)
523 data = data[count:]
524 if self._flags & self.FLAG_APPEND:
~/miniconda3/lib/python3.7/site-packages/paramiko/sftp_file.py in _write(self, data)
206 while len(self._reqs):
207 req = self._reqs.popleft()
--> 208 t, msg = self.sftp._read_response(req)
209 if t != CMD_STATUS:
210 raise SFTPError("Expected status")
~/miniconda3/lib/python3.7/site-packages/paramiko/sftp_client.py in _read_response(self, waitfor)
863 # synchronous
864 if t == CMD_STATUS:
--> 865 self._convert_status(msg)
866 return t, msg
867
~/miniconda3/lib/python3.7/site-packages/paramiko/sftp_client.py in _convert_status(self, msg)
896 raise IOError(errno.EACCES, text)
897 else:
--> 898 raise IOError(text)
899
900 def _adjust_cwd(self, path):
OSError: Failure
Can this be anything other than a time-out issue? I see that my first file was successfully uploaded at 09:08am and I got the error message at 11:49am.

The "Failure" is an error message for SFTP error code 4, returned by the OpenSSH SFTP server for various problems, for which there's no more specific code in the SFTP protocol version 3. While the server should at least return a specific plain-text error message, it fails to do so.
Common reasons you may get the generic "Failure" error message, while uploading are:
Uploading a file to a full filesystem (HDD).
Exceeding a user disk quota.
For details, see SFTP Status/Error Code 4 (Failure).

SecurityError: Failed to establish secure connection to 'EOF occurred in violation of protocol (_ssl.c:841)'

I am attempting to connect to Neo4j but I keep getting this error. I tried
from neo4j.v1 import GraphDatabase
driver = GraphDatabase.driver(uri="bolt://localhost:7687", auth=("neo4j", "12345"))
but I get this error when I try to connect
SecurityError: Failed to establish secure connection to 'EOF occurred in violation of protocol (_ssl.c:841)'
I can connect to the browser when I type http://localhost:7474/browser/
Here is the full error log:
--------------------------------------------------------------------------- SSLEOFError Traceback (most recent call
last)
~\AppData\Roaming\Python\Python36\site-packages\neobolt\direct.py in
_secure(s, host, ssl_context, **config)
853 try:
--> 854 s = ssl_context.wrap_socket(s, server_hostname=host if HAS_SNI and host else None)
855 except SSLError as cause:
c:\program files\python36\lib\ssl.py in wrap_socket(self, sock,
server_side, do_handshake_on_connect, suppress_ragged_eofs,
server_hostname, session)
406 server_hostname=server_hostname,
--> 407 _context=self, _session=session)
408
c:\program files\python36\lib\ssl.py in init(self, sock, keyfile,
certfile, server_side, cert_reqs, ssl_version, ca_certs,
do_handshake_on_connect, family, type, proto, fileno,
suppress_ragged_eofs, npn_protocols, ciphers, server_hostname,
_context, _session)
813 raise ValueError("do_handshake_on_connect should not be specified for
non-blocking sockets")
--> 814 self.do_handshake()
815
c:\program files\python36\lib\ssl.py in do_handshake(self, block)
1067 self.settimeout(None)
-> 1068 self._sslobj.do_handshake() 1069 finally:
c:\program files\python36\lib\ssl.py in do_handshake(self)
688 """Start the SSL/TLS handshake."""
--> 689 self._sslobj.do_handshake()
690 if self.context.check_hostname:
SSLEOFError: EOF occurred in violation of protocol (_ssl.c:841)
The above exception was the direct cause of the following exception:
SecurityError Traceback (most recent call
last) in
1
----> 2 driver = GraphDatabase.driver(uri="bolt://localhost:7687", auth=("neo4j", "12345"))
~\AppData\Roaming\Python\Python36\site-packages\neo4j__init__.py in
driver(cls, uri, **config)
118 :class:.Driver subclass instance directly.
119 """
--> 120 return Driver(uri, **config)
121
122
~\AppData\Roaming\Python\Python36\site-packages\neo4j__init__.py in
new(cls, uri, **config)
159 for subclass in Driver.subclasses():
160 if parsed_scheme in subclass.uri_schemes:
--> 161 return subclass(uri, **config)
162 raise ValueError("URI scheme %r not supported" % parsed.scheme)
163
~\AppData\Roaming\Python\Python36\site-packages\neo4j__init__.py in
new(cls, uri, **config)
233
234 pool = ConnectionPool(connector, instance.address, **config)
--> 235 pool.release(pool.acquire())
236 instance._pool = pool
237 instance._max_retry_time = config.get("max_retry_time", default_config["max_retry_time"])
~\AppData\Roaming\Python\Python36\site-packages\neobolt\direct.py in
acquire(self, access_mode)
713
714 def acquire(self, access_mode=None):
--> 715 return self.acquire_direct(self.address)
716
717
~\AppData\Roaming\Python\Python36\site-packages\neobolt\direct.py in
acquire_direct(self, address)
606 if can_create_new_connection:
607 try:
--> 608 connection = self.connector(address, error_handler=self.connection_error_handler)
609 except ServiceUnavailable:
610 self.remove(address)
~\AppData\Roaming\Python\Python36\site-packages\neo4j__init__.py in
connector(address, **kwargs)
230
231 def connector(address, **kwargs):
--> 232 return connect(address, **dict(config, **kwargs))
233
234 pool = ConnectionPool(connector, instance.address, **config)
~\AppData\Roaming\Python\Python36\site-packages\neobolt\direct.py in
connect(address, **config)
970 raise ServiceUnavailable("Failed to resolve addresses for %s" % address)
971 else:
--> 972 raise last_error
~\AppData\Roaming\Python\Python36\site-packages\neobolt\direct.py in
connect(address, **config)
961 host = address[0]
962 s = _connect(resolved_address, **config)
--> 963 s, der_encoded_server_certificate = _secure(s, host, security_plan.ssl_context, **config)
964 connection = _handshake(s, address, der_encoded_server_certificate, **config)
965 except Exception as error:
~\AppData\Roaming\Python\Python36\site-packages\neobolt\direct.py in
_secure(s, host, ssl_context, **config)
857 error = SecurityError("Failed to establish secure connection to {!r}".format(cause.args[1]))
858 error.cause = cause
--> 859 raise error
860 else:
861 # Check that the server provides a certificate
SecurityError: Failed to establish secure connection to 'EOF occurred
in violation of protocol (_ssl.c:841)'

I found the solution for people who might have the same issue. You need to add encrypted=False.
Instead of
from neo4j.v1 import GraphDatabase
driver = GraphDatabase.driver(uri="bolt://localhost:7687", auth=("neo4j", "12345"))
it should be:
driver = GraphDatabase.driver(uri="bolt://localhost:7687", auth=("neo4j", "12345"), encrypted=False)
Hope this will help someone

I had the same problem with the Object Graph Mapper Neomodel (connecting to neo4j v4). Adding the second line solved it:
config.DATABASE_URL = 'bolt://neo4j:password123#localhost:7687'
config.ENCRYPTED_CONNECTION = False

had the same issue, answer from Sam did not resolve it, however. Using
from neo4j import GraphDatabase
uri = "bolt://localhost:7687"
driver = GraphDatabase.driver( encrypted=False, uri, auth=('neo4j', 'asdf'))
I get the error SyntaxError: positional argument follows keyword argument
typing the uridirectly into the command like this: driver = GraphDatabase.driver( encrypted=False, uri = "bolt://localhost:7687", auth=('neo4j', 'asdf')) resulted in another error (SecurityError: Failed to establish secure connection to '[SSL: KRB5_S_TKT_NYV] unexpected eof while reading (_ssl.c:1076)')
When following a post in the neo4j community however, they switch positions of the arguments to this:
from neo4j import GraphDatabase
uri = "bolt://localhost:7687"
driver = GraphDatabase.driver( uri, auth=('neo4j', 'asdf')encrypted=False)
it works like a charm.

Unable to execute commands in remote mongo using pymongo

I am able to connect to my remote db (after authentication, of course) to a database. But I am not able to execute any commands or even list collections.
mongo_url = "blah.com:12345"
db_name = "db_name"
db_user_name = "user"
db_password = "password"
mongo_uri = "mongodb://" + db_user_name + ":" + db_password + "#" + mongo_url + "/" + db_name
connection = pymongo.MongoClient(mongo_uri)
db = connection[db_name]
print db.authenticate(db_user_name, db_password) // Returns True
However I am not able to use commands like :
db.collection_names() or any command using db.command()
I get this error stack (sayng Authentcation failed):
---------------------------------------------------------------------------
OperationFailure Traceback (most recent call last)
<ipython-input-13-1840c0979539> in <module>()
----> 1 db.collection_names()
D:\Continuum\Anaconda2\lib\site-packages\pymongo\database.pyc in collection_names(self, include_system_collections)
515 """
516 with self.__client._socket_for_reads(
--> 517 ReadPreference.PRIMARY) as (sock_info, slave_okay):
518
519 wire_version = sock_info.max_wire_version
D:\Continuum\Anaconda2\lib\contextlib.pyc in __enter__(self)
15 def __enter__(self):
16 try:
---> 17 return self.gen.next()
18 except StopIteration:
19 raise RuntimeError("generator didn't yield")
D:\Continuum\Anaconda2\lib\site-packages\pymongo\mongo_client.pyc in _socket_for_reads(self, read_preference)
796 topology = self._get_topology()
797 single = topology.description.topology_type == TOPOLOGY_TYPE.Single
--> 798 with self._get_socket(read_preference) as sock_info:
799 slave_ok = (single and not sock_info.is_mongos) or (
800 preference != ReadPreference.PRIMARY)
D:\Continuum\Anaconda2\lib\contextlib.pyc in __enter__(self)
15 def __enter__(self):
16 try:
---> 17 return self.gen.next()
18 except StopIteration:
19 raise RuntimeError("generator didn't yield")
D:\Continuum\Anaconda2\lib\site-packages\pymongo\mongo_client.pyc in _get_socket(self, selector)
762 server = self._get_topology().select_server(selector)
763 try:
--> 764 with server.get_socket(self.__all_credentials) as sock_info:
765 yield sock_info
766 except NetworkTimeout:
D:\Continuum\Anaconda2\lib\contextlib.pyc in __enter__(self)
15 def __enter__(self):
16 try:
---> 17 return self.gen.next()
18 except StopIteration:
19 raise RuntimeError("generator didn't yield")
D:\Continuum\Anaconda2\lib\site-packages\pymongo\server.pyc in get_socket(self, all_credentials, checkout)
161 #contextlib.contextmanager
162 def get_socket(self, all_credentials, checkout=False):
--> 163 with self.pool.get_socket(all_credentials, checkout) as sock_info:
164 yield sock_info
165
D:\Continuum\Anaconda2\lib\contextlib.pyc in __enter__(self)
15 def __enter__(self):
16 try:
---> 17 return self.gen.next()
18 except StopIteration:
19 raise RuntimeError("generator didn't yield")
D:\Continuum\Anaconda2\lib\site-packages\pymongo\pool.pyc in get_socket(self, all_credentials, checkout)
582 sock_info = self._get_socket_no_auth()
583 try:
--> 584 sock_info.check_auth(all_credentials)
585 yield sock_info
586 except:
D:\Continuum\Anaconda2\lib\site-packages\pymongo\pool.pyc in check_auth(self, all_credentials)
330
331 for credentials in cached - authset:
--> 332 auth.authenticate(credentials, self)
333 self.authset.add(credentials)
334
D:\Continuum\Anaconda2\lib\site-packages\pymongo\auth.pyc in authenticate(credentials, sock_info)
462 mechanism = credentials.mechanism
463 auth_func = _AUTH_MAP.get(mechanism)
--> 464 auth_func(credentials, sock_info)
465
466
D:\Continuum\Anaconda2\lib\site-packages\pymongo\auth.pyc in _authenticate_default(credentials, sock_info)
442 def _authenticate_default(credentials, sock_info):
443 if sock_info.max_wire_version >= 3:
--> 444 return _authenticate_scram_sha1(credentials, sock_info)
445 else:
446 return _authenticate_mongo_cr(credentials, sock_info)
D:\Continuum\Anaconda2\lib\site-packages\pymongo\auth.pyc in _authenticate_scram_sha1(credentials, sock_info)
226 ('conversationId', res['conversationId']),
227 ('payload', Binary(client_final))])
--> 228 res = sock_info.command(source, cmd)
229
230 parsed = _parse_scram_response(res['payload'])
D:\Continuum\Anaconda2\lib\site-packages\pymongo\pool.pyc in command(self, dbname, spec, slave_ok, read_preference, codec_options, check, allowable_errors, check_keys, read_concern)
237 check, allowable_errors, self.address,
238 check_keys, self.listeners, self.max_bson_size,
--> 239 read_concern)
240 except OperationFailure:
241 raise
D:\Continuum\Anaconda2\lib\site-packages\pymongo\network.pyc in command(sock, dbname, spec, slave_ok, is_mongos, read_preference, codec_options, check, allowable_errors, address, check_keys, listeners, max_bson_size, read_concern)
100 response_doc = unpacked['data'][0]
101 if check:
--> 102 helpers._check_command_response(response_doc, None, allowable_errors)
103 except Exception as exc:
104 if publish:
D:\Continuum\Anaconda2\lib\site-packages\pymongo\helpers.pyc in _check_command_response(response, msg, allowable_errors)
203
204 msg = msg or "%s"
--> 205 raise OperationFailure(msg % errmsg, code, response)
206
207
OperationFailure: Authentication failed.
But I am able to do these operations from my mongo shell.

ImportError, No module named rdbms_googleapi on GAE python SDK when using the remote shell

We're running a Django application on AppEngine (python).
When making a query using the remote shell so the sql driver points to CloudSQL, GAE is throwing an exception:
ImportError: No module named rdbms_googleapi.
This problem affects deployed apps when using the remote shell and attempting to run SQL queries.
We tested SDKs versions 1.9.36 and 1.9.37 under Mac OS X 10.11.4
You can reproduce this bugs with the following steps:
Have a GAE application with Django.
Use the remote shell to connect to a running instance of the application.
The database engine in use is google.appengine.ext.django.backends.rdbms.
Query the database.
The expected output is the query result, however we get:
ImportError: No module named rdbms_googleapi
This is the full traceback of the exception:
In [2]: MyModel.objects.get(id=1)
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
<ipython-input-2-d1e136dd4ed5> in <module>()
----> 1 MyModel.objects.get(id=1)
/usr/local/google_appengine/lib/django-1.5/django/db/models/manager.py in get(self, *args, **kwargs)
141
142 def get(self, *args, **kwargs):
--> 143 return self.get_query_set().get(*args, **kwargs)
144
145 def get_or_create(self, **kwargs):
/usr/local/google_appengine/lib/django-1.5/django/db/models/query.py in get(self, *args, **kwargs)
396 if self.query.can_filter():
397 clone = clone.order_by()
--> 398 num = len(clone)
399 if num == 1:
400 return clone._result_cache[0]
/usr/local/google_appengine/lib/django-1.5/django/db/models/query.py in __len__(self)
104 self._result_cache = list(self._iter)
105 else:
--> 106 self._result_cache = list(self.iterator())
107 elif self._iter:
108 self._result_cache.extend(self._iter)
/usr/local/google_appengine/lib/django-1.5/django/db/models/query.py in iterator(self)
315 klass_info = get_klass_info(model, max_depth=max_depth,
316 requested=requested, only_load=only_load)
--> 317 for row in compiler.results_iter():
318 if fill_cache:
319 obj, _ = get_cached_row(row, index_start, db, klass_info,
/usr/local/google_appengine/lib/django-1.5/django/db/models/sql/compiler.py in results_iter(self)
773 if self.query.select_for_update and transaction.is_managed(self.using):
774 transaction.set_dirty(self.using)
--> 775 for rows in self.execute_sql(MULTI):
776 for row in rows:
777 if has_aggregate_select:
/usr/local/google_appengine/lib/django-1.5/django/db/models/sql/compiler.py in execute_sql(self, result_type)
843 return
844
--> 845 cursor = self.connection.cursor()
846 cursor.execute(sql, params)
847
/usr/local/google_appengine/lib/django-1.5/django/db/backends/__init__.py in cursor(self)
324 cursor = self.make_debug_cursor(self._cursor())
325 else:
--> 326 cursor = util.CursorWrapper(self._cursor(), self)
327 return cursor
328
/usr/local/google_appengine/google/storage/speckle/python/django/backend/base.pyc in _cursor(self)
274 "You must specify a '%s' for database '%s'" %
275 (settings_key, self.alias))
--> 276 self.connection = Connect(**kwargs)
277 encoders = {safestring.SafeUnicode: self.connection.encoders[unicode],
278 safestring.SafeString: self.connection.encoders[str]}
/usr/local/google_appengine/google/storage/speckle/python/django/backend/base.pyc in Connect(driver_name, oauth2_refresh_token, **kwargs)
165 found in storage and no oauth2_refresh_token was given.
166 """
--> 167 driver = _GetDriver(driver_name)
168 server_software = os.getenv('SERVER_SOFTWARE', '').split('/')[0]
169 if (server_software in (DEV_SERVER_SOFTWARE, PROD_SERVER_SOFTWARE) and
/usr/local/google_appengine/google/storage/speckle/python/django/backend/base.pyc in _GetDriver(driver_name)
142 else:
143 driver_name = base_pkg_path + 'rdbms_googleapi'
--> 144 __import__(driver_name)
145 return sys.modules[driver_name]
146
ImportError: No module named rdbms_googleapi
Has anyone experienced something similar? We believe the SDK is not installing all the required files.
It's worth mentioning that previous versions of the AppEngine python SDK worked correctly.
After messing around with the SDK we noticed that there is a missing file compared to previous versions.
We copied the file at https://chromium.googlesource.com/external/googleappengine/python/+/master/google/storage/speckle/python/api/rdbms_googleapi.py
into our /usr/local/google_appengine/google/storage/speckle/python/api directory and we resolved the issue.
This indicates that the GAE SDK installer is missing that file.

RabbitMQ gives a "access refused, login refused for user" error when attempting to follow the celery tutorial

I'm attempting to follow the celery tutorial, but I run into a problem when I run python manage.py celeryd: my RabbitMQ server (installed on a virtual machine on my dev box) won't let my user login.
I get the following on my Django management console:
[ERROR/MainProcess] AMQP Listener: Connection Error: Socket closed. Trying again in 2 seconds...
and this shows up in my rabbit.log file on my RabbitMQ server:
exception on TCP connection <0.5814.0> from $DJANGO_BOX_IP
{channel0_error,starting,{amqp,access_refused,"login refused for user '$CONFIGURED_USER'",'connection.start_ok'}}
I've double-checked my user, permissions, and vhost info, and they all seem to match up. Any help troubleshooting is greatly appreciated.
UPDATE: Following the advice of #asksol I get the following traceback:
$MY_VIRTUAL_ENV/lib/python2.6/site-packages/carrot/connection.pyc in connection(self)
118 return
119 if not self._connection:
--> 120 self._connection = self._establish_connection()
121 self._closed = False
122 return self._connection
$MY_VIRTUAL_ENV/lib/python2.6/site-packages/carrot/connection.pyc in _establish_connection(self)
131
132 def _establish_connection(self):
--> 133 return self.create_backend().establish_connection()
134
135 def get_backend_cls(self):
$MY_VIRTUAL_ENV/lib/python2.6/site-packages/carrot/backends/pyamqplib.pyc in establish_connection(self)
110 insist=conninfo.insist,
111 ssl=conninfo.ssl,
--> 112 connect_timeout=conninfo.connect_timeout)
113
114 def close_connection(self, connection):
$MY_VIRTUAL_ENV/lib/python2.6/site-packages/amqplib/client_0_8/connection.pyc in __init__(self, host, userid, password, login_method, login_response, virtual_host, locale, client_properties, ssl, insist, connect_timeout, **kwargs)
138 self.wait(allowed_methods=[
139 (10, 20), # secure
--> 140 (10, 30), # tune
141 ])
142
$MY_VIRTUAL_ENV/lib/python2.6/site-packages/amqplib/client_0_8/abstract_channel.pyc in wait(self, allowed_methods)
88 method_sig, args, content = self.connection._wait_method(
---> 89 self.channel_id, allowed_methods)
90
91 if content \
$MY_VIRTUAL_ENV/lib/python2.6/site-packages/amqplib/client_0_8/connection.pyc in _wait_method(self, channel_id, allowed_methods)
196 while True:
197 channel, method_sig, args, content = \
--> 198 self.method_reader.read_method()
199
200 if (channel == channel_id) \
$MY_VIRTUAL_ENV/lib/python2.6/site-packages/amqplib/client_0_8/method_framing.pyc in read_method(self)
213 m = self.queue.get()
214 if isinstance(m, Exception):
--> 215 raise m
216 return m
217
IOError: Socket closed

Are you running django?
If so, then try this:
>>> from carrot.connection import DjangoBrokerConnection
>>> c = DjangoBrokerConnection()
>>> c.connection
Does it give the same thing?
Are you sure you're connecting to the right hostname, and that the username and password has access to the virtual host?
UPDATE:
>>> from carrot.connection import DjangoBrokerConnection
>>> c = DjangoBrokerConnection()
>>> for n in ("host", "userid", "password", "virtual_host", "ssl"):
... print("%s -> %s" % (n, repr(getattr(c, n, None))))
UPDATE: You have to do the above before running c.connection, as the connection is established lazily in carrot.