Develop Reference

httpx.ReadTimeout on Range filter on Supabase-py

I have 375000 items in my table.
I am doing a loop to obtain all id of all items, with API limit set to 20000 items per api call.
After 200000 I always start to get httpx.ReadTimeout: The read operation timed out sometime it may reach 240000 but never go ahead.
I have tried to have different wait time after each loop.
I have tried to change api limit to 10000 as well as increase it to 30000 or 50000 make less calls but in all cases it get's stuck at around 150000 or 200000.
existing_search_result = supabase.table('vehicles').select('ref_id', count='exact').order('id', desc=False).execute()
existing_items = []
range_step = len(existing_search_result.data)
total_existing_items = existing_search_result.count
print(total_existing_items)
while len(existing_items) < total_existing_items:
try:
existing_items += (
supabase.table(
'vehicles'
).select('ref_id')
.order('id', desc=False)
.range(range_start, range_start + range_step)
.execute()
).data
range_start += range_step
except Exception as e:
logging.exception(e)
print(range_start, len(existing_items))
time.sleep(0.30)
Error log:
2022-10-23 21:04:14,168:ERROR - The read operation timed out
Traceback (most recent call last):
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_exceptions.py", line 8, in map_exceptions
yield
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/backends/sync.py", line 26, in read
return self._sock.recv(max_bytes)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/ssl.py", line 1226, in recv
return self.read(buflen)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/ssl.py", line 1101, in read
return self._sslobj.read(len)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
yield
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 204, in handle_request
resp = self._pool.handle_request(req)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py", line 253, in handle_request
raise exc
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py", line 237, in handle_request
response = connection.handle_request(request)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/connection.py", line 90, in handle_request
return self._connection.handle_request(request)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 102, in handle_request
raise exc
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 81, in handle_request
) = self._receive_response_headers(**kwargs)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 143, in _receive_response_headers
event = self._receive_event(timeout=timeout)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 172, in _receive_event
data = self._network_stream.read(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/backends/sync.py", line 26, in read
return self._sock.recv(max_bytes)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/contextlib.py", line 135, in __exit__
self.gen.throw(type, value, traceback)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpcore/_exceptions.py", line 12, in map_exceptions
raise to_exc(exc)
httpcore.ReadTimeout: The read operation timed out
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/ak4zh/updater/main.py", line 278, in job
supabase.table(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/postgrest/_sync/request_builder.py", line 53, in execute
r = self.session.request(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 802, in request
return self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 889, in send
response = self._send_handling_auth(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 917, in _send_handling_auth
response = self._send_handling_redirects(
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 954, in _send_handling_redirects
response = self._send_single_request(request)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_client.py", line 990, in _send_single_request
response = transport.handle_request(request)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 204, in handle_request
resp = self._pool.handle_request(req)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/contextlib.py", line 135, in __exit__
self.gen.throw(type, value, traceback)
File "/Users/ak4zh/updater/venv/lib/python3.9/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ReadTimeout: The read operation timed out

I had the same problem, so I create a function to get 1000 item and merge it!
this is my code:
def count_data(tb_name: str, field_name: str):
return supabase.table(tb_name).select(field_name, count='exact').execute().count
def get_field_data(tb_name: str, src_field: str, len_record: int, id_field: str = 'id'):
if len_record <= 1000:
field_data = supabase.table(tb_name).select(id_field, src_field).order(
column=id_field).execute().data
else:
rnk = int(len_record / 1000)
field_data = []
for i in range(rnk):
min_rg = (i * 1000) + 1
max_rg = (i + 1) * 1000
field_data = field_data + supabase.table(tb_name).select(id_field, src_field).order(
column=id_field).range(min_rg - 1, max_rg).execute().data
field_data = field_data + supabase.table(tb_name).select(id_field, src_field).order(
column=id_field).range(max_rg, len_record).execute().data
return field_data

Multiprocessing BrokenPipeError

I have a following problem. I am running a parallel task. I am getting this error:
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "eclat_model.py", line 127, in do_work
function(*args, work_queue, valid_list)
File "eclat_model.py", line 115, in eclat_parallel_helper
valid_list.extend(next_vectors)
File "<string>", line 2, in extend
File "/usr/lib/python3.8/multiprocessing/managers.py", line 834, in _callmethod
conn.send((self._id, methodname, args, kwds))
File "/usr/lib/python3.8/multiprocessing/connection.py", line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "/usr/lib/python3.8/multiprocessing/connection.py", line 404, in _send_bytes
self._send(header)
File "/usr/lib/python3.8/multiprocessing/connection.py", line 368, in _send
n = write(self._handle, buf)
BrokenPipeError: [Errno 32] Broken pipe
Relevant functions in eclat_model.py look like this:
def eclat_parallel_helper(index, bit_vectors, min_support, work_queue, valid_list):
next_vectors = []
for j in range(index + 1, len(bit_vectors)):
item_vector = bit_vectors[index][0] | bit_vectors[j][0]
transaction_vector = bit_vectors[index][1] & bit_vectors[j][1]
support = get_vector_support(transaction_vector)
if support >= min_support:
next_vectors.append((item_vector, transaction_vector, support))
if len(next_vectors) > 0:
valid_list.extend(next_vectors)
for i in range(len(next_vectors)):
work_queue.put((eclat_parallel_helper, (i, next_vectors, min_support)))
def do_work(work_queue, valid_list, not_done):
# work queue entries have the form (function, args)
while not_done.value:
try:
function, args = work_queue.get_nowait()
except QueueEmptyError:
continue
function(*args, work_queue, valid_list)
work_queue.task_done()
work_queue.close()
EDIT:
Multiprocessing part of the code is as follows: bit_vectors is a list of lists, where each entry is of the form
[items, transactions, support], where items is a bit vector encoding which items appear in the itemset, vector is a bit vector encoding which transactions the itemset appears in, and support is the number of transactions in which the itemset occurs.
from multiprocessing import Process, JoinableQueue, Manager, Value, cpu_count
def eclat_parallel(bit_vectors, min_support):
not_done = Value('i', 1)
manager = Manager()
valid_list = manager.list()
work_queue = JoinableQueue()
for i in range(len(bit_vectors)):
work_queue.put((eclat_parallel_helper, (i, bit_vectors, min_support)))
processes = []
for i in range(cpu_count()):
p = Process(target=do_work, args=(work_queue, valid_list, not_done), daemon=True)
p.start()
processes.append(p)
work_queue.join()
not_done.value = 0
work_queue.close()
valid_itemset_vectors = bit_vectors
for element in valid_list:
valid_itemset_vectors.append(element)
for p in processes:
p.join()
return valid_itemset_vectors
What does this error mean, please? Am I appending too many elements into next_vectors list?

I had the same issue, in my case just added a delay (time.sleep(0.01)) to solve it.
The problem is that the individual processes are too fast on queue that causes the error.

How to stop blocking call and exit thread gracefully in python

I have a working script which invokes multiple blocking read methods in different threads and whenever it gets data in the blocking method it puts the data in queue that will be read by task.
import asyncio
import atexit
import concurrent.futures
import threading
def blocking_read(x, loop):
while True:
print(f"blocking_read {x}")
time.sleep(1)
# implementation is wait for data and put into the queue
class Recorder():
def __init__(self):
self.count = 0
def run_reader_threads(self, thread_pool, event):
loop = asyncio.get_running_loop()
threads = []
for rx in range(2):
threads.append(loop.run_in_executor(thread_pool, blocking_read, rx, loop))
return threads
def wiatforever():
while True:
time.sleep(1)
reader_futures = []
executor = concurrent.futures.ThreadPoolExecutor()
event = threading.Event()
def main():
async def doit():
recorder_app = Recorder()
global reader_futures
reader_futures = recorder_app.run_reader_threads(executor, event)
#reader_handlers = recorder_app.create_handlers()
await wiatforever()
try:
print("RUN DO IT")
asyncio.run(doit())
except KeyboardInterrupt:
# cancel all futures but geeting exception on cancelling 1st future
reader_futures[0].cancel()
pass
#--------------------------------------------------------------------------------------------
if __name__ == '__main__':
main()
The problem is whenever I want to stop the script gracefully by cancelling all futures I am getting exception that RuntimeError: Event loop is closed
OUTPUT
RUN DO IT
blocking_read 0
blocking_read 1
blocking_read 1
blocking_read 0
blocking_read 0
blocking_read 1
^CTraceback (most recent call last):
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 45, in main
asyncio.run(doit())
File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 629, in run_until_complete
self.run_forever()
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 596, in run_forever
self._run_once()
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 1890, in _run_once
handle._run()
File "/usr/local/lib/python3.9/asyncio/events.py", line 80, in _run
self._context.run(self._callback, *self._args)
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 41, in doit
await wiatforever()
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 27, in wiatforever
time.sleep(1)
KeyboardInterrupt
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 53, in <module>
main()
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 48, in main
reader_futures[0].cancel()
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 746, in call_soon
self._check_closed()
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 510, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
blocking_read 0
blocking_read 1
blocking_read 0
blocking_read 1
blocking_read 0
blocking_read 1
^CException ignored in: <module 'threading' from '/usr/local/lib/python3.9/threading.py'>
Traceback (most recent call last):
File "/usr/local/lib/python3.9/threading.py", line 1411, in _shutdown
atexit_call()
File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 31, in _python_exit
t.join()
File "/usr/local/lib/python3.9/threading.py", line 1029, in join
self._wait_for_tstate_lock()
File "/usr/local/lib/python3.9/threading.py", line 1045, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt:
so can anyone help me to find out the issue here and exit the script correctly without any exception.

Why isn't gunicorn working with my Flask app?

I have a simple Flask app set up which runs with the command flask run. I'd like to be able to run this app in a Docker container, which I'm trying to do using a gunicorn server. However, when I try to run using gunicorn I'm seeing error messages. I'm running gunicorn --worker-class eventlet -w 1 app:app which I got from their documentation
I have a simple flask-socketio app:
.
├── app.py
└── templates
└── index.html
Here's the contents of app.py:
import os
import sys
from eventlet import patcher, support
import six
select = patcher.original('select')
time = patcher.original('time')
from eventlet.hubs.hub import BaseHub, READ, WRITE, noop
if getattr(select, 'kqueue', None) is None:
raise ImportError('No kqueue implementation found in select module')
FILTERS = {READ: select.KQ_FILTER_READ,
WRITE: select.KQ_FILTER_WRITE}
class Hub(BaseHub):
MAX_EVENTS = 100
def __init__(self, clock=None):
super(Hub, self).__init__(clock)
self._events = {}
self._init_kqueue()
def _init_kqueue(self):
self.kqueue = select.kqueue()
self._pid = os.getpid()
def _reinit_kqueue(self):
self.kqueue.close()
self._init_kqueue()
kqueue = self.kqueue
events = [e for i in six.itervalues(self._events)
for e in six.itervalues(i)]
kqueue.control(events, 0, 0)
def _control(self, events, max_events, timeout):
try:
return self.kqueue.control(events, max_events, timeout)
except (OSError, IOError):
# have we forked?
if os.getpid() != self._pid:
self._reinit_kqueue()
return self.kqueue.control(events, max_events, timeout)
raise
def add(self, evtype, fileno, cb, tb, mac):
listener = super(Hub, self).add(evtype, fileno, cb, tb, mac)
events = self._events.setdefault(fileno, {})
if evtype not in events:
try:
event = select.kevent(fileno, FILTERS.get(evtype), select.KQ_EV_ADD)
self._control([event], 0, 0)
events[evtype] = event
except ValueError:
super(Hub, self).remove(listener)
raise
return listener
def _delete_events(self, events):
del_events = [
select.kevent(e.ident, e.filter, select.KQ_EV_DELETE)
for e in events
]
self._control(del_events, 0, 0)
def remove(self, listener):
super(Hub, self).remove(listener)
evtype = listener.evtype
fileno = listener.fileno
if not self.listeners[evtype].get(fileno):
event = self._events[fileno].pop(evtype, None)
if event is None:
return
try:
self._delete_events((event,))
except OSError:
pass
def remove_descriptor(self, fileno):
super(Hub, self).remove_descriptor(fileno)
try:
events = self._events.pop(fileno).values()
self._delete_events(events)
except KeyError:
pass
except OSError:
pass
def wait(self, seconds=None):
readers = self.listeners[READ]
writers = self.listeners[WRITE]
if not readers and not writers:
if seconds:
time.sleep(seconds)
return
result = self._control([], self.MAX_EVENTS, seconds)
SYSTEM_EXCEPTIONS = self.SYSTEM_EXCEPTIONS
for event in result:
fileno = event.ident
evfilt = event.filter
try:
if evfilt == FILTERS[READ]:
readers.get(fileno, noop).cb(fileno)
if evfilt == FILTERS[WRITE]:
writers.get(fileno, noop).cb(fileno)
except SYSTEM_EXCEPTIONS:
raise
except:
self.squelch_exception(fileno, sys.exc_info())
support.clear_sys_exc_info()
Here are the errors I'm seeing when running the gunicorn command:
Exception ignored in: <function _after_fork at 0x1121fc1f0>
Traceback (most recent call last):
File "/usr/local/Cellar/python#3.9/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py", line 1510, in _after_fork
thread._reset_internal_locks(True)
File "/usr/local/Cellar/python#3.9/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py", line 845, in _reset_internal_locks
self._started._at_fork_reinit()
File "/usr/local/Cellar/python#3.9/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py", line 527, in _at_fork_reinit
self._cond._at_fork_reinit()
File "/usr/local/Cellar/python#3.9/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py", line 253, in _at_fork_reinit
self._lock._at_fork_reinit()
AttributeError: 'Semaphore' object has no attribute '_at_fork_reinit'
[2022-01-31 20:57:29 +0000] [51963] [INFO] Booting worker with pid: 51963
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/hub.py", line 460, in fire_timers
timer()
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/timer.py", line 59, in __call__
cb(*args, **kw)
File "/usr/local/lib/python3.9/site-packages/eventlet/greenthread.py", line 219, in main
result = function(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/gunicorn/workers/geventlet.py", line 78, in _eventlet_serve
conn, addr = sock.accept()
File "/usr/local/lib/python3.9/site-packages/eventlet/greenio/base.py", line 228, in accept
self._trampoline(fd, read=True, timeout=self.gettimeout(), timeout_exc=_timeout_exc)
File "/usr/local/lib/python3.9/site-packages/eventlet/greenio/base.py", line 206, in _trampoline
return trampoline(fd, read=read, write=write, timeout=timeout,
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/__init__.py", line 160, in trampoline
listener = hub.add(hub.READ, fileno, current.switch, current.throw, mark_as_closed)
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/kqueue.py", line 55, in add
self._control([event], 0, 0)
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/kqueue.py", line 41, in _control
return self.kqueue.control(events, max_events, timeout)
OSError: [Errno 9] Bad file descriptor
[2022-01-31 20:57:29 +0000] [51963] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/gunicorn/arbiter.py", line 589, in spawn_worker
worker.init_process()
File "/usr/local/lib/python3.9/site-packages/gunicorn/workers/geventlet.py", line 134, in init_process
super().init_process()
File "/usr/local/lib/python3.9/site-packages/gunicorn/workers/base.py", line 142, in init_process
self.run()
File "/usr/local/lib/python3.9/site-packages/gunicorn/workers/geventlet.py", line 166, in run
eventlet.sleep(1.0)
File "/usr/local/lib/python3.9/site-packages/eventlet/greenthread.py", line 36, in sleep
hub.switch()
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/hub.py", line 297, in switch
return self.greenlet.switch()
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/hub.py", line 349, in run
self.wait(sleep_time)
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/kqueue.py", line 100, in wait
result = self._control([], self.MAX_EVENTS, seconds)
File "/usr/local/lib/python3.9/site-packages/eventlet/hubs/kqueue.py", line 41, in _control
return self.kqueue.control(events, max_events, timeout)
I'm not sure if I'm using the wrong command or if there is some other issue preventing me from using gunicorn?

Logging exception error upgrading to python 3.4 from python 2.7

My application was running well but as I upgraded it to python version 3.4 . I am getting the error shown bellow. Looking at the error message I am not exactly able to debug the problem
Error:
--- Logging error ---
Traceback (most recent call last):
File "/usr/lib64/python3.4/logging/__init__.py", line 978, in emit
msg = self.format(record)
File "/usr/lib64/python3.4/logging/__init__.py", line 828, in format
return fmt.format(record)
File "/usr/lib64/python3.4/logging/__init__.py", line 573, in format
record.exc_text = self.formatException(record.exc_info)
File "/usr/lib64/python3.4/logging/__init__.py", line 523, in formatException
traceback.print_exception(ei[0], ei[1], tb, None, sio)
File "/usr/lib64/python3.4/traceback.py", line 169, in print_exception
for line in _format_exception_iter(etype, value, tb, limit, chain):
File "/usr/lib64/python3.4/traceback.py", line 146, in _format_exception_iter
for value, tb in values:
File "/usr/lib64/python3.4/traceback.py", line 125, in _iter_chain
context = exc.__context__
AttributeError: 'NoneType' object has no attribute '__context__'
Call stack:
File "main.py", line 253, in <module>
main()
File "main.py", line 144, in main
plugininfo.plugin_object.run(cons.MAIN_CONFIG_PATH, outputpath, finallogs, plugininfo.name, args.inventory)
File "/home/sjoshi/python3.4/check-acess/plugins/plugin_etc_passwd/plugin_etc_passwd.py", line 107, in run
self.result = phelper.executeCommand(runcommand)
File "/home/sjoshi/python3.4/check-acess/lib/sshplugin/sshplugin.py", line 86, in executeCommand
logging.exception("ErrorCode:%d %s", cmdout.returncode, output)
Message: 'ErrorCode:%d %s'
Arguments: (255, b'')
No Result found or all users filtered sbx32 /etc/passwd
Permission denied (publickey).
--- Logging error ---
Traceback (most recent call last):
File "/usr/lib64/python3.4/logging/__init__.py", line 978, in emit
msg = self.format(record)
File "/usr/lib64/python3.4/logging/__init__.py", line 828, in format
return fmt.format(record)
File "/usr/lib64/python3.4/logging/__init__.py", line 573, in format
record.exc_text = self.formatException(record.exc_info)
File "/usr/lib64/python3.4/logging/__init__.py", line 523, in formatException
traceback.print_exception(ei[0], ei[1], tb, None, sio)
File "/usr/lib64/python3.4/traceback.py", line 169, in print_exception
for line in _format_exception_iter(etype, value, tb, limit, chain):
File "/usr/lib64/python3.4/traceback.py", line 146, in _format_exception_iter
for value, tb in values:
File "/usr/lib64/python3.4/traceback.py", line 125, in _iter_chain
context = exc.__context__
AttributeError: 'NoneType' object has no attribute '__context__'
Call stack:
File "main.py", line 253, in <module>
main()
File "main.py", line 144, in main
plugininfo.plugin_object.run(cons.MAIN_CONFIG_PATH, outputpath, finallogs, plugininfo.name, args.inventory)
File "/home/sjoshi/python3.4/check-acess/plugins/plugin_etc_passwd/plugin_etc_passwd.py", line 107, in run
self.result = phelper.executeCommand(runcommand)
File "/home/sjoshi/python3.4/check-acess/lib/sshplugin/sshplugin.py", line 86, in executeCommand
logging.exception("ErrorCode:%d %s", cmdout.returncode, output)
Message: 'ErrorCode:%d %s'
Arguments: (255, b'')
sshplugin.py
import os
import subprocess as subp
import logging
import lib.exceptions.errors as error
import lib.inventory.bashhostlist as hostname
class SSHPlugin:
"""
Helper class for the SSH based plugins
"""
def makeCommand(self, user, host, filepath, sudo, timeout, attempt, changetols=None):
"""
-Two types of commands with or without ssh depends on hostname
-Other depends on parameter set or unset
-ConnectionAttempts and ConnectTimeout to occur must be SSH
-last arg(changetols): This is to checkif the plugin type is authorized_keys. Because in that case we need ls not cat
"""
command=""
if changetols is None:
command = ["cat", filepath]
else:
command=["ls", filepath]
if host.lower() == 'localhost':
if sudo.lower() == 'yes':
command.insert(0, 'sudo')
else:
command.insert(0, "ssh")
if timeout and not attempt:
command.insert(1, '-o')
command.insert(
2, "ConnectTimeout={timeout}".format(timeout=timeout))
command.insert(3, user+"#"+host)
if sudo.lower() == 'yes':
command.insert(4, 'sudo')
elif timeout and attempt:
command.insert(1, '-o')
command.insert(
2, "ConnectTimeout={timeout}".format(timeout=timeout))
command.insert(3, '-o')
command.insert(
4, "ConnectionAttempts={att}".format(att=attempt))
command.insert(5, user+"#"+host)
if sudo.lower() == 'yes':
command.insert(6, 'sudo')
elif attempt and not timeout:
command.insert(1, '-o')
command.insert(
2, "ConnectionAttempts={att}".format(att=attempt))
command.insert(3, user+"#"+host)
if sudo.lower() == 'yes':
command.insert(4, 'sudo')
else:
command.insert(1, user+"#"+host)
if sudo.lower() == 'yes':
command.insert(2, 'sudo')
return command
def executeCommand(self, command, filtercommand=None):
"""
Multuple Popen used to execute the process,Command followed by filter if any
if no filter than just a single commad run else
The output of first will act as the input to the filtering command
"""
if filtercommand is None:
cmdout = subp.Popen(command, stdout=subp.PIPE)
output, err = cmdout.communicate()
if cmdout.returncode is 0:
logging.info("Result success,status code %d", cmdout.returncode)
return output
else:
logging.exception("ErrorCode:%d %s %s", cmdout.returncode, output,err)
return False
else:
cmdout = subp.Popen(command, stdout=subp.PIPE)
filtered = subp.Popen(filtercommand, stdin=cmdout.stdout, stdout=subp.PIPE)
output, err = filtered.communicate()
if filtered.returncode is 0:
logging.info("Result success,status code %d", filtered.returncode)
return output
else:
logging.exception("ErrorCode:%d %s", filtered.returncode, output)
return False
why is the logging.exception throwing errors. Is it the problem with the logging module. I tried to replace the logging.exception with logging.info instead but that didn't work.