Develop Reference

Where can I temporarily store file byte data in a python flask server?

Right now, I am using fine-uploader as my front-end and python-flask as my backend. What I need to do is that I need to send large files from the front end to my server. I can do this easily without concurrent chunking. But once I turn it on, the file gets corrupted. I think this is because when you concurrently chunk, I would guess part 7 could get added before part 5 gets added. Again, non-concurrent chunking is OK since it is sequential.
I need to know if there is some sort of temporary, global variable or way that I can store the chunk parts temporarily. I tried redis but unfortunately when I get the data from the redis, it shows a decoding error because probably redis tries to turn it into a string when I just need it to be bytes.
If all else fails, I'll just go to my last resort, which is to put the parts into their own little files then open them later on to combine them one by one.
Here is my code for your reference. It still has the redis methods on it.
def upload_temp_file_part(redis, request):
try:
# Remember the paramName was set to 'file', we can use that here to grab it
file = request.files['qqfile']
uuid = request.form['qquuid']
part = request.form['qqpartindex']
offset = request.form['qqpartbyteoffset']
key_content = 'file_content_' + uuid + part
key_offset = 'file_offset_' + uuid + part
value_content = file.stream.read()
value_offset = offset
logging.info("Setting part " + part + " of " + uuid)
redis.set(key_content, value_content)
redis.set(key_offset, value_offset)
except Exception as e:
logging.error(e)
def combine_temp_file_part(redis, request):
try:
uuid = request.form['qquuid']
total_parts = request.form['qqtotalparts']
save_path = os.path.join(os.getenv('UPLOAD_FOLDER_TEMP'), uuid)
with open(save_path, 'ab') as f:
for index in range(0, int(total_parts)):
key_content = 'file_content_' + uuid + str(index)
key_offset = 'file_offset_' + uuid + str(index)
logging.info("Get part " + str(index) + " of " + uuid)
value_content = redis.get(key_content)
value_offset = redis.get(key_offset)
if value_content is None or value_offset is None:
pass
# Throw Error
logging.info("Placing part " + str(index) + " of " + uuid)
f.seek(value_offset)
f.write(value_content)
redis.delete(value_offset)
redis.delete(value_content)
except Exception as e:
logging.error(e)

Python Module snmpSessionBaseClass: Where to download

I am trying to import the snmpSessionBaseClass python module in a script I am running, but I do not have the module installed and I can't seem to find where to download it. Does anyone know the pip or yum command to download and install this module? Thanks!
import netsnmp
sys.path.insert(1, os.path.join(sys.path[0], os.pardir))
from snmpSessionBaseClass import add_common_options, get_common_options, verify_host, get_data
from pynag.Plugins import PluginHelper,ok,critical

The following code needs to be added to a file called snmpSessionBaseClass.py and that file needs to be placed in a directory that is in pythons path.
#!/usr/bin/env python
# Copyright (C) 2016 rsmuc <rsmuc#mailbox.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with health_monitoring_plugins. If not, see <http://www.gnu.org/licenses/>.
import pynag
import netsnmp
import os
import sys
dev_null = os.open(os.devnull, os.O_WRONLY)
tmp_stdout = os.dup(sys.stdout.fileno())
def dev_null_wrapper(func, *a, **kwargs):
"""
Temporarily swap stdout with /dev/null, and execute given function while stdout goes to /dev/null.
This is useful because netsnmp writes to stdout and disturbes Icinga result in some cases.
"""
os.dup2(dev_null, sys.stdout.fileno())
return_object = func(*a, **kwargs)
sys.stdout.flush()
os.dup2(tmp_stdout, sys.stdout.fileno())
return return_object
def add_common_options(helper):
# Define the common command line parameters
helper.parser.add_option('-H', help="Hostname or ip address", dest="hostname")
helper.parser.add_option('-C', '--community', dest='community', help='SNMP community of the SNMP service on target host.', default='public')
helper.parser.add_option('-V', '--snmpversion', dest='version', help='SNMP version. (1 or 2)', default=2, type='int')
def get_common_options(helper):
# get the common options
host = helper.options.hostname
version = helper.options.version
community = helper.options.community
return host, version, community
def verify_host(host, helper):
if host == "" or host is None:
helper.exit(summary="Hostname must be specified"
, exit_code=pynag.Plugins.unknown
, perfdata='')
netsnmp_session = dev_null_wrapper(netsnmp.Session,
DestHost=helper.options.hostname,
Community=helper.options.community,
Version=helper.options.version)
try:
# Works around lacking error handling in netsnmp package.
if netsnmp_session.sess_ptr == 0:
helper.exit(summary="SNMP connection failed"
, exit_code=pynag.Plugins.unknown
, perfdata='')
except ValueError as error:
helper.exit(summary=str(error)
, exit_code=pynag.Plugins.unknown
, perfdata='')
# make a snmp get, if it fails (or returns nothing) exit the plugin
def get_data(session, oid, helper, empty_allowed=False):
var = netsnmp.Varbind(oid)
varl = netsnmp.VarList(var)
data = session.get(varl)
value = data[0]
if value is None:
helper.exit(summary="snmpget failed - no data for host "
+ session.DestHost + " OID: " +oid
, exit_code=pynag.Plugins.unknown
, perfdata='')
if not empty_allowed and not value:
helper.exit(summary="snmpget failed - no data for host "
+ session.DestHost + " OID: " +oid
, exit_code=pynag.Plugins.unknown
, perfdata='')
return value
# make a snmp get, but do not exit the plugin, if it returns nothing
# be careful! This funciton does not exit the plugin, if snmp get fails!
def attempt_get_data(session, oid):
var = netsnmp.Varbind(oid)
varl = netsnmp.VarList(var)
data = session.get(varl)
value = data[0]
return value
# make a snmp walk, if it fails (or returns nothing) exit the plugin
def walk_data(session, oid, helper):
tag = []
var = netsnmp.Varbind(oid)
varl = netsnmp.VarList(var)
data = list(session.walk(varl))
if len(data) == 0:
helper.exit(summary="snmpwalk failed - no data for host " + session.DestHost
+ " OID: " +oid
, exit_code=pynag.Plugins.unknown
, perfdata='')
for x in range(0, len(data)):
tag.append(varl[x].tag)
return data, tag
# make a snmp walk, but do not exit the plugin, if it returns nothing
# be careful! This function does not exit the plugin, if snmp walk fails!
def attempt_walk_data(session, oid):
tag = []
var = netsnmp.Varbind(oid)
varl = netsnmp.VarList(var)
data = list(session.walk(varl))
for x in range(0, len(data)):
tag.append(varl[x].tag)
return data, tag
def state_summary(value, name, state_list, helper, ok_value = 'ok', info = None):
"""
Always add the status to the long output, and if the status is not ok (or ok_value),
we show it in the summary and set the status to critical
"""
# translate the value (integer) we receive to a human readable value (e.g. ok, critical etc.) with the given state_list
state_value = state_list[int(value)]
summary_output = ''
long_output = ''
if not info:
info = ''
if state_value != ok_value:
summary_output += ('%s status: %s %s ' % (name, state_value, info))
helper.status(pynag.Plugins.critical)
long_output += ('%s status: %s %s\n' % (name, state_value, info))
return (summary_output, long_output)
def add_output(summary_output, long_output, helper):
"""
if the summary output is empty, we don't add it as summary, otherwise we would have empty spaces (e.g.: '. . . . .') in our summary report
"""
if summary_output != '':
helper.add_summary(summary_output)
helper.add_long_output(long_output)

Python variable substitution

I have a script that calls a list of linux guests I am trying to tidy up. Here is the code:
#!/usr/bin/python
guests = ['guest1','guest2','guest3','guest*']
def serverCheck(guestList)
for g in guestList:
server = AdminControl.completeObjectName('cell=tstenvironment,node=guest1,name=uatenvironment,type=Server,*')
try:
status = AdminControl.getAttribute(server, 'state')
print g + status
except:
print "Error %s is down." % g
serverCheck(guests)
The problem lies in this line:
server = AdminControl.completeObjectName('cell=Afcutst,node=%s,name=afcuuat1,type=Server,*') % g
How do I use my list to populate the node variable while still being able to pass the info within the parentheses to the AdminControl function?

The argument string itself is the argument to the % operator, not the return value of the function call.
server = AdminControl.completeObjectName(
'cell=Afcutst,node=%s,name=afcuuat1,type=Server,*' % (g,)
)
Peeking into the crystal ball, Python 3.6 will allow you to write
server = AdminControl.completeObjectName(
f'cell=Afcutst,node={g},name=afcuuat1,type=Server,*'
)
embedding the variable directly into a special format string literal.

can you try like this
AdminControl.completeObjectName('cell=tstenvironment,node=%s,name=uatenvironment,type=Server,*'%g)

For more readability I would suggest this and also using the same way to format strings from variables (here I chose str.format)
guests = ['guest1','guest2','guest3','guest*']
def serverCheck(guestList)
name_tpl = 'cell=tstenvironment,node={},name=uatenvironment,type=Server,*'
for g in guestList:
obj_name = name_tpl.format(g)
server = AdminControl.completeObjectName(obj_name)
try:
status = AdminControl.getAttribute(server, 'state')
print '{}: {}'.format(g, status)
except:
print 'Error {} is down'.format(g)
serverCheck(guests)

Python program crashing

So I've designed a program that runs on a computer, looks for particular aspects of files that have been plaguing us, and deletes the files if a flag is passed. Unfortunately the program seems to be almost-randomly shutting down/crashing. I say almost-randomly, because the program always exits after it deletes a file, though it will commonly stay up after a success.
I've run a parallel Python program that counts upwards in the same intervals, but does nothing else. This program does not crash/exit, and stays open.
Is there perhaps a R/W access issue? I am running the program as administrator, so I'm not sure why that would be the case.
Here's the code:
import glob
import os
import time
import stat
#logging
import logging
logging.basicConfig(filename='disabledBots.log')
import datetime
runTimes = 0
currentPhp = 0
output = 0
output2 = 0
while runTimes >= 0:
#Cycles through .php files
openedProg = glob.glob('*.php')
openedProg = openedProg[currentPhp:currentPhp+1]
progInput = ''.join(openedProg)
if progInput != '':
theBot = open(progInput,'r')
#Singles out "$output" on this particular line and closes the process
readLines = theBot.readlines()
wholeLine = (readLines[-4])
output = wholeLine[4:11]
#Singles out "set_time_limit(0)"
wholeLine2 = (readLines[0])
output2 = wholeLine2[6:23]
theBot.close()
if progInput == '':
currentPhp = -1
#Kills the program if it matches the code
currentTime = datetime.datetime.now()
if output == '$output':
os.chmod(progInput, stat.S_IWRITE)
os.remove(progInput)
logging.warning(str(currentTime) +' ' + progInput + ' has been deleted. Please search for a faux httpd.exe process and kill it.')
currentPhp = 0
if output2 == 'set_time_limit(0)':
os.chmod(progInput, stat.S_IWRITE)
os.remove(progInput)
logging.warning(str(currentTime) +' ' + progInput + ' has been deleted. Please search for a faux httpd.exe process and kill it.')
currentPhp = 0
else:
currentPhp = currentPhp + 1
time.sleep(30)
#Prints the number of cycles
runTimes = runTimes + 1
logging.warning((str(currentTime) + ' botKiller2.0 has scanned '+ str(runTimes) + ' times.'))
print('botKiller3.0 has scanned ' + str(runTimes) + ' times.')

Firstly, it'll be hell of a lot easier to work out what's going on if you base your code around something like this...
for fname in glob.glob('*.php'):
with open(fname) as fin:
lines = fin.readlines()
if '$output' in lines[-4] or 'set_time_limit(0)' in lines[0]:
try:
os.remove(fname)
except IOError as e:
print "Couldn't remove:", fname
And err, that's not actually a secondly at the moment, your existing code is just too tricky to follow fullstop, let alone all the bits that could cause a strange error that we don't know yet!

if os.path.exists(progInput):
os.chmod(progInput, stat.S_IWRITE)
os.remove(progInput)
ALSO:
You never reset the output or output2 variables in the loop?
is this on purpose?

python configparser hangs silently

I have a feeling I'm being stupid. Given this ini file:
[main]
source1 = ./testing/sdir1
sync1 = ./testing/sydir1
archive1 = ./testing/adir1
source2 = ./testing/sdir2
sync2 = ./testing/sydir2
archive2 = ./testing/adir2
[logging]
log_dir = .
log_file = pixelsync.log
log_level = DEBUG
The following code hangs:
import ConfigParser
CONFIG_FILE = 'pixelsync.ini'
def parse_config() :
"""read details from the config file"""
global CONFIG_FILE
config = ConfigParser.SafeConfigParser()
config.read(CONFIG_FILE)
index = 1
while True :
if config.has_option('main', 'source' + str(index)) and \
config.has_option('main', 'sync' + str(index)) and \
config.has_option('main', 'archive' + str(index)) :
result = ( config.get('main', 'source' + str(index)),
config.get('main', 'sync' + str(index)),
config.get('main', 'archive' + str(index)))
index += 1
else :
if index == 1 :
print "could not setup any trios from the config file. exiting."
sys.exit(1)
return result
if __name__ == '__main__' :
options = parse_config()
It hangs on the 'if' clause.
If I replace the 'if' clause with :
if config.has_option('main', 'source1' ) and \
config.has_option('main', 'sync1' ) and \
config.has_option('main', 'archive1' ) :
it doesn't hang. (doesn't do what I want since I need to loop through an arbitrary number of sets of three, but it doesn't silently hang.
Python v2.7.3 on ubuntu 12.04 (Precise), 32bit.

The reason your program hangs is it never breaks out of the loop - it goes on forever. Rather than simply setting result, you need to return it. (An alternative is to set it and then use break to break out of the loop and return, but that is somewhat roundabout. It's better to simply return it straight away.
Note that doing while True: and counting like that isn't very Pythonic, the preferred approach is to instead use itertools.count().
E.g:
import itertools
...
for index in itertools.count(1):
...
Note that this shows a design flaw. You probably want to have a way of knowing if you are never going to get a suitable result. Infinite loops are generally bad.