How does the "Module: CLI (argparse)" template in Pydev (Eclipse) work?
When I run it, it just prints out the usage help and exits, and my code doesn't get executed. As I understand it from usage output, I need to pass a 'path' argument. But I got the same result when setting the path to something.
The template looks as follows:
#!/usr/local/bin/python2.7
# encoding: utf-8
'''
test1.test -- shortdesc
test1.test is a description
It defines classes_and_methods
#author: user_name
#copyright: 2015 organization_name. All rights reserved.
#license: license
#contact: user_email
#deffield updated: Updated
'''
import sys
import os
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
__all__ = []
__version__ = 0.1
__date__ = '2015-07-31'
__updated__ = '2015-07-31'
DEBUG = 1
TESTRUN = 0
PROFILE = 0
class CLIError(Exception):
'''Generic exception to raise and log different fatal errors.'''
def __init__(self, msg):
super(CLIError).__init__(type(self))
self.msg = "E: %s" % msg
def __str__(self):
return self.msg
def __unicode__(self):
return self.msg
def main(argv=None): # IGNORE:C0111
'''Command line options.'''
if argv is None:
argv = sys.argv
else:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
program_license = '''%s
Created by user_name on %s.
Copyright 2015 organization_name. All rights reserved.
Licensed under the Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_shortdesc, str(__date__))
try:
# Setup argument parser
parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("-r", "--recursive", dest="recurse", action="store_true", help="recurse into subfolders [default: %(default)s]")
parser.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]")
parser.add_argument("-i", "--include", dest="include", help="only include paths matching this regex pattern. Note: exclude is given preference over include. [default: %(default)s]", metavar="RE" )
parser.add_argument("-e", "--exclude", dest="exclude", help="exclude paths matching this regex pattern. [default: %(default)s]", metavar="RE" )
parser.add_argument('-V', '--version', action='version', version=program_version_message)
parser.add_argument(dest="paths", help="paths to folder(s) with source file(s) [default: %(default)s]", metavar="path", nargs='+')
# Process arguments
args = parser.parse_args()
paths = args.paths
verbose = args.verbose
recurse = args.recurse
inpat = args.include
expat = args.exclude
if verbose > 0:
print("Verbose mode on")
if recurse:
print("Recursive mode on")
else:
print("Recursive mode off")
if inpat and expat and inpat == expat:
raise CLIError("include and exclude pattern are equal! Nothing will be processed.")
for inpath in paths:
### do something with inpath ###
print(inpath)
return 0
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
except Exception, e:
if DEBUG or TESTRUN:
raise(e)
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help")
return 2
if __name__ == "__main__":
if DEBUG:
sys.argv.append("-h")
sys.argv.append("-v")
sys.argv.append("-r")
if TESTRUN:
import doctest
doctest.testmod()
if PROFILE:
import cProfile
import pstats
profile_filename = 'test1.test_profile.txt'
cProfile.run('main()', profile_filename)
statsfile = open("profile_stats.txt", "wb")
p = pstats.Stats(profile_filename, stream=statsfile)
stats = p.strip_dirs().sort_stats('cumulative')
stats.print_stats()
statsfile.close()
sys.exit(0)
sys.exit(main())
In DEBUG mode, the -h added to the sys.argv. That has priority over all other inputs, producing the help message (and exit).
Comment out thE line that adds the -h to DEBUG to see the effect of your arguments (plus v and r).
In other modes (except those defined in the main), the commandline arguments should have effect (i.e. the ones in sys.argv).
You're running in DEBUG mode and all you do in that mode is append stuff to argv:
if DEBUG:
sys.argv.append("-h") # <-- that is your problem
sys.argv.append("-v")
sys.argv.append("-r")
Since you're adding an "unknown" argument, when you run:
args = parser.parse_args()
it raises an exception. You can either solve it by removing that line (preferably), or by using:
parser.parse_known_args([])
(the latter is a patch to hide the underlying issue and should be used with cautious. IMO it shouldn't be used in this case!).
Related
The Python script below has worked for me in converting multiple PDF documents to a single PDF when running on Python 2.7. I'm now trying to use it on 3.10 and am getting errors.
I believe I've conquered most of them, especially changing print to print( and disabling imports of CoreFoundation and Quartz.CoreGraphics.
It seems that the only remaining error is:
line 85, in main
writeContext = CGPDFContextCreateWithURL(CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault,
arg, len(arg), False), None, None) NameError: name
'CGPDFContextCreateWithURL' is not defined
If I declare CGPDFContextCreateWithURL as an empty global, the error shifts to CFURLCreateFromFileSystemRepresentation. Declare that and the error is kCFAllocatorDefault.
Here's how I'm trying to handle those.
global CGPDFContextCreateWithURL CGPDFContextCreateWithURL = ""
I don't understand that entire line so any help in making it right would be appreciated.
#
# join
# Joing pages from a a collection of PDF files into a single PDF file.
#
# join [--output <file>] [--shuffle] [--verbose]"
#
# Parameter:
#
# --shuffle
# Take a page from each PDF input file in turn before taking another from each file.
# If this option is not specified then all of the pages from a PDF file are appended
# to the output PDF file before the next input PDF file is processed.
#
# --verbose
# Write information about the doings of this tool to stderr.
#
import sys
import os
import getopt
import tempfile
import shutil
# from CoreFoundation import *
# from Quartz.CoreGraphics import *
global verbose
verbose = False
def createPDFDocumentWithPath(path):
if verbose:
print("Creating PDF document from file %s" % (path))
return CGPDFDocumentCreateWithURL(CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, path, len(path), False))
def writePageFromDoc(writeContext, doc, pageNum):
page = CGPDFDocumentGetPage(doc, pageNum)
if page:
mediaBox = CGPDFPageGetBoxRect(page, kCGPDFMediaBox)
if CGRectIsEmpty(mediaBox):
mediaBox = None
CGContextBeginPage(writeContext, mediaBox)
CGContextDrawPDFPage(writeContext, page)
CGContextEndPage(writeContext)
if verbose:
print("Copied page %d from %s" % (pageNum, doc))
def shufflePages(writeContext, docs, maxPages):
for pageNum in xrange(1, maxPages + 1):
for doc in docs:
writePageFromDoc(writeContext, doc, pageNum)
def append(writeContext, docs, maxPages):
for doc in docs:
for pageNum in xrange(1, maxPages + 1) :
writePageFromDoc(writeContext, doc, pageNum)
def main(argv):
global verbose
# The PDF context we will draw into to create a new PDF
writeContext = None
# If True then generate more verbose information
source = None
shuffle = False
# Parse the command line options
try:
options, args = getopt.getopt(argv, "o:sv", ["output=", "shuffle", "verbose"])
except getopt.GetoptError:
usage()
sys.exit(2)
for option, arg in options:
if option in ("-o", "--output") :
if verbose:
print("Setting %s as the destination." % (arg))
writeContext = CGPDFContextCreateWithURL(CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, arg, len(arg), False), None, None)
elif option in ("-s", "--shuffle") :
if verbose :
print("Shuffle pages to the output file.")
shuffle = True
elif option in ("-v", "--verbose") :
print("Verbose mode enabled.")
verbose = True
else :
print("Unknown argument: %s" % (option))
if writeContext:
# create PDFDocuments for all of the files.
docs = map(createPDFDocumentWithPath, args)
# find the maximum number of pages.
maxPages = 0
for doc in docs:
if CGPDFDocumentGetNumberOfPages(doc) > maxPages:
maxPages = CGPDFDocumentGetNumberOfPages(doc)
if shuffle:
shufflePages(writeContext, docs, maxPages)
else:
append(writeContext, docs, maxPages)
CGPDFContextClose(writeContext)
del writeContext
#CGContextRelease(writeContext)
def usage():
print("Usage: join [--output <file>] [--shuffle] [--verbose]")
if __name__ == "__main__":
main(sys.argv[1:])
I wanna do same cli tools and tried docopt but i struggled many times:
here's my docstring:
"""
usage:
wplan [--progress] [--forced] [--path DIR]
[--verbosity VMODE]
wplan -h
options:
--progress show progress bar
--forced force overwrite
--path DIR ddsfwefafef [default: /path/to/file]
--verbosity VMODE asdfasdf [default: 1]
-h some help
"""
But no one of the following cli strings results in a docopt dictionary:
"wplan --progress" -> no dictionary -> usage screen
"wplan --forced" -> no dictionary -> usage screen
"wplan --verbose" -> no dictionary -> usage screen
Ok - i testet the docopt stuff with that file:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
import os
# to run docopt.docopt in a test: replace DocoptExit in docopt
# func
from docopt import \
DocoptExit, printable_usage, parse_defaults, parse_pattern, \
formal_usage, TokenStream, parse_argv, Option, Dict, \
AnyOptions, extras
def pseudo_docopt(doc, argv=None, help=True, version=None,
options_first=False):
"""Thats a copy of docopt.docopt function. Only the last line
the "DocoptExit()" statement ist replaced by "raise RuntimeError".
"""
if argv is None:
argv = sys.argv[1:]
DocoptExit.usage = printable_usage(doc)
options = parse_defaults(doc)
pattern = parse_pattern(formal_usage(DocoptExit.usage), options)
argv = parse_argv(TokenStream(argv, DocoptExit), list(options),
options_first)
pattern_options = set(pattern.flat(Option))
for ao in pattern.flat(AnyOptions):
doc_options = parse_defaults(doc)
ao.children = list(set(doc_options) - pattern_options)
extras(help, version, argv, doc)
matched, left, collected = pattern.fix().match(argv)
if matched and left == []: # better error message if left?
return Dict((a.name, a.value)
for a in (pattern.flat() + collected))
# if code goes here no dict is sent
raise RuntimeError()
# some command line tests
cltests = [
[],
# 1 arg
["--progress"],
["--forced"],
# 2 args
["--progress", "--forced"],
["--forced", "--progress"],
["--path /path/to/file"],
["--verbosity", "2"],
# 3 args
["--progress", "--path", "/path/to/file"],
["--forced", "--path", "/path/to/file"],
["--progress", "--verbosity", "2"],
["--forced", "--verbosity", "2"],
# 4 args
["--forced", "--progress", "--path", "/path/to/file"],
["--progress", "--forced", "--path", "/path/to/file"],
["--forced", "--progress", "--verbosity", "2"],
["--progress", "--forced", "--verbosity", "2"],
]
# the __doc__ fake
doc = """
usage:
wplan [--progress] [--forced] [--path DIR]
[--verbosity VMODE]
wplan -h
options:
--progress show progress bar
--forced force overwrite
--path DIR ddsfwefafef [default: /path/to/file]
--verbosity VMODE asdfasdf [default: 1]
-h some help
"""
for args in cltests:
cmd = ["wplan", ] + args
info = "run {}".format(cmd)
print(info + os.linesep + "-" * len(info))
try:
data = pseudo_docopt(doc=doc, argv=cmd)
assert isinstance(data, dict)
assert "--progress" in data
assert "--progress" in data
assert "--forced" in data
assert "--path" in data
assert "--verbosity" in data
print("... OK there is a docopt dict")
except RuntimeError:
print("... ERROR: no docopt dict for: {}".format(cmd))
print()
So if i run this docoptest.py file ("python3.7 docopttest.py") only the ERROR info appears!
First two actually work, here are the live demos:
wplan --progress
wplan --forced
The last needs to be modified to match the usage pattern, e.g. like this:
wplan --verbosity 3
Reading your code, I think this will avoid the need for that pseudo_docopt function, allowing use of the real docopt:
for args in cltests:
... # same
try:
data = docopt.docopt(doc=doc, argv=cmd)
... # assert, print... same
except DocoptExit:
print("... ERROR: no docopt dict for: {}".format(cmd))
Note that DocoptExit inherits from SystemExit which inherits form BaseException, so using just except Exception won't work.
I am trying to import the snmpSessionBaseClass python module in a script I am running, but I do not have the module installed and I can't seem to find where to download it. Does anyone know the pip or yum command to download and install this module? Thanks!
import netsnmp
sys.path.insert(1, os.path.join(sys.path[0], os.pardir))
from snmpSessionBaseClass import add_common_options, get_common_options, verify_host, get_data
from pynag.Plugins import PluginHelper,ok,critical
The following code needs to be added to a file called snmpSessionBaseClass.py and that file needs to be placed in a directory that is in pythons path.
#!/usr/bin/env python
# Copyright (C) 2016 rsmuc <rsmuc#mailbox.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with health_monitoring_plugins. If not, see <http://www.gnu.org/licenses/>.
import pynag
import netsnmp
import os
import sys
dev_null = os.open(os.devnull, os.O_WRONLY)
tmp_stdout = os.dup(sys.stdout.fileno())
def dev_null_wrapper(func, *a, **kwargs):
"""
Temporarily swap stdout with /dev/null, and execute given function while stdout goes to /dev/null.
This is useful because netsnmp writes to stdout and disturbes Icinga result in some cases.
"""
os.dup2(dev_null, sys.stdout.fileno())
return_object = func(*a, **kwargs)
sys.stdout.flush()
os.dup2(tmp_stdout, sys.stdout.fileno())
return return_object
def add_common_options(helper):
# Define the common command line parameters
helper.parser.add_option('-H', help="Hostname or ip address", dest="hostname")
helper.parser.add_option('-C', '--community', dest='community', help='SNMP community of the SNMP service on target host.', default='public')
helper.parser.add_option('-V', '--snmpversion', dest='version', help='SNMP version. (1 or 2)', default=2, type='int')
def get_common_options(helper):
# get the common options
host = helper.options.hostname
version = helper.options.version
community = helper.options.community
return host, version, community
def verify_host(host, helper):
if host == "" or host is None:
helper.exit(summary="Hostname must be specified"
, exit_code=pynag.Plugins.unknown
, perfdata='')
netsnmp_session = dev_null_wrapper(netsnmp.Session,
DestHost=helper.options.hostname,
Community=helper.options.community,
Version=helper.options.version)
try:
# Works around lacking error handling in netsnmp package.
if netsnmp_session.sess_ptr == 0:
helper.exit(summary="SNMP connection failed"
, exit_code=pynag.Plugins.unknown
, perfdata='')
except ValueError as error:
helper.exit(summary=str(error)
, exit_code=pynag.Plugins.unknown
, perfdata='')
# make a snmp get, if it fails (or returns nothing) exit the plugin
def get_data(session, oid, helper, empty_allowed=False):
var = netsnmp.Varbind(oid)
varl = netsnmp.VarList(var)
data = session.get(varl)
value = data[0]
if value is None:
helper.exit(summary="snmpget failed - no data for host "
+ session.DestHost + " OID: " +oid
, exit_code=pynag.Plugins.unknown
, perfdata='')
if not empty_allowed and not value:
helper.exit(summary="snmpget failed - no data for host "
+ session.DestHost + " OID: " +oid
, exit_code=pynag.Plugins.unknown
, perfdata='')
return value
# make a snmp get, but do not exit the plugin, if it returns nothing
# be careful! This funciton does not exit the plugin, if snmp get fails!
def attempt_get_data(session, oid):
var = netsnmp.Varbind(oid)
varl = netsnmp.VarList(var)
data = session.get(varl)
value = data[0]
return value
# make a snmp walk, if it fails (or returns nothing) exit the plugin
def walk_data(session, oid, helper):
tag = []
var = netsnmp.Varbind(oid)
varl = netsnmp.VarList(var)
data = list(session.walk(varl))
if len(data) == 0:
helper.exit(summary="snmpwalk failed - no data for host " + session.DestHost
+ " OID: " +oid
, exit_code=pynag.Plugins.unknown
, perfdata='')
for x in range(0, len(data)):
tag.append(varl[x].tag)
return data, tag
# make a snmp walk, but do not exit the plugin, if it returns nothing
# be careful! This function does not exit the plugin, if snmp walk fails!
def attempt_walk_data(session, oid):
tag = []
var = netsnmp.Varbind(oid)
varl = netsnmp.VarList(var)
data = list(session.walk(varl))
for x in range(0, len(data)):
tag.append(varl[x].tag)
return data, tag
def state_summary(value, name, state_list, helper, ok_value = 'ok', info = None):
"""
Always add the status to the long output, and if the status is not ok (or ok_value),
we show it in the summary and set the status to critical
"""
# translate the value (integer) we receive to a human readable value (e.g. ok, critical etc.) with the given state_list
state_value = state_list[int(value)]
summary_output = ''
long_output = ''
if not info:
info = ''
if state_value != ok_value:
summary_output += ('%s status: %s %s ' % (name, state_value, info))
helper.status(pynag.Plugins.critical)
long_output += ('%s status: %s %s\n' % (name, state_value, info))
return (summary_output, long_output)
def add_output(summary_output, long_output, helper):
"""
if the summary output is empty, we don't add it as summary, otherwise we would have empty spaces (e.g.: '. . . . .') in our summary report
"""
if summary_output != '':
helper.add_summary(summary_output)
helper.add_long_output(long_output)
It could be I'm completely misunderstanding the getopt module
I am trying to parse [--magic-m] to my program, but it does not set the correct field.
Part of Encrypt Function
def encrypt(filename, text, magic):
if not magic is None:
hash = pbkdf2_sha256.encrypt(magic, rounds=10000, salt_size=16)
print pbkdf2_sha256.verify(magic, hash)
try:
d = load_image( filename )
except Exception,e:
print str(e)
Part of Load function
def load_image( filename ) :
img = Image.open( os.path.join(__location__, filename) )
img.load()
data = np.asarray( img, dtype="int32" )
return data
Main
if __name__ == "__main__":
if not len(sys.argv[1:]):
usage()
try:
opts,args = getopt.getopt(sys.argv[1:],"hedm:",["help", "encrypt", "decrypt", "magic="])
except getopt.GetoptError as err:
print str(err)
usage()
magic = None
for o,a in opts:
if o in ("-h","--help"):
usage()
elif o in ("-e","--encrypt"):
to_encrypt = True
elif o in ("-d","--decrypt"):
to_encrypt = False
elif o in ("-m", "--magic"):
magic = a
else:
assert False,"Unhandled Option"
print magic
if not to_encrypt:
filename = sys.argv[2]
decrypt(filename, magic)
else:
filename = sys.argv[2]
text = sys.argv[3]
encrypt(filename, text, magic)
I tried calling the program above like this:
[1] python stego.py -e test.jpeg lol -m h
or like this:
[2] python stego.py -e -m h test.jpeg lol
Output becomes:
[1] None
[2] lol
[2] True
[2] [Errno 2] No such file or directory: 'C:\\Users\\Educontract\\Steganography\\-m'
Whitout the option -m everything works fine
The colon should come after m to indicate that it requires an argument. You should also include an equals sign after the long option magic to indicate that it requires an argument.
getopt.getopt(sys.argv[1:],"hedm:",["help", "encrypt", "decrypt", "magic="])
You should put all your options before the arguments, as in your second example.
python stego.py -e -m h test.jpeg lol
If you print sys.argv, I think you'll find that sys.argv[2] and sys.argv[3] are not what you expect. I would fetch the arguments from args, rather than sys.argv.
filename = args[0]
text = args[1]
Note that you may find it easier to use the argparse library instead of getopt. It isn't as strict about requiring options before arguments.
I have a script which has certain options that can either be passed on the command line, or from environment variables. The CLI should take precedence if both are present, and an error occur if neither are set.
I could check that the option is assigned after parsing, but I prefer to let argparse to do the heavy lifting and be responsible for displaying the usage statement if parsing fails.
I have come up with a couple of alternative approaches to this (which I will post below as answers so they can be discussed separately) but they feel pretty kludgey to me and I think that I am missing something.
Is there an accepted "best" way of doing this?
(Edit to make the desired behaviour clear when both the CLI option and environment variable are unset)
You can set the default= of the argument to a .get() of os.environ with the environment variable you want to grab.
You can also pass a 2nd argument in the .get() call, which is the default value if .get() doesn't find an environment variable by that name (by default .get() returns None in that case).
import argparse
import os
parser = argparse.ArgumentParser(description='test')
parser.add_argument('--url', default=os.environ.get('URL'))
args = parser.parse_args()
if not args.url:
exit(parser.print_usage())
I use this pattern frequently enough that I have packaged a simple action class to handle it:
import argparse
import os
class EnvDefault(argparse.Action):
def __init__(self, envvar, required=True, default=None, **kwargs):
if not default and envvar:
if envvar in os.environ:
default = os.environ[envvar]
if required and default:
required = False
super(EnvDefault, self).__init__(default=default, required=required,
**kwargs)
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, values)
I can then call this from my code with:
import argparse
from envdefault import EnvDefault
parser=argparse.ArgumentParser()
parser.add_argument(
"-u", "--url", action=EnvDefault, envvar='URL',
help="Specify the URL to process (can also be specified using URL environment variable)")
args=parser.parse_args()
I usually have to do this for multiple arguments (authentication and API keys).. this is simple and straight forward. Uses **kwargs.
def environ_or_required(key):
return (
{'default': os.environ.get(key)} if os.environ.get(key)
else {'required': True}
)
parser.add_argument('--thing', **environ_or_required('THING'))
ConfigArgParse adds support for environment variables to argparse, so you can do things like:
p = configargparse.ArgParser()
p.add('-m', '--moo', help='Path of cow', env_var='MOO_PATH')
options = p.parse_args()
One option is to check whether the environment variable is set, and to modify the calls to add_argument accordingly
e.g.
import argparse
import os
parser=argparse.ArgumentParser()
if 'CVSWEB_URL' in os.environ:
cvsopt = { 'default': os.environ['CVSWEB_URL'] }
else:
cvsopt = { 'required': True }
parser.add_argument(
"-u", "--cvsurl", help="Specify url (overrides CVSWEB_URL environment variable)",
**cvsopt)
args=parser.parse_args()
The topic is quite old, but I had similar problem and I thought I would share my solution with you. Unfortunately custom action solution suggested by #Russell Heilling doesn't work for me for couple of reasons:
It prevents me from using predefined actions (like store_true)
I would rather like it to fallback to default when envvar is not in os.environ (that could be easily fixed)
I would like to have this behaviour for all of my arguments without specifying action or envvar (which should always be action.dest.upper())
Here's my solution (in Python 3):
class CustomArgumentParser(argparse.ArgumentParser):
class _CustomHelpFormatter(argparse.ArgumentDefaultsHelpFormatter):
def _get_help_string(self, action):
help = super()._get_help_string(action)
if action.dest != 'help':
help += ' [env: {}]'.format(action.dest.upper())
return help
def __init__(self, *, formatter_class=_CustomHelpFormatter, **kwargs):
super().__init__(formatter_class=formatter_class, **kwargs)
def _add_action(self, action):
action.default = os.environ.get(action.dest.upper(), action.default)
return super()._add_action(action)
There is an example use-case for ChainMap where you merge together defaults, environment variables and command line arguments.
import os, argparse
defaults = {'color': 'red', 'user': 'guest'}
parser = argparse.ArgumentParser()
parser.add_argument('-u', '--user')
parser.add_argument('-c', '--color')
namespace = parser.parse_args()
command_line_args = {k:v for k, v in vars(namespace).items() if v}
combined = ChainMap(command_line_args, os.environ, defaults)
Came to me from a great talk about beautiful and idiomatic python.
However, I'm not sure how to go about the difference of lower- and uppercase dictionary keys. In the case where both -u foobar is passed as an argument and environment is set to USER=bazbaz, the combined dictionary will look like {'user': 'foobar', 'USER': 'bazbaz'}.
Thought I'd post my solution as the original question/answer gave me a lot of help.
My problem is a little different to Russell's. I'm using OptionParser and instead of an environmental variable for each argument I have just one which simulates the command line.
i.e.
MY_ENVIRONMENT_ARGS = --arg1 "Maltese" --arg2 "Falcon" -r "1930" -h
Solution:
def set_defaults_from_environment(oparser):
if 'MY_ENVIRONMENT_ARGS' in os.environ:
environmental_args = os.environ[ 'MY_ENVIRONMENT_ARGS' ].split()
opts, _ = oparser.parse_args( environmental_args )
oparser.defaults = opts.__dict__
oparser = optparse.OptionParser()
oparser.add_option('-a', '--arg1', action='store', default="Consider")
oparser.add_option('-b', '--arg2', action='store', default="Phlebas")
oparser.add_option('-r', '--release', action='store', default='1987')
oparser.add_option('-h', '--hardback', action='store_true', default=False)
set_defaults_from_environment(oparser)
options, _ = oparser.parse_args(sys.argv[1:])
Here I don't throw an error if an argument is not found. But if I wish to I could just do something like
for key in options.__dict__:
if options.__dict__[key] is None:
# raise error/log problem/print to console/etc
You can use OptionParser()
from optparse import OptionParser
def argument_parser(self, parser):
parser.add_option('--foo', dest="foo", help="foo", default=os.environ.get('foo', None))
parser.add_option('--bar', dest="bar", help="bar", default=os.environ.get('bar', None))
return(parser.parse_args())
parser = OptionParser()
(options, args) = argument_parser(parser)
foo = options.foo
bar = options.bar
print("foo: {}".format(foo))
print("bar: {}".format(bar))
shell:
export foo=1
export bar=2
python3 script.py
The Click library handles this explicitly:
import click
#click.command()
#click.argument('src', envvar='SRC', type=click.File('r'))
def echo(src):
"""Print value of SRC environment variable."""
click.echo(src.read())
And from the command line:
$ export SRC=hello.txt
$ echo
Hello World!
https://click.palletsprojects.com/en/master/arguments/#environment-variables
You can install it with
pip install click
Here's a relatively simple (looks longer because it's well-commented) yet complete solution that avoids kludging default by using the namespace argument of parse_args. By default it parses environment variables no differently than command-line arguments though that can easily be changed.
import shlex
# Notes:
# * Based on https://github.com/python/cpython/blob/
# 15bde92e47e824369ee71e30b07f1624396f5cdc/
# Lib/argparse.py
# * Haven't looked into handling "required" for mutually exclusive groups
# * Probably should make new attributes private even though it's ugly.
class EnvArgParser(argparse.ArgumentParser):
# env_k: The keyword to "add_argument" as well as the attribute stored
# on matching actions.
# env_f: The keyword to "add_argument". Defaults to "env_var_parse" if
# not provided.
# env_i: Basic container type to identify unfilled arguments.
env_k = "env_var"
env_f = "env_var_parse"
env_i = type("env_i", (object,), {})
def add_argument(self, *args, **kwargs):
map_f = (lambda m,k,f=None,d=False:
(k, k in m, m.pop(k,f) if d else m.get(k,f)))
env_k = map_f(kwargs, self.env_k, d=True, f="")
env_f = map_f(kwargs, self.env_f, d=True, f=self.env_var_parse)
if env_k[1] and not isinstance(env_k[2], str):
raise ValueError(f"Parameter '{env_k[0]}' must be a string.")
if env_f[1] and not env_k[1]:
raise ValueError(f"Parameter '{env_f[0]}' requires '{env_k[0]}'.")
if env_f[1] and not callable(env_f[2]):
raise ValueError(f"Parameter '{env_f[0]}' must be callable.")
action = super().add_argument(*args, **kwargs)
if env_k[1] and not action.option_strings:
raise ValueError(f"Positional parameters may not specify '{env_k[0]}'.")
# We can get the environment now:
# * We need to know now if the keys exist anyway
# * os.environ is static
env_v = map_f(os.environ, env_k[2], f="")
# Examples:
# env_k:
# ("env_var", True, "FOO_KEY")
# env_v:
# ("FOO_KEY", False, "")
# ("FOO_KEY", True, "FOO_VALUE")
#
# env_k:
# ("env_var", False, "")
# env_v:
# ("" , False, "")
# ("", True, "RIDICULOUS_VALUE")
# Add the identifier to all valid environment variable actions for
# later access by i.e. the help formatter.
if env_k[1]:
if env_v[1] and action.required:
action.required = False
i = self.env_i()
i.a = action
i.k = env_k[2]
i.f = env_f[2]
i.v = env_v[2]
i.p = env_v[1]
setattr(action, env_k[0], i)
return action
# Overriding "_parse_known_args" is better than "parse_known_args":
# * The namespace will already have been created.
# * This method runs in an exception handler.
def _parse_known_args(self, arg_strings, namespace):
"""precedence: cmd args > env var > preexisting namespace > defaults"""
for action in self._actions:
if action.dest is argparse.SUPPRESS:
continue
try:
i = getattr(action, self.env_k)
except AttributeError:
continue
if not i.p:
continue
setattr(namespace, action.dest, i)
namespace, arg_extras = super()._parse_known_args(arg_strings, namespace)
for k,v in vars(namespace).copy().items():
# Setting "env_i" on the action is more effective than using an
# empty unique object() and mapping namespace attributes back to
# actions.
if isinstance(v, self.env_i):
fv = v.f(v.a, v.k, v.v, arg_extras)
if fv is argparse.SUPPRESS:
delattr(namespace, k)
else:
# "_parse_known_args::take_action" checks for action
# conflicts. For simplicity we don't.
v.a(self, namespace, fv, v.k)
return (namespace, arg_extras)
def env_var_parse(self, a, k, v, e):
# Use shlex, yaml, whatever.
v = shlex.split(v)
# From "_parse_known_args::consume_optional".
n = self._match_argument(a, "A"*len(v))
# From the main loop of "_parse_known_args". Treat additional
# environment variable arguments just like additional command-line
# arguments (which will eventually raise an exception).
e.extend(v[n:])
return self._get_values(a, v[:n])
# Derived from "ArgumentDefaultsHelpFormatter".
class EnvArgHelpFormatter(argparse.HelpFormatter):
"""Help message formatter which adds environment variable keys to
argument help.
"""
env_k = EnvArgParser.env_k
# This is supposed to return a %-style format string for "_expand_help".
# Since %-style strings don't support attribute access we instead expand
# "env_k" ourselves.
def _get_help_string(self, a):
h = super()._get_help_string(a)
try:
i = getattr(a, self.env_k)
except AttributeError:
return h
s = f" ({self.env_k}: {i.k})"
if s not in h:
h += s
return h
# An example mix-in.
class DefEnvArgHelpFormatter\
( EnvArgHelpFormatter
, argparse.ArgumentDefaultsHelpFormatter
):
pass
Example program:
parser = EnvArgParser\
( prog="Test Program"
, formatter_class=DefEnvArgHelpFormatter
)
parser.add_argument\
( '--bar'
, required=True
, env_var="BAR"
, type=int
, nargs="+"
, default=22
, help="Help message for bar."
)
parser.add_argument\
( 'baz'
, type=int
)
args = parser.parse_args()
print(args)
Example program output:
$ BAR="1 2 3 '45 ' 6 7" ./envargparse.py 123
Namespace(bar=[1, 2, 3, 45, 6, 7], baz=123)
$ ./envargparse.py -h
usage: Test Program [-h] --bar BAR [BAR ...] baz
positional arguments:
baz
optional arguments:
-h, --help show this help message and exit
--bar BAR [BAR ...] Help message for bar. (default: 22) (env_var: BAR)
Another option:
parser = argparse.ArgumentParser()
env = os.environ
def add_argument(key, *args, **kwargs):
if key in env:
kwargs['default'] = env[key]
parser.add_argument(*args, **kwargs)
add_argument('--type', type=str)
Or this one, using os.getenv for setting default value:
parser = argparse.ArgumentParser()
parser.add_argument('--type', type=int, default=os.getenv('type',100))