I am trying to create a python click Command using the API instead of decorators. This is because I am trying to create commands dynamically from a yaml file.
parsed yaml file:
{'adhoc': {'args': ['abcd',
{'analytic-type': {'type': 'click.Choice(["prof", "fac"], '
'case_sensitive=False)'}},
{'lobplat': {'default': 'ALL',
'type': 'click.Choice(["A","B","C","D","ALL",],case_sensitive=False)'}}],
'options': [{'environment': {'alias': '-e',
'envvar': 'env',
'show_default': 'loc',
'type': 'click.Choice(["a", "b", '
'"c", "d", "e"], '
'case_sensitive=False)'}},
{'email': {'alias': '-m',
'default': 'test#test.com',
'multiple': True}},
{'runtype': {'alias': '-r',
'default': 'ADHOC',
'type': 'click.Choice(["TEST","ADHOC","SCHEDULED"], '
'case_sensitive=False)'}}],
'script': 'nohup '
'/path/to/script/script'}}
At the top level it defines a command called adhoc which has 3 parts:
Arguments (args)
Options (options)
Script (This is the function of the command)
Both argument and options have a list of different Parameters that I want to create.
Here is the class that I have written:
import click
import yaml
class Commander():
def __init__(self) -> None:
pass
def run_command(self, script):
pass
def str_to_class(self,classname):
return getattr(sys.modules[__name__], classname)
def create_args(self,arguments):
all_args = []
for arg in arguments:
if isinstance(arg, str):
all_args.append(click.Argument([arg]))
else:
attributes = arg[list(arg.keys())[0]]
print(attributes)
all_args.append(click.Argument([arg],**attributes))
return all_args
def convert_to_command(self,yaml):
for key, value in yaml.items():
name = key
script = value["script"]
options = value["options"]
args = value["args"]
click_arg = self.create_args(args)
print(click_arg)
if __name__ == "__main__":
commander = Commander()
yaml = {'adhoc': {'args': ['abcd',
{'analytic-type': {'type': 'click.Choice(["prof", "fac"], '
'case_sensitive=False)'}},
{'lobplat': {'default': 'ALL',
'type': 'click.Choice(["A","B","C","D","ALL",],case_sensitive=False)'}}],
'options': [{'environment': {'alias': '-e',
'envvar': 'env',
'show_default': 'loc',
'type': 'click.Choice(["a", "b", '
'"c", "d", "e"], '
'case_sensitive=False)'}},
{'email': {'alias': '-m',
'default': 'test#test.com',
'multiple': True}},
{'runtype': {'alias': '-r',
'default': 'ADHOC',
'type': 'click.Choice(["TEST","ADHOC","SCHEDULED"], '
'case_sensitive=False)'}}],
'script': 'nohup '
'/path/to/script/script'}}
commander.convert_to_command(yaml)
These functions are not complete. Currently I am working on writing a function to create Arguments out of the Yaml dictionary. However upon running create_command() I get the following error:
File "/project/helper/commander.py", line 111, in <module>
commander.convert_to_command(yaml)
File "/project/hassle/helper/commander.py", line 45, in convert_to_command
click_arg = self.create_args(args)
File "/project/hassle/helper/commander.py", line 32, in create_args
all_args.append(click.Argument([arg],**attributes))
File "/home/myself/miniconda3/envs/py_copier/lib/python3.7/site-packages/click/core.py", line 2950, in __init__
super().__init__(param_decls, required=required, **attrs)
File "/home/myself/miniconda3/envs/py_copier/lib/python3.7/site-packages/click/core.py", line 2073, in __init__
param_decls or (), expose_value
File "/home/myself/miniconda3/envs/py_copier/lib/python3.7/site-packages/click/core.py", line 2983, in _parse_decls
name = name.replace("-", "_").lower()
AttributeError: 'dict' object has no attribute 'replace'
Thank you for the updated code snippet!
Now, what is going on here:
all_args.append(click.Argument([arg],**attributes))
Where arg is:
{'analytic-type': {'type': 'click.Choice(["prof", "fac"], case_sensitive=False)'}}
but if you look at the documentation about the click.Argument class, you'll see the following:
class click.Argument(param_decls, required=None, **attrs)
param_decls (Sequence[str]) –
You made the sequence part, but inside the sequence you still have the dict, instead of the string(s) with the argument name(s). So you should provide click.Argument with something like this:
[list(arg.keys())[0]]
# e.g.
all_args.append(click.Argument([list(arg.keys())[0]],**attributes))
But now you have another problem:
AttributeError: 'str' object has no attribute '__name__'
which is now related to your attributes, as you pass the function as string, but click expects callable. Take a look at this question. It should resolve the issue.
PS: I've just tested it with a quick and dirty eval and it works, so when you fix the callable problem in a proper way, you are good to move on.
Related
I have a function that verifies if a given input string is a proper GCP zone:
def validate_zone(compute, project_id, zone):
try:
zone_response = compute.zones().get(project=project_id, zone=zone).execute()
print(zone_response)
print(zone_response.return_value)
if ['status'] in zone_response:
zone_details = {
'status': zone_response['status'],
'region': zone_response['region'],
'name': zone_response['name']
}
return zone_details
else:
return "Zone {} not found for project {}".format(zone, project_id)
except HttpError as error:
print("Error calling zone {}: \n {}".format(zone, error))
I am trying to write a test to verify that but I can't mock the output of the compute method correctly.
#patch('googleapiclient.discovery')
def test_validate_zone(self, mock_response):
compute = mock_response.build(serviceName='compute', version='v1')
compute.zones().get(project_id=self.project_id, zone=self.zone).execute().return_value = {
'status': 'status',
'region': 'region',
'name': 'name'
}
zone_response = inventory.validate_zone(compute, self.project_id, self.zone)
print(zone_response)
This results in the zone_response output being a MagicMock object with its return_value being correct as developed in the test.
zone_response = MagicMock name='discovery.build().zones().get().execute()' id='139870134525456'
zone_response.return_value = {'status': 'status', 'region': 'region', 'name': 'name'}
Any ideas on what I'm doing wrong? I've been trying to write tests for this for quite a while so maybe my approach is just off.
Turns out the issue was the () on the execute method in the test. So the correct test should be:
#patch('inventory.discovery.build', serviceName='compute', version='v1')
def test_validate_zone(self, compute):
print(compute)
compute.zones().get(project_id=self.project_id, zone=self.zone).execute.return_value = {
'status': 'status',
'region': 'region',
'name': 'name'
}
zone_response = inventory.validate_zone(compute, self.project_id, self.zone)
print(zone_response)
Source can be found at: https://realpython.com/python-mock-library/#managing-a-mocks-return-value
I'm using ConfigObj (5.0.6, both on python 2.7 and python 3.8) to manage my configs, but when I'm writing to file config with some sections only presented in configspec, they're appearing only as empty sections, which is not desired. I would appreciate any suggestions of fixing that behaviour of ConfigObj.
Minimal example of what happening:
from configobj import ConfigObj
from validate import Validator
spec = ["[Section]", "option = boolean(default=True)"]
config = ConfigObj(infile={'Section2': {'option2': False}}, configspec=spec)
config.validate(Validator())
print(config)
print(config.write())
Output:
{'Section2': {'option2': False}, 'Section': {'option': True}}
['[Section2]', ' option2 = False', '[Section]']
Desired output (there should be no empty sections when writing):
{'Section2': {'option2': False}, 'Section': {'option': True}}
['[Section2]', ' option2 = False']
Edit 1: I'm using write() to actually write into file so I would prefer not just mess with returned list of strings
To put default values in the output config file, pass copy = True to the validate:
from configobj import ConfigObj
from validate import Validator
spec = ["[Section]", "option = boolean(default=True)"]
config = ConfigObj(infile={'Section2': {'option2': False}}, configspec=spec)
# set copy = True vvvvvvvvvvv
config.validate(Validator(), copy = True)
print(config)
print(config.write())
which gives your desired output
{'Section2': {'option2': False}, 'Section': {'option': True}}
['[Section2]', 'option2 = False', '[Section]', 'option = True']
I have a pretty big dictionary which looks like this:
{
'startIndex': 1,
'username': 'myemail#gmail.com',
'items': [{
'id': '67022006',
'name': 'Adopt-a-Hydrant',
'kind': 'analytics#accountSummary',
'webProperties': [{
'id': 'UA-67522226-1',
'name': 'Adopt-a-Hydrant',
'websiteUrl': 'https://www.udemy.com/,
'internalWebPropertyId': '104343473',
'profiles': [{
'id': '108333146',
'name': 'Adopt a Hydrant (Udemy)',
'type': 'WEB',
'kind': 'analytics#profileSummary'
}, {
'id': '132099908',
'name': 'Unfiltered view',
'type': 'WEB',
'kind': 'analytics#profileSummary'
}],
'level': 'STANDARD',
'kind': 'analytics#webPropertySummary'
}]
}, {
'id': '44222959',
'name': 'A223n',
'kind': 'analytics#accountSummary',
And so on....
When I copy this dictionary on my Jupyter notebook and I run the exact same function I run on my django code it runs as expected, everything is literarily the same, in my django code I'm even printing the dictionary out then I copy it to the notebook and run it and I get what I'm expecting.
Just for more info this is the function:
google_profile = gp.google_profile # Get google_profile from DB
print(google_profile)
all_properties = []
for properties in google_profile['items']:
all_properties.append(properties)
site_selection=[]
for single_property in all_properties:
single_propery_name=single_property['name']
for single_view in single_property['webProperties'][0]['profiles']:
single_view_id = single_view['id']
single_view_name = (single_view['name'])
selections = single_propery_name + ' (View: '+single_view_name+' ID: '+single_view_id+')'
site_selection.append(selections)
print (site_selection)
So my guess is that my notebook has some sort of json parser installed or something like that? Is that possible? Why in django I can't access dictionaries the same way I can on my ipython notebooks?
EDITS
More info:
The error is at the line: for properties in google_profile['items']:
Django debug is: TypeError at /gconnect/ string indices must be integers
Local Vars are:
all_properties =[]
current_user = 'myemail#gmail.com'
google_profile = `the above dictionary`
So just to make it clear for who finds this question:
If you save a dictionary in a database django will save it as a string, so you won't be able to access it after.
To solve this you can re-convert it to a dictionary:
The answer from this post worked perfectly for me, in other words:
import json
s = "{'muffin' : 'lolz', 'foo' : 'kitty'}"
json_acceptable_string = s.replace("'", "\"")
d = json.loads(json_acceptable_string)
# d = {u'muffin': u'lolz', u'foo': u'kitty'}
There are many ways to convert a string to a dictionary, this is only one. If you stumbled in this problem you can quickly check if it's a string instead of a dictionary with:
print(type(var))
In my case I had:
<class 'str'>
before converting it with the above method and then I got
<class 'dict'>
and everything worked as supposed to
I have 2 doit tasks, one having a dependency on the other. For example:
def task_deploy():
return {
'actions': ['do some deploy commands'],
'file_dep': ['dist'],
'params': [{'name': 'projectName',
'short': 'p',
'long': 'projectName',
'default': 'project',
'type': str,
'help': 'The project name to deploy.'}]
}
def task_create_distibution_archive():
return {
'actions': ['do something that requires projectName'],
'doc': 'Creates a zip archive of the application in "dist"',
'targets': ['dist']
}
Is there a way to share or pass the arguments of a task to another one? I have read pretty much everything I could on task creation and dependency on pydoit.org, but haven't found anything similar to what I want.
I am aware that I could use yield to create these two tasks at the same time, but I'd like to use a parameter when executing the task, not when I am creating it.
Is there a way to share or pass the arguments of a task to another one?
Yes. Using getargs: http://pydoit.org/dependencies.html#getargs
In your example, you would need to add another action to the task deploy just to save the passed parameter.
You could just use a global variable like commonCommand. If you have more complex needs, create a class to handle it.
class ComplexCommonParams(object):
def __init__(self):
self.command = 'echo'
params = ComplexCommonParams()
commonCommand='echo'
def task_x():
global commonCommand
return {
'actions': [ commonCommand + ' Hello2 > asdf' ],
'targets': ['asdf']
}
def task_y():
global commonCommand
return {
'actions': [ commonCommand+' World' ],
'file_dep': ['asdf'],
'verbosity':2}
For my project, I need to connect to multiple databases and get information from them. I didn't think this would be a problem with web2py, but it was. I thought maybe I need to rebuild the db from scratch, but still had problems. Finally, I went through the introductory 'images' tutorial and changed it to use an alternate mysql database. I still got the same errors, below is the code:
db.py
db = DAL("mysql://root:#localhost/web2py")
images_db = DAL("mysql://root:#localhost/images_test")
images_db.define_table('image',
Field('title', unique=True),
Field('file', 'upload'),
format = '%(title)s')
images_db.define_table('comment',
Field('image_id', images_db.image),
Field('author'),
Field('email'),
Field('body', 'text'))
Then I went to the admin page for 'images' and clicked the 'shell' link under 'controllers' and did the following: (after I went to the index page to generate the 'images':
Shell:
In [1] : print db(images_db.image).select()
Traceback (most recent call last):
File "/home/cody/Downloads/web2py/gluon/contrib/shell.py", line 233, in run
exec compiled in statement_module.__dict__
File "<string>", line 1, in <module>
File "/home/cody/Downloads/web2py/gluon/dal.py", line 7577, in select
fields = adapter.expand_all(fields, adapter.tables(self.query))
File "/home/cody/Downloads/web2py/gluon/dal.py", line 1172, in expand_all
for field in self.db[table]:
File "/home/cody/Downloads/web2py/gluon/dal.py", line 6337, in __getitem__
return dict.__getitem__(self, str(key))
KeyError: 'image'
In [2] : print images_db.has_key('image')
True
In [3] : print images_db
<DAL {'_migrate_enabled': True, '_lastsql': "SET sql_mode='NO_BACKSLASH_ESCAPES';", '_db_codec': 'UTF-8', '_timings': [('SET FOREIGN_KEY_CHECKS=1;', 0.00017380714416503906), ("SET sql_mode='NO_BACKSLASH_ESCAPES';", 0.00016808509826660156)], '_fake_migrate': False, '_dbname': 'mysql', '_request_tenant': 'request_tenant', '_adapter': <gluon.dal.MySQLAdapter object at 0x2b84750>, '_tables': ['image', 'comment'], '_pending_references': {}, '_fake_migrate_all': False, 'check_reserved': None, '_uri': 'mysql://root:#localhost/images_test', 'comment': <Table {'body': <gluon.dal.Field object at 0x2b844d0>, 'ALL': <gluon.dal.SQLALL object at 0x2b84090>, '_fields': ['id', 'image_id', 'author', 'email', 'body'], '_sequence_name': 'comment_sequence', '_plural': 'Comments', 'author': <gluon.dal.Field object at 0x2b84e10>, '_referenced_by': [], '_format': None, '_db': <DAL {...}>, '_dbt': 'applications/images/databases/e1e448013737cddc822e303fe20f8bec_comment.table', 'email': <gluon.dal.Field object at 0x2b84490>, '_trigger_name': 'comment_sequence', 'image_id': <gluon.dal.Field object at 0x2b84050>, '_actual': True, '_singular': 'Comment', '_tablename': 'comment', '_common_filter': None, 'virtualfields': [], '_id': <gluon.dal.Field object at 0x2b84110>, 'id': <gluon.dal.Field object at 0x2b84110>, '_loggername': 'applications/images/databases/sql.log'}>, 'image': <Table {'ALL': <gluon.dal.SQLALL object at 0x2b84850>, '_fields': ['id', 'title', 'file'], '_sequence_name': 'image_sequence', 'file': <gluon.dal.Field object at 0x2b847d0>, '_plural': 'Images', 'title': <gluon.dal.Field object at 0x2b84610>, '_referenced_by': [('comment', 'image_id')], '_format': '%(title)s', '_db': <DAL {...}>, '_dbt': 'applications/images/databases/e1e448013737cddc822e303fe20f8bec_image.table', '_trigger_name': 'image_sequence', '_loggername': 'applications/images/databases/sql.log', '_actual': True, '_tablename': 'image', '_common_filter': None, 'virtualfields': [], '_id': <gluon.dal.Field object at 0x2b848d0>, 'id': <gluon.dal.Field object at 0x2b848d0>, '_singular': 'Image'}>, '_referee_name': '%(table)s', '_migrate': True, '_pool_size': 0, '_common_fields': [], '_uri_hash': 'e1e448013737cddc822e303fe20f8bec'}>
Now I don't quite understand why I am getting errors here, everything appears to be in order. I thought web2py supported multiple databases? Am I doing it wrong? The appadmin works fine, perhaps I'll edit it and get it to raise an error with the code it's generating... any help would be appreciated.
Cody
UPDATE:
I just tried this:
MODELS/DB.PY
db = DAL("mysql://root:#localhost/web2py")
images_db = DAL("mysql://root:#localhost/images_test")
images_db.define_table('image',
Field('title', unique=True),
Field('file', 'upload'),
format = '%(title)s')
images_db.define_table('comment',
Field('image_id', images_db.image),
Field('author'),
Field('email'),
Field('body', 'text'))
CONTROLLERS/DEFAULT.PY
def index():
"""
example action using the internationalization operator T and flash
rendered by views/default/index.html or views/generic.html
"""
if images_db.has_key('image'):
rows = db(images_db.image).select()
else:
rows = 'nope'
#rows = dir(images_db)
return dict(rows=rows)
VIEWS/DEFAULT/INDEX.HTML
{{left_sidebar_enabled,right_sidebar_enabled=False,True}}
{{extend 'layout.html'}}
these are the rows:
{{=rows }}
Again, very confused by all of this. Appreciate any help.
If you want to query the images_db connection, you have to call images_db(), not db(). So, it would be:
images_db(images_db.image).select()