Develop Reference

Ansible Python - Get vars including group vars and host vars

I'm trying to get vars of specified host including his host/group vars but I can't get the vars inside group var file
For example:
from ansible.parsing.dataloader import DataLoader
from ansible.vars.manager import VariableManager
from ansible.inventory.manager import InventoryManager
from ansible.vars.manager import Host
class AnsibleConnector:
def __init__(self):
self.ansible_inventory_file_path = "inventory.yml"
self.loader = DataLoader()
self.inventory = InventoryManager(self.loader, sources=self.ansible_inventory_file_path)
self.variable_manager = VariableManager(loader=self.loader, inventory=self.inventory)
def get_host_vars(self, host, include_group_vars=False, groups=None):
if groups is None:
groups = []
group_vars = {}
if include_group_vars:
for group in groups:
group_vars.update(self.get_group_vars(group))
if isinstance(host, str):
host = Host(name=host)
# combine 3 dict in 1 with priority of host_vars (as ansible does)
host_vars = {**group_vars, **self.variable_manager.get_vars(host=host, include_hostvars=True), **self.inventory.get_host(host.name).vars}
return host_vars
def get_group_vars(self, group):
return self.inventory.groups[group].get_vars()
if __name__ == "__main__":
ansible_connector = AnsibleConnector()
ansible_connector.get_host_vars('hpserver01', True, ['hp-servers'])
The code above gives me the host var of hpserver01 and gives me the group vars that are only inside my inventory.ini file but I can't get vars that are in groups_vars/hp-servers.yml
How can I get those vars?
I've searched in many threads, but I couldn't find any reference for how to get the group vars file.
I tried using get_group_vars from ansible.helpers but it didn't help...

Access section key value from another section in config.ini

I have a config.ini file
[env]
name = local
[dir]
proj_home = /apps/${env:name}/project
Python code:
config = configparser.ConfigParser()
config.read('config.ini')
projHome = config.get('dir', 'proj_home')
But projHome value is coming '/apps/${env:name}/project', it should be '/apps/local/project'
Am I doing something wrong in syntax?

That ${section:option} syntax is only available when you're using extended interpolation:
>>> cf = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
>>> cf.add_section('env')
>>> cf.set('env', 'name', 'local')
>>> cf.add_section('dir')
>>> cf.set('dir', 'proj_home', '/apps/${env:name}/project')
>>> cf.get('dir', 'proj_home')
'/apps/local/project'

How to get a list of variables in specific Python module?

Let's assume I have the following file structure:
data.py
foo = []
bar = []
abc = "def"
core.py
import data
# do something here #
# a = ...
print a
# ['foo', 'bar', 'abc']
I need to get all the variables defined in data.py file. How can I achieve that? I could use dir(), but it returns all the attributes of the module including __name__ and so on.

print [item for item in dir(adfix) if not item.startswith("__")]
Is usually the recipe for doing this, but it begs the question.
Why?

#!/usr/local/bin/python
# coding: utf-8
__author__ = 'spouk'
def get_book_variable_module_name(module_name):
module = globals().get(module_name, None)
book = {}
if module:
book = {key: value for key, value in module.__dict__.iteritems() if not (key.startswith('__') or key.startswith('_'))}
return book
import config
book = get_book_variable_module_name('config')
for key, value in book.iteritems():
print "{:<30}{:<100}".format(key, value)
example config
#!/usr/local/bin/python
# coding: utf-8
__author__ = 'spouk'
import os
_basedir = os.path.abspath(os.path.dirname(__file__))
# database section MYSQL section
DBHOST = 'localhost'
DBNAME = 'simple_domain'
DBPORT = 3306
DBUSER = 'root'
DBPASS = 'root'
# global section
DEBUG = True
HOSTNAME = 'simpledomain.com'
HOST = '0.0.0.0'
PORT = 3000
ADMINS = frozenset(['admin#localhost'])
SECRET_KEY = 'dfg45DFcx4rty'
CSRF_ENABLED = True
CSRF_SESSION_KEY = "simplekey"
result function
/usr/local/bin/python2 /home/spouk/develop/python/2015/utils_2015/parse_config_py.py
DBPORT 3306
os <module 'os' from '/usr/local/lib/python2.7/os.pyc'>
DBHOST localhost
HOSTNAME simpledomain.com
HOST 0.0.0.0
DBPASS root
PORT 3000
ADMINS frozenset(['admin#localhost'])
CSRF_SESSION_KEY simplekey
DEBUG 1
DBUSER root
SECRET_KEY dfg45DFcx4rty
CSRF_ENABLED 1
DBNAME simple_domain
Process finished with exit code 0
Enjoy, dude. :)

I have to make a dictionary of these variables. I used this code.
print({item:getattr(my_module, item) for item in dir(my_module) if not item.startswith("__") and not item.endswith("__")})

This is the version I wrote for python 3.7 (it excludes the internal dunder methods via the condition in the comprehension)
print([v for v in dir(data) if v[:2] != "__"])
A longer but complete working example is below:
"""an example of a config file whose variables may be accessed externally"""
# Module variables
server_address = "172.217.167.68"
server_port = 8010
server_to_client_port = 8020
client_to_server_port = 8030
client_buffer_length = 4096
server_buffer_length = 2048
def printVariables(variable_names):
"""Renders variables and their values on the terminal."""
max_name_len = max([len(k) for k in variable_names])
max_val_len = max([len(str(globals()[k])) for k in variable_names])
for k in variable_names:
print(f' {k:<{max_name_len}}: {globals()[k]:>{max_val_len}}')
if __name__ == "__main__":
print(__doc__)
ks = [k for k in dir() if (k[:2] != "__" and not callable(globals()[k]))]
printVariables(ks)
The above code outputs:
an example of a config file whose variables may be accessed externally
client_buffer_length : 4096
client_to_server_port: 8030
server_address : 172.217.167.68
server_buffer_length : 2048
server_port : 8010
server_to_client_port: 8020

I offer my solution. It is convenient in that it allows you to display variables from any imported module.
If you do not specify the name of the module, then the list of variables of the current module is displayed.
import sys
def print_settings(module_name=None):
module_name = sys.modules[__name__] if not module_name else module_name
variables = [
(key, value)
for (key, value) in vars(module_name).items()
if (type(value) == str or type(value) == int or type(value) == float)
and not key.startswith("_")
]
for (key, value) in variables:
print(f"{key: <20} {value}")

If you need the variable and the value assigned to it then
import data
for name ,values in vars(data).items():
print(name, values)
You can choose to store name (all the variable names in the script) or the value attached to it .

Here's a roundabout way, if you prefer to be more explicit:
data.py
a = [
foo := [],
bar := [],
abc := "def",
]
core.py
import data
print(data.foo)
print(data.a)

Try:
for vars in dir():
if vars.startswith("var"):
print vars

Reading array from config file in python

I have a problem. My program is using config file to set options, and one of those options is a tuple. Here's what i mean:
[common]
logfile=log.txt
db_host=localhost
db_user=root
db_pass=password
folder[1]=/home/scorpil
folder[2]=/media/sda5/
folder[3]=/media/sdb5/
etc...
Can i parse this into tuple with ConfigParser module in Python? Is there some easy way to do this?

if you can change config format like this:
folder = /home/scorpil
/media/sda5/
/media/sdb5/
then in python:
config.get("common", "folder").split("\n")

Your config could be:
[common]
logfile=log.txt
db_host=localhost
db_user=root
db_pass=password
folder = ("/home/scorpil", "/media/sda5/", "/media/sdb5/")
Assuming that you have config in a file named foo.cfg, you can do the following:
import ConfigParser
cp = ConfigParser.ConfigParser()
cp.read("foo.cfg")
folder = eval(cp.get("common", "folder"), {}, {})
print folder
print type(folder)
which should produce:
('/home/scorpil', '/media/sda5/', '/media/sdb5/')
<type 'tuple'>
-- EDIT --
I've since changed my mind about this, and would take the position today that using eval in this context is a bad idea. Even with a restricted environment, if the configuration file is under user control it may be a very bad idea. Today I'd probably recommend doing something interesting with split to avoid malicious code execution.

You can get the items list and use a list comprehension to create a list of all the items which name starts with a defined prefix, in your case folder
folders = tuple([ item[1] for item in configparser.items() if item[0].startswith("folder")])

Create configuration:
folders = ['/home/scorpil', '/media/sda5/', '/media/sdb5/']
config.set('common', 'folders', json.dumps(folders))
Load configuration:
tuple(json.loads(config.get('common', 'folders')))

I don't know ConfigParser, but you can easily read it into a list (perhaps using .append()) and then do myTuple = tuple(myList)

#!/usr/bin/env python
sample = """
[common]
logfile=log.txt
db_host=localhost
db_user=root
db_pass=password
folder[1]=/home/scorpil
folder[2]=/media/sda5/
folder[3]=/media/sdb5/
"""
from cStringIO import StringIO
import ConfigParser
import re
FOLDER_MATCH = re.compile(r"folder\[(\d+)\]$").match
def read_list(items,pmatch=FOLDER_MATCH):
if not hasattr(pmatch,"__call__"):
pmatch = re.compile(pmatch).match
folder_list = []
for k,v in items:
m = pmatch(k)
if m:
folder_list.append((int(m.group(1)),v))
return tuple( kv[1] for kv in sorted(folder_list) )
if __name__ == '__main__':
cp = ConfigParser.SafeConfigParser()
cp.readfp(StringIO(sample),"sample")
print read_list(cp.items("common"))

You could stick to json completely
tst.json
{
"common": {
"logfile":"log.txt",
"db_host":"localhost",
"db_user":"root",
"db_pass":"password",
"folder": [
"/home/scorpil",
"/media/sda5/",
"/media/sdb5/"
]
}
}
then work with it
$ python3
>>> import json
>>> with open("tst.json", "r", encoding="utf8") as file_object:
... job = json.load(file_object)
...
>>> job
{'common': {'db_pass': 'password', 'logfile':
'log.txt', 'db_user': 'root', 'folder':
['/home/scorpil', '/media/sda5/', '/media/sdb5/'],
'db_host': 'localhost'}}
>>> print (job["common"]["folder"][0])
/home/scorpil
>>> print (job["common"]["folder"][1])
/media/sda5/
print (job["common"]["folder"][2])
/media/sdb5/
>>> folder_tuple = tuple(job["common"]["folder"])
>>> folder_tuple
('/home/scorpil', '/media/sda5/', '/media/sdb5/')

Convert ConfigParser.items('') to dictionary

How can I convert the result of a ConfigParser.items('section') to a dictionary to format a string like here:
import ConfigParser
config = ConfigParser.ConfigParser()
config.read('conf.ini')
connection_string = ("dbname='%(dbname)s' user='%(dbuser)s' host='%(host)s' "
"password='%(password)s' port='%(port)s'")
print connection_string % config.items('db')

Have you tried
print connection_string % dict(config.items('db'))
?

How I did it in just one line.
my_config_parser_dict = {s:dict(config.items(s)) for s in config.sections()}
No more than other answers but when it is not the real businesses of your method and you need it just in one place use less lines and take the power of dict comprehension could be useful.

This is actually already done for you in config._sections. Example:
$ cat test.ini
[First Section]
var = value
key = item
[Second Section]
othervar = othervalue
otherkey = otheritem
And then:
>>> from ConfigParser import ConfigParser
>>> config = ConfigParser()
>>> config.read('test.ini')
>>> config._sections
{'First Section': {'var': 'value', '__name__': 'First Section', 'key': 'item'}, 'Second Section': {'__name__': 'Second Section', 'otherkey': 'otheritem', 'othervar': 'othervalue'}}
>>> config._sections['First Section']
{'var': 'value', '__name__': 'First Section', 'key': 'item'}
Edit: My solution to the same problem was downvoted so I'll further illustrate how my answer does the same thing without having to pass the section thru dict(), because config._sections is provided by the module for you already.
Example test.ini:
[db]
dbname = testdb
dbuser = test_user
host = localhost
password = abc123
port = 3306
Magic happening:
>>> config.read('test.ini')
['test.ini']
>>> config._sections
{'db': {'dbname': 'testdb', 'host': 'localhost', 'dbuser': 'test_user', '__name__': 'db', 'password': 'abc123', 'port': '3306'}}
>>> connection_string = "dbname='%(dbname)s' user='%(dbuser)s' host='%(host)s' password='%(password)s' port='%(port)s'"
>>> connection_string % config._sections['db']
"dbname='testdb' user='test_user' host='localhost' password='abc123' port='3306'"
So this solution is not wrong, and it actually requires one less step. Thanks for stopping by!

I know this was asked a long time ago and a solution chosen, but the solution selected does not take into account defaults and variable substitution. Since it's the first hit when searching for creating dicts from parsers, thought I'd post my solution which does include default and variable substitutions by using ConfigParser.items().
from ConfigParser import SafeConfigParser
defaults = {'kone': 'oneval', 'ktwo': 'twoval'}
parser = SafeConfigParser(defaults=defaults)
parser.set('section1', 'kone', 'new-val-one')
parser.add_section('section1')
parser.set('section1', 'kone', 'new-val-one')
parser.get('section1', 'ktwo')
parser.add_section('section2')
parser.get('section2', 'kone')
parser.set('section2', 'kthree', 'threeval')
parser.items('section2')
thedict = {}
for section in parser.sections():
thedict[section] = {}
for key, val in parser.items(section):
thedict[section][key] = val
thedict
{'section2': {'ktwo': 'twoval', 'kthree': 'threeval', 'kone': 'oneval'}, 'section1': {'ktwo': 'twoval', 'kone': 'new-val-one'}}
A convenience function to do this might look something like:
def as_dict(config):
"""
Converts a ConfigParser object into a dictionary.
The resulting dictionary has sections as keys which point to a dict of the
sections options as key => value pairs.
"""
the_dict = {}
for section in config.sections():
the_dict[section] = {}
for key, val in config.items(section):
the_dict[section][key] = val
return the_dict

For an individual section, e.g. "general", you can do:
dict(parser['general'])

Another alternative would be:
config.ini
[DEFAULT]
potato=3
[foo]
foor_property=y
potato=4
[bar]
bar_property=y
parser.py
import configparser
from typing import Dict
def to_dict(config: configparser.ConfigParser) -> Dict[str, Dict[str, str]]:
"""
function converts a ConfigParser structure into a nested dict
Each section name is a first level key in the the dict, and the key values of the section
becomes the dict in the second level
{
'section_name': {
'key': 'value'
}
}
:param config: the ConfigParser with the file already loaded
:return: a nested dict
"""
return {section_name: dict(config[section_name]) for section_name in config.sections()}
main.py
import configparser
from parser import to_dict
def main():
config = configparser.ConfigParser()
# By default section names are parsed to lower case, optionxform = str sets to no conversion.
# For more information: https://docs.python.org/3/library/configparser.html#configparser-objects
# config.optionxform = str
config.read('config.ini')
print(f'Config read: {to_dict(config)}')
print(f'Defaults read: {config.defaults()}')
if __name__ == '__main__':
main()

In Python +3.6 you could do this
file.ini
[SECTION1]
one = 1
two = 2
[SECTION2]
foo = Hello
bar = World
[SECTION3]
param1 = parameter one
param2 = parameter two
file.py
import configparser
cfg = configparser.ConfigParser()
cfg.read('file.ini')
# Get one section in a dict
numbers = {k:v for k, v in cfg['SECTION1'].items()}
If you need all sections listed you should use cfg.sections()

Combining Michele d'Amico and Kyle's answer (no dict),
produces a less readable but somehow compelling:
{i: {i[0]: i[1] for i in config.items(i)} for i in config.sections()}

Here is another approach using Python 3.7 with configparser and ast.literal_eval:
game.ini
[assets]
tileset = {0:(32, 446, 48, 48),
1:(96, 446, 16, 48)}
game.py
import configparser
from ast import literal_eval
config = configparser.ConfigParser()
config.read('game.ini')
# convert a string to dict
tileset = literal_eval(config['assets']['tileset'])
print('tileset:', tileset)
print('type(tileset):', type(tileset))
output
tileset: {0: (32, 446, 48, 48), 1: (96, 446, 16, 48)}
type(tileset): <class 'dict'>