Develop Reference

Ansible Python - Get vars including group vars and host vars

I'm trying to get vars of specified host including his host/group vars but I can't get the vars inside group var file
For example:
from ansible.parsing.dataloader import DataLoader
from ansible.vars.manager import VariableManager
from ansible.inventory.manager import InventoryManager
from ansible.vars.manager import Host
class AnsibleConnector:
def __init__(self):
self.ansible_inventory_file_path = "inventory.yml"
self.loader = DataLoader()
self.inventory = InventoryManager(self.loader, sources=self.ansible_inventory_file_path)
self.variable_manager = VariableManager(loader=self.loader, inventory=self.inventory)
def get_host_vars(self, host, include_group_vars=False, groups=None):
if groups is None:
groups = []
group_vars = {}
if include_group_vars:
for group in groups:
group_vars.update(self.get_group_vars(group))
if isinstance(host, str):
host = Host(name=host)
# combine 3 dict in 1 with priority of host_vars (as ansible does)
host_vars = {**group_vars, **self.variable_manager.get_vars(host=host, include_hostvars=True), **self.inventory.get_host(host.name).vars}
return host_vars
def get_group_vars(self, group):
return self.inventory.groups[group].get_vars()
if __name__ == "__main__":
ansible_connector = AnsibleConnector()
ansible_connector.get_host_vars('hpserver01', True, ['hp-servers'])
The code above gives me the host var of hpserver01 and gives me the group vars that are only inside my inventory.ini file but I can't get vars that are in groups_vars/hp-servers.yml
How can I get those vars?
I've searched in many threads, but I couldn't find any reference for how to get the group vars file.
I tried using get_group_vars from ansible.helpers but it didn't help...

Access STEP Instance ID's with PythonOCC

Let's suppose I'm using this STEP file data as input:
#417=ADVANCED_FACE('face_1',(#112),#405,.F.);
#418=ADVANCED_FACE('face_2',(#113),#406,.F.);
#419=ADVANCED_FACE('face_3',(#114),#407,.F.);
I'm using pythonocc-core to read the STEP file.
Then the following code will print the names of the ADVANCED_FACE instances (face_1,face_2 and face_3):
from OCC.Core.STEPControl import STEPControl_Reader
from OCC.Core.TopExp import TopExp_Explorer
from OCC.Core.TopAbs import TopAbs_FACE
from OCC.Core.StepRepr import StepRepr_RepresentationItem
reader = STEPControl_Reader()
tr = reader.WS().TransferReader()
reader.ReadFile('model.stp')
reader.TransferRoots()
shape = reader.OneShape()
exp = TopExp_Explorer(shape, TopAbs_FACE)
while exp.More():
s = exp.Current()
exp.Next()
item = tr.EntityFromShapeResult(s, 1)
item = StepRepr_RepresentationItem.DownCast(item)
name = item.Name().ToCString()
print(name)
How can I access the identifiers of the individual shapes? (#417,#418 and #419)
Minimal reproduction
https://github.com/flolu/step-occ-instance-ids

Create a STEP model after reader.TransferRoots() like this:
model = reader.StepModel()
And access the ID like this in the loop:
id = model.IdentLabel(item)
The full code looks like this and can also be found on GitHub:
from OCC.Core.STEPControl import STEPControl_Reader
from OCC.Core.TopExp import TopExp_Explorer
from OCC.Core.TopAbs import TopAbs_FACE
from OCC.Core.StepRepr import StepRepr_RepresentationItem
reader = STEPControl_Reader()
tr = reader.WS().TransferReader()
reader.ReadFile('model.stp')
reader.TransferRoots()
model = reader.StepModel()
shape = reader.OneShape()
exp = TopExp_Explorer(shape, TopAbs_FACE)
while exp.More():
s = exp.Current()
exp.Next()
item = tr.EntityFromShapeResult(s, 1)
item = StepRepr_RepresentationItem.DownCast(item)
label = item.Name().ToCString()
id = model.IdentLabel(item)
print('label', label)
print('id', id)
Thanks to temurka1 for pointing this out!

I was unable to run your code due to issues installing the pythonocc module, however, I suspect that you should be able to inspect the StepRep_RepresentationItem object (prior to string conversion) by traversing __dict__ on it to discover/access whatever attributes/properties/methods of the object you may need:
entity = tr.EntityFromShapeResult(s, 1)
item = StepRepr_RepresentationItem.DownCast(entity)
print(entity.__dict__)
print(item.__dict__)
If necessary the inspect module exists to pry deeper into the object.
References
https://docs.python.org/3/library/stdtypes.html#object.__dict__
https://docs.python.org/3/library/inspect.html
https://github.com/tpaviot/pythonocc-core/blob/66d6e1ef6b7552a1110a90e86a1ed34eb12ecf16/src/SWIG_files/wrapper/StepElement.pyi

Getting name is not defined error, but it is defined

I used the exec() function and the for loop to create various class objects and store them in variables.
I think the variables were created successfully as shown in the vs code log, but when I try to access it to change the value or print it, I get this error (name 'membro_1' is not defined) as if it's not defined
What did i do wrong here? how can i access those variables? Is there another better way to create all these objects?
see the variables defined at left
the code from image
class Membros:
def __init__(self, tag, pontos=0) -> None:
self.tag = tag
self.pontos = pontos
# war_cla_member have a len() of 47
war_cla_members = ['#VYQPR', '#82PP2LL20', '#LP0RVCPLV', '#LUVQQ2G2', '#PRP20LUL', '#8GUY0V92R',
'#Y02UVP0UV', '#9U0J8GVJL', '#9P2GGVR9Y', '#20QRJLVU8', '#QRYPRQGP9', '#8GRVUG8', '#PCJYUP2L8', '#22VJPRQRL', '#RJQ8JQ8QR', '#2CVR9C2U9', '#PG2UGPJP', '#L0QG9CG2U', '#9R0PR9Q0U', '#2G8VGQ208', '#8GJ8PGY0C', '#9QLLPJQ90', '#C9PGG8JC', '#8YG8RJV90', '#9YLLQLJGU', '#2GQQ2PU92', '#2PYU080Q', '#22QCRQCPG', '#C9JRU9U2', '#9JQLPGLJJ', '#8RR8QVR09', '#9QY2CLVJR', '#U0V0G2YY', '#28PR280CJ', '#P2RC2G9CL', '#9QVVY2P8', '#CVUGYPCP', '#9PVYQP080', '#29P2V8GLJ', '#YUJ88YRU', '#2RU0UGCUU', '#Y08LY8GJY', '#9R00QQU20', '#P08UJ920', '#2C00L02RU', '#YYQP9JGVC', '#YLULUC8L']
for idx, val in enumerate(war_cla_members):
exec(f'membro_{idx} = Membros("{val}")')
for idx, _ in enumerate(war_cla_members):
if current_river_race['clan']['participants'][idx]['tag'] in war_cla_members: #current_river_race is the return of the clashroyale api request. (https://developer.clashroyale.com/#/documentation)
membro_1.pontos = current_river_race['clan']['participants'][idx]['decksUsedToday']

How to get the scope of a class in terms of starting line and end line in Python?

I want to check if class c uses a certain module m. I am able to get two things:
- the module's usage in that file(f),
- the starting line number of the classes in file f.
However, in order to know whether a particular class uses the module, I need to know the starting and end line of class c. I don't know how to get the end line of class c.
I tried going through the documentation of ast but could not find any method of finding the scope of a class.
My current code is:
source_code = "car"
source_code_data = pyclbr.readmodule(source_code)
This gives the following in source_code_data variable:
1
As you can see in the image, there is lineno but no end lineno to denote the end line of a class.
I expect to get the end line of the class in order to know its scope and finally knowing the usage of the module. Currently, the module has this structure:
Module: vehicles used in file: /Users/aviralsrivastava/dev/generate_uml/inheritance_and_dependencies/car.py at: [('vehicles.Vehicle', 'Vehicle', 5), ('vehicles.Vehicle', 'Vehicle', 20)]
So, with this information, I will be able to know if a module is used in a range: used in a class.
My whole code base is:
import ast
import os
import pyclbr
import subprocess
import sys
from operator import itemgetter
from dependency_collector import ModuleUseCollector
from plot_uml_in_excel import WriteInExcel
class GenerateUML:
def __init__(self):
self.class_dict = {} # it will have a class:children mapping.
def show_class(self, name, class_data):
print(class_data)
self.class_dict[name] = []
self.show_super_classes(name, class_data)
def show_methods(self, class_name, class_data):
methods = []
for name, lineno in sorted(class_data.methods.items(),
key=itemgetter(1)):
# print(' Method: {0} [{1}]'.format(name, lineno))
methods.append(name)
return methods
def show_super_classes(self, name, class_data):
super_class_names = []
for super_class in class_data.super:
if super_class == 'object':
continue
if isinstance(super_class, str):
super_class_names.append(super_class)
else:
super_class_names.append(super_class.name)
for super_class_name in super_class_names:
if self.class_dict.get(super_class_name, None):
self.class_dict[super_class_name].append(name)
else:
self.class_dict[super_class_name] = [name]
# adding all parents for a class in one place for later usage: children
return super_class_names
def get_children(self, name):
if self.class_dict.get(name, None):
return self.class_dict[name]
return []
source_code = "car"
source_code_data = pyclbr.readmodule(source_code)
print(source_code_data)
generate_uml = GenerateUML()
for name, class_data in sorted(source_code_data.items(), key=lambda x: x[1].lineno):
print(
"Class: {}, Methods: {}, Parent(s): {}, File: {}".format(
name,
generate_uml.show_methods(
name, class_data
),
generate_uml.show_super_classes(name, class_data),
class_data.file
)
)
print('-----------------------------------------')
# create a list with all the data
# the frame of the list is: [{}, {}, {},....] where each dict is: {"name": <>, "methods": [], "children": []}
agg_data = []
files = {}
for name, class_data in sorted(source_code_data.items(), key=lambda x: x[1].lineno):
methods = generate_uml.show_methods(name, class_data)
children = generate_uml.get_children(name)
# print(
# "Class: {}, Methods: {}, Child(ren): {}, File: {}".format(
# name,
# methods,
# children,
# class_data.file
# )
# )
agg_data.append(
{
"Class": name,
"Methods": methods,
"Children": children,
"File": class_data.file
}
)
files[class_data.file] = name
print('-----------------------------------------')
# print(agg_data)
for data_index in range(len(agg_data)):
agg_data[data_index]['Dependents'] = None
module = agg_data[data_index]["File"].split('/')[-1].split('.py')[0]
used_in = []
for file_ in files.keys():
if file_ == agg_data[data_index]["File"]:
continue
collector = ModuleUseCollector(module)
source = open(file_).read()
collector.visit(ast.parse(source))
print('Module: {} used in file: {} at: {}'.format(
module, file_, collector.used_at))
if len(collector.used_at):
used_in.append(files[file_])
agg_data[data_index]['Dependents'] = used_in
'''
# checking the dependencies
dependencies = []
for data_index in range(len(agg_data)):
collector = ModuleUseCollector(
agg_data[data_index]['File'].split('/')[-1].split('.py')[0]
)
collector.visit(ast.parse(source))
dependencies.append(collector.used_at)
agg_data[source_code_index]['Dependencies'] = dependencies
# next thing, for each class, find the dependency in each class.
'''
print('------------------------------------------------------------------')
print('FINAL')
for data in agg_data:
print(data)
print('-----------')
print('\n')
# The whole data is now collected and we need to form the dataframe of it:
'''
write_in_excel = WriteInExcel(file_name='dependency_1.xlsx')
df = write_in_excel.create_pandas_dataframe(agg_data)
write_in_excel.write_df_to_excel(df, 'class_to_child_and_dependents')
'''
'''
print(generate_uml.class_dict)
write_in_excel = WriteInExcel(
classes=generate_uml.class_dict, file_name='{}.xlsx'.format(source_code))
write_in_excel.form_uml_sheet_for_classes()
'''

Edit: Invasive Method
I did manage to find a way to do this with pyclbr but it involves changing parts of the source code. Essentially you make the stack (which is normally a list) a custom class. When an item is removed from the stack (when its scope has ended) the ending line number is added. I tried to make it as un invasive as possible.
First define a stack class in the pyclbr module:
class Stack(list):
def __init__(self):
import inspect
super().__init__()
def __delitem__(self, key):
frames = inspect.stack()
setattr(self[key][0], 'endline', frames[1].frame.f_locals['start'][0] - 1)
super().__delitem__(key)
Then you change the stack in the _create_tree function which is originally on line 193:
stack = Stack()
Now you can access the ending line as well by using
class_data.endline
I was not able to find a solution that uses the libraries you are using. So I wrote a simple parser that finds the start and end lines of every class contained in a module. You must pass in either the text wrapper itself or the list created by readlines() on your file. Its also worth noting that if a class continues until the end of the file then the end is -1.
import re
def collect_classes(source):
classes = {}
current_class = None
for lineno, line in enumerate(source, start=1):
if current_class and not line.startswith(' '):
if line != '\n':
classes[current_class]['end'] = lineno - 1
current_class = None
if line.startswith('class'):
current_class = re.search(r'class (.+?)(?:\(|:)', line).group(1)
classes[current_class] = {'start': lineno}
if current_class:
classes[current_class]['end'] = -1
return classes
import datetime
file = open(datetime.__file__, 'r') # opening the source of datetime for a test
scopes = collect_classes(file)
print(scopes)
This outputs:
{'timedelta': {'start': 454, 'end': 768}, 'date': {'start': 774, 'end': 1084}, 'tzinfo': {'start': 1092, 'end': 1159}, 'time': {'start': 1162, 'end': 1502}, 'datetime': {'start': 1509, 'end': 2119}, 'timezone': {'start': 2136, 'end': 2251}}

Thanks! The endline is really a missing feature there.
The discussion on this led to an alternative (still invasive) method to achieve this but that is almost a one-liner that does not require to subclass list and override the del operator:
https://github.com/python/cpython/pull/16466#issuecomment-539664180
Basically, just add
stack[-1].endline = start - 1
before all del stack[-1] in pyclbr.py

python cassandra get big result of select * in generator (without storage result in ram)

I want to get all data in cassandra table "user"
i have 840000 users and i don't want to get all users in python list.
i want get users in packs of 100 users
in cassandra doc https://datastax.github.io/python-driver/query_paging.html
i see i can use fetch_size, but in my python code i have database object that contains all cql instruction
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement
class Database:
def __init__(self, name, salary):
self.cluster = Cluster(['192.168.1.1', '192.168.1.2'])
self.session = cluster.connect()
def get_users(self):
users_list = []
query = "SELECT * FROM users"
statement = SimpleStatement(query, fetch_size=10)
for user_row in session.execute(statement):
users_list.append(user_row.name)
return users_list
actually get_users return very big list of user name
but i want to transform return get_users to a "generator"
i don't want get all users name in 1 list and 1 call of function get_users, but i want to have lot of call get_users and return list with only 100 users max every call function
for example :
list1 = database.get_users()
list2 = database.get_users()
...
listn = database.get_users()
list1 contains 100 first user in query
list2 contains 100 "second" users in query
listn contains the latest elements in query (<=100)
is this possible ?
thanks for advance for your answer

According to Paging Large Queries:
Whenever there are no more rows in the current page, the next page
will be fetched transparently.
So, if you execute your code like this, you will still the whole result set, but this is paged in a transparent manner.
In order to achieve what you need to use callbacks. You can also find some code sample on the link above.
I added below the full code for reference.
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement
from threading import Event
class PagedResultHandler(object):
def __init__(self, future):
self.error = None
self.finished_event = Event()
self.future = future
self.future.add_callbacks(
callback=self.handle_page,
errback=self.handle_error)
def handle_page(self, rows):
for row in rows:
process_row(row)
if self.future.has_more_pages:
self.future.start_fetching_next_page()
else:
self.finished_event.set()
def handle_error(self, exc):
self.error = exc
self.finished_event.set()
def process_row(user_row):
print user_row.name, user_row.age, user_row.email
cluster = Cluster()
session = cluster.connect()
query = "SELECT * FROM myschema.users"
statement = SimpleStatement(query, fetch_size=5)
future = session.execute_async(statement)
handler = PagedResultHandler(future)
handler.finished_event.wait()
if handler.error:
raise handler.error
cluster.shutdown()
Moving to next page is done in handle_page when start_fetching_next_page is called.
If you replace the if statement with self.finished_event.set() you will see that the iteration stops after the first 5 rows as defined in fetch_size