I want to be able to output a value of a group_var variable for a host in my ansible inventory from a Python script. I can do this with an Ansible ad hoc command:
$ ansible all -i inventory/myInventory.yml -l,3.xx.7xx.175 -m debug -a'var=vertype'
3.xx.xx.175 | SUCCESS => {
"vertype": "rt"
}
But if I try this in Python I can't even find the "vertype" variable.
from ansible.inventory.manager import InventoryManager
from ansible.parsing.dataloader import DataLoader
import glob
from ansible.vars.manager import VariableManager
...
dl = DataLoader()
inventory_files = glob.glob('inventory/*yml')
im = InventoryManager(loader=dl, sources=inventory_files)
vm = VariableManager(loader=dl, inventory=im)
...
my_host = im.get_host(public_ip_address)
print(vm.get_vars(host=my_host).get('vertype'))
But vm.get_vars(host=my_host) does not seem to contain the key "vertype", so I get "None" as output. How can I access the value "rt"?
Related
Objective : To Create UnitTestCases for main.py
How can we import another python file which will dynamically return result
File : main.py
import os
_ENV = os.popen("echo ${RPM_ENVIRONMENT}").read().split('\n')[0]
_HOST = os.popen("echo $(hostname)").read().split('\n')[0]
_JOB_TYPE=os.popen("echo ${JOB_TYPE}").read().split('\n')[0]
SERVER_URL = {
'DEV':{'ENV_URL':'https://dev.net'},
'UAT':{'ENV_URL':'https://uat.net'},
'PROD':{'ENV_URL':'https://prod.net'}
}[_ENV]
Import another python file to our testing script
when i import main.py , i will receive error on SERVER_URL = { KeyError '${RPM_ENVIRONMENT}'
I believe the only reason why its returning error is because it does not recognize RPM_ENVIRONMENT, how can we mock _ENV variables in our main file and print server url as https://dev.net
After i have succesfully import main.py in my test case file, then i will create my unitTest cases as the ENV variable is required for me.
Testing : test_main.py
import unittest, sys
from unittest import mock
# --> Error
sys.path.insert(1, 'C:/home/src')
import main.py as conf
# For an example : we should be able to print https://dev.net when we include this in our line
URL = conf.SERVER_URL.get('ENV_URL')
ENV = conf._ENV
#URL should return https://dev.net & ENV should return DEV
class test_tbrp_case(unittest.TestCase):
def test_port(self):
#function yet to be created
pass
if __name__=='__main__':
unittest.main()
There's little reason to shell out of Python. You can read an environment variable with os.environ. os.environ['RPM_ENVIRONMENT'].
import os
_ENV = os.environ['RPM_ENVIRONMENT']
SERVER_URL = {
'DEV':{'ENV_URL':'https://dev.net'},
'UAT':{'ENV_URL':'https://uat.net'},
'PROD':{'ENV_URL':'https://prod.net'}
}[_ENV]
Now your test can set RPM_ENVIRONMENT before importing main.py.
os.environ['RPM_ENVIRONMENT'] = 'UAT'
sys.path.insert(1, 'C:/home/src')
import main.py as conf
Attempting to write a pipeline [in Apache Beam (Python)] to BigQuery using Jupyter Notebook (local), but seems like its failing because the following import is unsuccessful.
[File ~\AppData\Local\Programs\Python\Python310\lib\site-packages\apache_beam\io\gcp\bigquery_tools.py]
try:
from apache_beam.io.gcp.internal.clients.bigquery import DatasetReference
from apache_beam.io.gcp.internal.clients.bigquery import TableReference
except ImportError:
DatasetReference = None
TableReference = None
Actual Error thrown:
--> 240 if isinstance(table, TableReference):
241 return TableReference(
242 projectId=table.projectId,
243 datasetId=table.datasetId,
244 tableId=table.tableId)
245 elif callable(table):
TypeError: isinstance() arg 2 must be a type, a tuple of types, or a union
I wasn't able to find the 'TableReference' module under apache_beam.io.gcp.internal.clients.bigquery
Screenshot of the windows explorer folder location
These are the libraries I have imported..
!pip install google-cloud
!pip install google-cloud-pubsub
!pip install google-cloud-bigquery
!pip install apache-beam[gcp]
I read in an article that I need to import apache-beam[gcp] instead of apache-beam (which I was doing earlier).
Note! The code runs perfectly fine in Google Colabs.
Here is the complete code:
from google.cloud.bigquery import table
import argparse
import json
import os
import time
import logging
import pandas as pd
import apache_beam as beam
from google.cloud import bigquery, pubsub_v1
from apache_beam.options.pipeline_options import PipelineOptions, StandardOptions, GoogleCloudOptions, SetupOptions
import apache_beam.transforms.window as window
PROJECT_ID = "commanding-bee-322221"
BIGQUERY_DATASET = "Sample_Dataset"
BIGQUERY_MESSAGE_TABLE = "Message_Table"
SUBSCRIPTION_ID = "My-First-Test-Topic-Sub"
BQ_CLIENT = bigquery.Client(project=PROJECT_ID)
BQ_DATASET = BQ_CLIENT.dataset(BIGQUERY_DATASET)
BQ_MSG_TABLE = BQ_DATASET.table(BIGQUERY_MESSAGE_TABLE)
logging.basicConfig(level=logging.INFO)
logging.getLogger().setLevel(logging.INFO)
pipeline_options = {
'project': PROJECT_ID,
# 'runner': 'DataflowRunner',
'region': 'us-east1',
'staging_location': 'gs://my-test-bucket/tmp/',
'temp_location': 'gs://my-test-bucket/tmp/',
# 'template_location': 'gs://my-test-bucket/tmp/My_Template',
'save_main_session':False,
'streaming': True
}
pipeline_options = PipelineOptions.from_dictionary(pipeline_options)
p=beam.Pipeline(options=pipeline_options)
table_schema='MessageType:STRING,Location:STRING,Building:STRING,DateTime:DATETIME,ID:STRING,TID:STRING'
table= f'{PROJECT_ID}:{BIGQUERY_DATASET}.{BIGQUERY_MESSAGE_TABLE}'
class create_formatted_message(beam.DoFn):
def process(self,record):
message_str = record.decode("utf-8")
json_message = eval(message_str)
print(json_message)
# Construct the record
rows_to_insert = [{u"MessageType": u"{}".format(json_message["MessageType"]),
u"Location": u"{}".format(json_message["Location"]),
u"Building": u"{}".format(json_message["Building"]),
u"DateTime": u"{}".format(json_message["DateTime"]),
u"ID": u"{}".format(json_message["ID"]),
u"TID": u"{}".format(json_message["TID"])}]
return rows_to_insert
print(table)
processPubSubMessage = (
p
| "ReadFromPubSub" >> beam.io.gcp.pubsub.ReadFromPubSub(subscription=f'projects/{PROJECT_ID}/subscriptions/{SUBSCRIPTION_ID}', timestamp_attribute=None)
| "Create Formatted Message" >> beam.ParDo(create_formatted_message())
| "Write TO BQ" >> beam.io.WriteToBigQuery(
table,
schema=table_schema,
write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
custom_gcs_temp_location='gs://my-test-bucket/tmp/'
)
)
result = p.run()
# result.wait_until_finish()
time.sleep(20)
result.cancel()
token = "lkjaskdl1jsa53dlksq34ajdsadasd99qqq" # you would need the source code to change this
value = os.getenv(token)
When I release my program using getenv(), users won't be able to see the sourcecode to be able to change the token "value". Using message input, how could I write code so they can enter their token? (instead of them changing it in the sourcecode)
Use dotenv instead.
https://pypi.org/project/python-dotenv/
example:
$ pip install "python-dotenv[cli]"
$ dotenv set USER=foo
$ dotenv set EMAIL=foo#example.org
$ dotenv list
USER=foo
EMAIL=foo#example.org
$ dotenv run -- python foo.py
Set token in the same way as shown here for email and user
Or...
import os
my_var = str(input("Enter your token"))
os.environ['key']=my_var
x = os.environ['key']
print(x)
~
I'm trying to Ansible's Python API in order to write a test API (in Python) which can take advantage of a playbook programmatically and add new nodes to a Hadoop cluster. As we know, at least node in the cluster has to be the Namenode and JobTracker (MRv1). For simplicity lets say the JobTracker and the Namenode are in the same node (namenode_ip).
Thus, in order to use Ansible to create a new node, and have it self registered with the Namenode I've created this following Python utility:
from ansible.playbook import PlayBook
from ansible.inventory import Inventory
from ansible.inventory import Group
from ansible.inventory import Host
from ansible import constants as C
from ansible import callbacks
from ansible import utils
import os
import logging
import config
def run_playbook(ipaddress, namenode_ip, playbook, keyfile):
utils.VERBOSITY = 0
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
stats = callbacks.AggregateStats()
runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
host = Host(name=ipaddress)
group = Group(name="new-nodes")
group.add_host(host)
inventory = Inventory(host_list=[], vault_password="Hello123")
inventory.add_group(group)
key_file = keyfile
playbook_path = os.path.join(config.ANSIBLE_PLAYBOOKS, playbook)
pb = PlayBook(
playbook=playbook_path,
inventory = inventory,
remote_user='deploy',
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats,
private_key_file=key_file
)
results = pb.run()
print results
However, Ansible documentation for the Python API is very poorly written (doesn't give any detail, except for a simple example). What I needed was to have a similar thing as:
ansible-playbook -i hadoop_config -e "namenode_ip=1.2.3.4"
That's it, pass the value of namenode_ip dynamically to the inventory file using the Python API. How can I do that?
This should be as simple as adding one or more of these lines to your script after instantiating your group object and before running your playbook:
group.set_variable("foo", "BAR")
I'm trying to use the database configuration set on settings files to make a database dump using fabric.
There's more than one settings file, so I'd like to be able to do so based on the environment I choose.
by now, my task is like this
def dump_database():
with cd('~/project_folder'), prefix(WORKON_VIRTUALENV):
django.settings_module(env.settings)
from django.conf import settings
dbname = settings.DATABASES['default']['NAME']
dbuser = settings.DATABASES['default']['USER']
dbpassword = settings.DATABASES['default']['PASSWORD']
fname = '/tmp/{0}-backup-{1}.sql.gz'.format(
dbname,
time.strftime('%Y%m%d%H%M%S')
)
run('mysqldump -u %s -p=%s %s | gzip -9 /tmp/backup-%s.sql.gz' % (
dbuser,
dbpassword,
dbname,
fname))
But I'm getting an ImportError:
ImportError: Could not import settings 'project.settings.production'
I've tried to use shell_env() to set the DJANGO_SETTINGS_MODULE instead of django.settings_module(env.settings), with the same result.
I use a task to change the environment based on a environment dict:
def environment(name):
env.update(environments[name])
env.environment = name
This way, I want to be able to create a dump from multiple hosts like:
fab environment:live dump_database
fab environment:otherhost dump_database
Without having to reproduce database settings from all hosts on fabfile.
Importing your Django settings file in fabric is explained here.
http://fabric.readthedocs.org/en/1.3.3/api/contrib/django.html
Quoting from the above link:
from fabric.api import run
from fabric.contrib import django
django.settings_module('myproject.settings')
from django.conf import settings
def dump_production_database():
run('mysqldump -u %s -p=%s %s > /tmp/prod-db.sql' % (
settings.DATABASE_USER,
settings.DATABASE_PASSWORD,
settings.DATABASE_NAME
))
NOTE: I don't answer the question but offer a different solution
I had the same problem.. so I did custom .py script like that:
I created a file named dump_db.py (placed next to fabfile.py for example, that is on the remote machine)
import os
import sys
from datetime import datetime
from django.conf import settings
def dump_mysql():
os.environ.setdefault("DJANGO_SETTINGS_MODULE", SETTINGS_MODULE)
DB_NAME = settings.DATABASES['default']['NAME']
DB_USER = settings.DATABASES['default']['USER']
DB_PASSWORD = settings.DATABASES['default']['PASSWORD']
dump_file_name = '{time}_{db_name}.sql'.format(
time=datetime.now().strftime('%Y_%m_%d'),
db_name=DB_NAME,
)
os.system('mysqldump -u {db_user} -p{db_password} {db_name} > {to_file}'.format(
db_user=DB_USER,
db_password=DB_PASSWORD,
db_name=DB_NAME,
to_file=dump_file_name,
))
return dump_file_name
if __name__ == '__main__':
try:
SETTINGS_MODULE = sys.argv[1:].pop()
except IndexError:
SETTINGS_MODULE = 'project_name.settings'
print dump_mysql()
As you see sys.argv[1:].pop() tries to take optional argument (the setting module in this case).
So in my fabfile:
import os
from fabric.api import env, local, run, prefix, cd
.....
def dump():
current_dir = os.getcwd()
with prefix('source {}bin/activate'.format(env.venv)), cd('{}'.format(env.home)):
dumped_file = run('python dump_db.py {}'.format(env.environment)) # the optional argument given
file_path = os.path.join(env.home, dumped_file)
copy_to = os.path.join(current_dir, dumped_file)
scp(file_path, copy_to)
def scp(file_path, copy_to):
local('scp {}:{} {}'.format(env.host, file_path, copy_to))
where env.environment = 'project_name.settings.env_module'
And this is how I dump my DB and copy it back to me.
Hope it comes handy to someone! :)