Develop Reference

Argo - submit workflow from python with input parameter file

I basically want to run this command: argo submit -n argo workflows/workflow.yaml -f params.json through the official python SDK.
This example covers how to submit a workflow manifest, but I don't know where to add the input parameter file.
import os
from pprint import pprint
import yaml
from pathlib import Path
import argo_workflows
from argo_workflows.api import workflow_service_api
from argo_workflows.model.io_argoproj_workflow_v1alpha1_workflow_create_request import \
IoArgoprojWorkflowV1alpha1WorkflowCreateRequest
configuration = argo_workflows.Configuration(host="https://localhost:2746")
configuration.verify_ssl = False
with open("workflows/workflow.yaml", "r") as f:
manifest = yaml.safe_load(f)
api_client = argo_workflows.ApiClient(configuration)
api_instance = workflow_service_api.WorkflowServiceApi(api_client)
api_response = api_instance.create_workflow(
namespace="argo",
body=IoArgoprojWorkflowV1alpha1WorkflowCreateRequest(workflow=manifest, _check_type=False),
_check_return_type=False)
pprint(api_response)
Where to pass in the params.json file?

I found this snippet in the docs of WorkflowServiceApi.md (which was apparently too big to render as markdown):
import time
import argo_workflows
from argo_workflows.api import workflow_service_api
from argo_workflows.model.grpc_gateway_runtime_error import GrpcGatewayRuntimeError
from argo_workflows.model.io_argoproj_workflow_v1alpha1_workflow_submit_request import IoArgoprojWorkflowV1alpha1WorkflowSubmitRequest
from argo_workflows.model.io_argoproj_workflow_v1alpha1_workflow import IoArgoprojWorkflowV1alpha1Workflow
from pprint import pprint
# Defining the host is optional and defaults to http://localhost:2746
# See configuration.py for a list of all supported configuration parameters.
configuration = argo_workflows.Configuration(
host = "http://localhost:2746"
)
# Enter a context with an instance of the API client
with argo_workflows.ApiClient() as api_client:
# Create an instance of the API class
api_instance = workflow_service_api.WorkflowServiceApi(api_client)
namespace = "namespace_example" # str |
body = IoArgoprojWorkflowV1alpha1WorkflowSubmitRequest(
namespace="namespace_example",
resource_kind="resource_kind_example",
resource_name="resource_name_example",
submit_options=IoArgoprojWorkflowV1alpha1SubmitOpts(
annotations="annotations_example",
dry_run=True,
entry_point="entry_point_example",
generate_name="generate_name_example",
labels="labels_example",
name="name_example",
owner_reference=OwnerReference(
api_version="api_version_example",
block_owner_deletion=True,
controller=True,
kind="kind_example",
name="name_example",
uid="uid_example",
),
parameter_file="parameter_file_example",
parameters=[
"parameters_example",
],
pod_priority_class_name="pod_priority_class_name_example",
priority=1,
server_dry_run=True,
service_account="service_account_example",
),
) # IoArgoprojWorkflowV1alpha1WorkflowSubmitRequest |
# example passing only required values which don't have defaults set
try:
api_response = api_instance.submit_workflow(namespace, body)
pprint(api_response)
except argo_workflows.ApiException as e:
print("Exception when calling WorkflowServiceApi->submit_workflow: %s\n" % e)
Have you tried using a IoArgoprojWorkflowV1alpha1WorkflowSubmitRequest? Looks like it has submit_options of type IoArgoprojWorkflowV1alpha1SubmitOpts which has a parameter_file param.

pysmb from linux to Windows, Unable to connect to shared device

Trying to connect to an smb share via pysmb and getting error...
smb.smb_structs.OperationFailure: Failed to list on \\\\H021BSBD20\\shared_folder: Unable to connect to shared device
The code I am using looks like...
from smb.SMBConnection import SMBConnection
import json
import pprint
import warnings
pp = pprint.PrettyPrinter(indent=4)
PROJECT_HOME = "/path/to/my/project/"
# load configs
CONF = json.load(open(f"{PROJECT_HOME}/configs/configs.json"))
pp.pprint(CONF)
# list all files in storage smb dir
#https://pysmb.readthedocs.io/en/latest/api/smb_SMBConnection.html#smb.SMBConnection.SMBConnection.listPath
IS_DIRECT_TCP = False
CNXN_PORT = 139 if not IS_DIRECT_TCP else 445
LOCAL_IP = "172.18.4.69"
REMOTE_NAME = "H021BSBD20" # exact name shown as Device Name in System Settings
SERVICE_NAME = "\\\\H021BSBD20\\shared_folder"
REMOTE_IP = "172.18.7.102"
try:
conn = SMBConnection(CONF['smb_creds']['username'], CONF['smb_creds']['password'],
my_name=LOCAL_IP, remote_name=REMOTE_NAME,
use_ntlm_v2=True,
is_direct_tcp=IS_DIRECT_TCP)
conn.connect(REMOTE_IP, CNXN_PORT)
except Exception:
warnings.warn("\n\nFailed to initially connect, attempting again with param use_ntlm_v2=False\n\n")
conn = SMBConnection(CONF['smb_creds']['username'], CONF['smb_creds']['password'],
my_name=LOCAL_IP, remote_name=REMOTE_NAME,
use_ntlm_v2=False,
is_direct_tcp=IS_DIRECT_TCP)
conn.connect(REMOTE_IP, CNXN_PORT)
files = conn.listPath(f'{SERVICE_NAME}', '\\')
pp.pprint(files)
Using smbclient on my machine, I can successfully connect to the share by doing...
[root#airflowetl etl]# smbclient -U my_user \\\\H021BSBD20\\shared_folder
The amount of backslashes I use in the python code is so that I can create the same string that works when using this smbclient (have tried with less backslashes in the code and that has not helped).
Note that the user that I am using the access the shared folder in the python code and with smbclient is not able to access / log on to the actual machine that the share is hosted on (they are only allowed to access that particular shared folder as shown above).
Does anyone know what could be happening here? Any other debugging steps that could be done?

After asking on the github repo Issues section (https://github.com/miketeo/pysmb/issues/169), I was able to fix the problem. It was just due to the arg I was using for the conn.listPath() servicename param.
When looking closer at the docs for that function (https://pysmb.readthedocs.io/en/latest/api/smb_SMBConnection.html), I saw...
service_name (string/unicode) – the name of the shared folder for the path
Originally, I was only looking at the function signature, which said service_name, so I assumed it would be the same as with the smbclient command-line tool (which I have been entering the servicename param as \\\\devicename\\sharename (unlike with pysmb which we can see from the docstring wants just the share as the service_name)).
So rather than
files = conn.listPath("\\\\H021BSBD20\\shared_folder", '\\')
I do
files = conn.listPath("shared_folder", '\\')
The full refactored snippet is shown below, just for reference.
import argparse
import json
import os
import pprint
import socket
import sys
import traceback
import warnings
from smb.SMBConnection import SMBConnection
def parseArguments():
# Create argument parser
parser = argparse.ArgumentParser()
# Positional mandatory arguments
parser.add_argument("project_home", help="project home path", type=str)
parser.add_argument("device_name", help="device (eg. NetBIOS) name in configs of share to process", type=str)
# Optional arguments
# parser.add_argument("-dfd", "--data_file_dir",
# help="path to data files dir to be pushed to sink, else source columns based on form_type",
# type=str, default=None)
# Parse arguments
args = parser.parse_args()
return args
args = parseArguments()
for a in args.__dict__:
print(str(a) + ": " + str(args.__dict__[a]))
pp = pprint.PrettyPrinter(indent=4)
PROJECT_HOME = args.project_home
REMOTE_NAME = args.device_name
# load configs
CONF = json.load(open(f"{PROJECT_HOME}/configs/configs.json"))
CREDS = json.load(open(f"{PROJECT_HOME}/configs/creds.json"))
pp.pprint(CONF)
SMB_CONFS = next(info for info in CONF["smb_server_configs"] if info["device_name"] == args.device_name)
print("\nUsing details for device:")
pp.pprint(SMB_CONFS)
# list all files in storage smb dir
#https://pysmb.readthedocs.io/en/latest/api/smb_SMBConnection.html#smb.SMBConnection.SMBConnection.listPath
IS_DIRECT_TCP = False
CNXN_PORT = 139 if IS_DIRECT_TCP is False else 445
LOCAL_IP = socket.gethostname() #"172.18.4.69"
REMOTE_NAME = SMB_CONFS["device_name"]
SHARE_FOLDER = SMB_CONFS["share_folder"]
REMOTE_IP = socket.gethostbyname(REMOTE_NAME) # "172.18.7.102"
print(LOCAL_IP)
print(REMOTE_NAME)
try:
conn = SMBConnection(CREDS['smb_creds']['username'], CREDS['smb_creds']['password'],
my_name=LOCAL_IP, remote_name=REMOTE_NAME,
use_ntlm_v2=False,
is_direct_tcp=IS_DIRECT_TCP)
conn.connect(REMOTE_IP, CNXN_PORT)
except Exception:
traceback.print_exc()
warnings.warn("\n\nFailed to initially connect, attempting again with param use_ntlm_v2=True\n\n")
conn = SMBConnection(CREDS['smb_creds']['username'], CREDS['smb_creds']['password'],
my_name=LOCAL_IP, remote_name=REMOTE_NAME,
use_ntlm_v2=True,
is_direct_tcp=IS_DIRECT_TCP)
conn.connect(REMOTE_IP, CNXN_PORT)
files = conn.listPath(SHARE_FOLDER, '\\')
if len(files) > 0:
print("Found listed files")
for f in files:
print(f.filename)
else:
print("No files to list, this likely indicates a problem. Exiting...")
exit(255)

How to extract Deployed OSB Source code from Environment or SB Console or Weblogic

Could anyone please help me in getting a way to get the source code from Environment or SB Console or Weblogic.
I created the python script whick exports the JAR, but I need the source code. Because if I unjar the jar, I do not get the exact source code, as file names are shorten and some code is added by itself in wsdls, xqueries etc. I don't want that.
Here's my wlst Python/Jython Script:
from java.io import FileInputStream
from java.io import FileOutputStream
from java.util import ArrayList
from java.util import Collections
from com.bea.wli.sb.util import EnvValueTypes
from com.bea.wli.config.env import EnvValueQuery;
from com.bea.wli.config import Ref
from com.bea.wli.config.customization import Customization
from com.bea.wli.config.customization import FindAndReplaceCustomization
import sys
#=======================================================================================
# Utility function to load properties from a config file
#=======================================================================================
def exportAll(exportConfigFile):
def exportAll(exportConfigFile):
try:
print "Loading export config from :", exportConfigFile
exportConfigProp = loadProps(exportConfigFile)
adminUrl = exportConfigProp.get("adminUrl")
exportUser = exportConfigProp.get("exportUser")
exportPasswd = exportConfigProp.get("exportPassword")
exportJar = exportConfigProp.get("exportJar")
customFile = exportConfigProp.get("customizationFile")
passphrase = exportConfigProp.get("passphrase")
project = sys.argv[2]
if project == None :
project = exportConfigProp.get("project")
connectToServer(exportUser, exportPasswd, adminUrl)
ALSBConfigurationMBean = findService("ALSBConfiguration", "com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
print "ALSBConfiguration MBean found"
print "Input project: ", project
if project == None :
ref = Ref.DOMAIN
collection = Collections.singleton(ref)
if passphrase == None :
print "Export the config"
theBytes = ALSBConfigurationMBean.exportProjects(collection, None)
else :
print "Export and encrypt the config"
theBytes = ALSBConfigurationMBean.export(collection, true, passphrase)
else :
ref = Ref.makeProjectRef(project);
print "Export the project", project
collection = Collections.singleton(ref)
theBytes = ALSBConfigurationMBean.export(collection, false, None)
aFile = File(exportJar)
out = FileOutputStream(aFile)
out.write(theBytes)
out.close()
print "ALSB Configuration file: "+ exportJar + " has been exported"
if customFile != None:
print collection
query = EnvValueQuery(None, Collections.singleton(EnvValueTypes.WORK_MANAGER), collection, false, None, false)
customEnv = FindAndReplaceCustomization('Set the right Work Manager', query, 'Production System Work Manager')
print 'EnvValueCustomization created'
customList = ArrayList()
customList.add(customEnv)
print customList
aFile = File(customFile)
out = FileOutputStream(aFile)
Customization.toXML(customList, out)
out.close()
print "ALSB Dummy Customization file: "+ customFile + " has been created"
except:
raise
#=======================================================================================
# Utility function to load properties from a config file
#=======================================================================================
def loadProps(configPropFile):
propInputStream = FileInputStream(configPropFile)
configProps = Properties()
configProps.load(propInputStream)
return configProps
#=======================================================================================
# Connect to the Admin Server
#=======================================================================================
def connectToServer(username, password, url):
connect(username, password, url)
domainRuntime()
# EXPORT script init
try:
exportAll(sys.argv[1])
except:
print "Unexpected error: ", sys.exc_info()[0]
dumpStack()
raise
Any help would be appreciated.

What you get as a result of the export is the deployed unit. Yes, there is some metadata added/modified as a result of the deployment on the OSB runtime (deployment could also mean creating/editing components directly on the servicebus console).
To get it back as "source code" from the exported jar, you can simply import it back into JDeveloper (12c) or Eclipse with OEPE (11g)

Argparse - configparser.Interpolation missing option error

I am using Argparse for fetching the necessary value from config file.
For example:
python arg.py --event_conf=/opt/open-stack-tools/track_events.conf --openstack_conf=/etc/nova/nova.conf
I need to fetch value from two different files.
I can be able to get the results as needed for one local config file.
But In case of fetching the necessary values from nova.conf file, it results in following error:
Traceback (most recent call last):
File "arg.py", line 36, in <module>
oslo_messaging_rabbit= dict(config.items("oslo_messaging_rabbit"))
File "/usr/lib/python2.7/ConfigParser.py", line 655, in items
for option in options]
File "/usr/lib/python2.7/ConfigParser.py", line 691, in _interpolate
self._interpolate_some(option, L, rawval, section, vars, 1)
File "/usr/lib/python2.7/ConfigParser.py", line 723, in _interpolate_some
option, section, rest, var)
ConfigParser.InterpolationMissingOptionError: Bad value substitution:
section: [oslo_messaging_rabbit]
option : logging_exception_prefix
key : color
rawval : %(asctime)s.%(msecs)03d TRACE %(name)s %(instance)s
Is there any way to fix the same.
I have copied the necessary contents and created a new local file, I can see that it is working fine.
When I am using the nova.conf file it results in error.
I can't change the file which I am using.
So I need a fix for the particular error.
Note:
Adding more details as needed:
parser.add_argument("-c", "--event_conf",
help="Specify config file 1", metavar="FILE")
args1, remaining_argv1 = parser.parse_known_args()
parser.add_argument("-o", "--openstack_conf",
help="Specify config file 2", metavar="FILE")
args2, remaining_argv2 = parser.parse_known_args()
if args1.event_conf:
config = ConfigParser.SafeConfigParser()
print config.read([args1.event_conf])
config.read([args1.event_conf])
configdetails_section1 = dict(config.items("configdetails_section1"))

I found the solution for the same.
Actually issue was with the configparser which I have used.
Instead of SafeConfigParser I changed it to RawConfigParser.
Then I can be able to see that it is working fine.

Interpolation allows you to refer to values of the configuration while defining others. For example, in this config file:
[bug_tracker]
protocol = http
server = localhost
port = 8080
url = %(protocol)s://%(server)s:%(port)s/bugs/
username = dhellmann
password = SECRET
if you read like this:
from ConfigParser import SafeConfigParser
parser = SafeConfigParser()
parser.read('interpolation.ini')
print 'value =', parser.get('bug_tracker', 'url')
you will get:
value = http://localhost:8080/bugs/
that can be very useful, the problem seems that that you pass some values in run time. I mean you need the format string to actually substitute the values by yourself.
You can use RawConfigParser instead of SafeConfigParser but then you loose all the interpolations.
Instead you can suppress the interpolation for one specific value:
print 'value =', parser.get('bug_tracker', 'url', raw=True)
and the result would be:
value = %(protocol)s://%(server)s:%(port)s/bugs/
There is also the possibility that you need to combine interpolated values with some of them given in evaluation time. For example if you want to give the user in evaluation time, you can also include it into the config expression:
[bug_tracker]
protocol = http
server = localhost
port = 8080
url = %(protocol)s://%(user)s#%(server)s:%(port)s/bugs/
username = dhellmann
password = SECRET
and then you need to make something like this:
from ConfigParser import SafeConfigParser
parser = SafeConfigParser()
parser.read('interpolation.ini')
parser.set('bug_tracker', 'user', 'admin')
print 'value =', parser.get('bug_tracker', 'url')
and you'll get:
value = http://admin#localhost:8080/bugs/
Sorry that I didn't use your example. I did take the one in the documentation of another project. See the section: Combining Values with Interpolation

running python script through shell_plus from command line

I have a python script that I run to populate my database. I usually run the script inside shell_plus because of the dependencies required. Is there a way to load the script into shell_plus and run everything from my linux command line without actually opening the shell_plus interface?

"Standalone Django scripts"

You bet!
I don't even recommend using shell_plus. I tend to store my utilities scripts in my app utility folder. Then I simply call them from a cron job or manually as needed. Here is framework script I base this off of. (Somewhat simplified)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import time
import time
import optparse
# DO NOT IMPORT DJANGO MODELS HERE - THIS NEED TO HAPPEN BELOW!!
# This needs to be able to be run when django isn't in the picture (cron) so we need
# to be able to add in the django paths when needed.
def getArgs():
"""
Simply get the options for running update people.
"""
p = optparse.OptionParser()
help = "The Python path to a settings module, e.g. "
help += "\"myproject.settings.main\". If this isn't provided, the "
help += "DJANGO_SETTINGS_MODULE environment variable will be used."
p.add_option("-s", "--settings", action = "store", type = "string",
dest = "settings", help = help)
help = "A directory to add to the Python path, e.g."
help += " \"/home/djangoprojects/myproject\"."
p.add_option("-y", "--pythonpath", action = "store", type = "string",
dest = "pythonpath", help = help)
p.add_option("-v", "--verbose", action = "count", dest = "verbose",
help = "Turn on verbose debugging")
p.set_defaults(settings = os.environ.get("DJANGO_SETTINGS_MODULE",
"settings"),
pythonpath = "", verbose = 0,
)
return p.parse_args()
def update(opt, loglevel=None):
"""
This is the main script used for updating people
"""
start = time.time()
# This ensures that our sys.path is ready.
for path in opt.pythonpath.split(":"):
if os.path.abspath(path) not in sys.path and os.path.isdir(path):
sys.path.append(os.path.abspath(path))
os.environ['DJANGO_SETTINGS_MODULE'] = opt.settings
from django.conf import settings
try:
if settings.SITE_ROOT not in sys.path: pass
except ImportError:
return("Your setting file cannot be imported - not in sys.path??")
# IMPORT YOUR CODE MODELS HERE
from apps.core.utility.ExampleExtractor import ExampleExtractor
# YOUR DJANGO STUFF GOES HERE..
example = ExampleExtractor(loglevel=loglevel, singleton=not(opt.multiple))
raw = example.get_raw()
results = example.update_django(raw)
log.info("Time to update %s entries : %s" % (len(results), time.time() - start))
return results
if __name__ == '__main__':
logging.basicConfig(format = "%(asctime)s %(levelname)-8s %(module)s \
%(funcName)s %(message)s", datefmt = "%H:%M:%S", stream = sys.stderr)
log = logging.getLogger("")
log.setLevel(logging.DEBUG)
opts, args = getArgs()
sys.exit(update(opts))
HTH!