Develop Reference

Unable to run KubernetesPodOperator

I am using airflow 2.4.3 and running KubernetesPodOperator
Below is the code and error:-
Please help me with creating a KubernetesPosOperator in python. I have tried on both GCP and Azure.
Also adding the kubernetes documentation for reference:-
https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/_api/airflow/providers/cncf/kubernetes/operators/kubernetes_pod/index.html#airflow.providers.cncf.kubernetes.operators.kubernetes_pod.KubernetesPodOperator
I can also share any other info if required.
from kubernetes.client import models as k8s
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
#custom modules
from spark_API.spark_submit import SparkSubmit
#import modules
import json
import datetime
import logging
import airflow_API
import time
airflow_dag_object = AirflowDagUtilities("aaa_test_airflow_api")
def def_func1(**kwargs):
print("In func1")
namespace = "segmentation-pipeline"
docker_image = "****:v6" # name commented out
is_delete_operator_pod = True
docker_image_creds = [k8s.V1LocalObjectReference("****")] # harbor name commented out
submit_command = ["/bin/bash","-c"]
max_cores = 60
driver_memory = "4g"
executor_memory = "4g"
submit_args = "/usr/local/spark/bin/spark-submit --master local[" + str(max_cores) + "] --driver-memory " + \
driver_memory + " --executor-memory " + executor_memory + " "
submit_spark_pipeline_config_conf = "--conf " + '\'' + 'spark.pipelineConfig' + "=" + json.dumps(_infra_config.get_infra_config(),separators=(',',':')) + '\'' + " "
submit_spark_broadcast_timeout = "--conf " + '\"' + "spark.sql.broadcastTimeout" + "=" + str("36000") + '\"' + " "
submit_spark_max_result_size = "--conf " + '\"' + "spark.driver.maxResultSize" + "=" + str("0") + '\"' + " "
final_dependency_jars = ["./resources/mysql_connector_java_5.1.45.jar",\
"./resources/commons_httpclient_3.0.1.jar"]
dependency_jars_string = ','.join(list(set(final_dependency_jars)))
submit_spark_dependency_jars = "--conf " + '\"' + "spark.jars" + "=" + dependency_jars_string + '\"' + " "
extra_conf = []
extra_conf_final = []
for conf in extra_conf:
conf_appended_string = "--conf " + '\"' + conf + '\'' + " "
extra_conf_final.append(conf_appended_string)
extra_conf = " ".join(extra_conf_final) + " "
airflow_task_settings = airflow_API.extract_airflow_task_details(kwargs['task_instance'])
submit_spark_airflow_task_details = "--conf " + '\"' + "spark.airflowTaskDetails" + "=" + json.dumps(airflow_task_settings) + '\'' + " "
common_submit_args_beginning = submit_args + submit_spark_broadcast_timeout + submit_spark_max_result_size + submit_spark_dependency_jars + extra_conf + submit_spark_airflow_task_details
application_resource = "/update_scores.py"
application_arguments = ["test_args"]
string_application_arguments = " "
for i in range(0,len(application_arguments)):
string_application_arguments = string_application_arguments + " " + json.dumps(application_arguments[i])
common_submit_args_end = application_resource + string_application_arguments
platform_utilities = PlatformUtilities(_infra_config)
print("platform_utilities.get_python_modules_path() -> ",str(platform_utilities.get_python_modules_path()))
submit_spark_python_module_path = "--conf " + '\"' + "spark.modulePath" + "=" + str(platform_utilities.get_python_modules_path()) + '\"' + " "
submit_spark_args = [common_submit_args_beginning + submit_spark_pipeline_config_conf + submit_spark_python_module_path + common_submit_args_end]
print("submit_spark_args -> ",submit_spark_args)
submit_in_cluster = True
submit_spark_pod_affinity = k8s.V1Affinity(
node_affinity=k8s.V1NodeAffinity(k8s.V1NodeSelectorTerm(
match_expressions=[
k8s.V1NodeSelectorRequirement(key="****", operator="In", values=["n2-highmem-8"]),
k8s.V1NodeSelectorRequirement(key="deployment", operator="In", values=["dynamic"]),
]
)
)
)
submit_spark_pod_tolerations = [k8s.V1Toleration(key="deployment", operator="Equal", value="dynamic", effect="NoSchedule")]
application_name = "test_airflow_api_test_task_id"
container_resources = k8s.V1ResourceRequirements(
requests={
'memory': str("10Gi"),
'cpu': str("2")
},
limits={
'memory': str("50Gi"),
'cpu': str("5")
}
)
submit_startup_timeout_seconds = 600
submit_get_logs = True
kube_submssion = KubernetesPodOperator(namespace = namespace,
image = docker_image,
is_delete_operator_pod = is_delete_operator_pod,
image_pull_secrets = docker_image_creds,
cmds = submit_command,
arguments = submit_spark_args,
in_cluster = submit_in_cluster,
affinity = submit_spark_pod_affinity,
tolerations = submit_spark_pod_tolerations,
container_resources = container_resources,
name = application_name,
task_id = application_name,
startup_timeout_seconds = submit_startup_timeout_seconds,
get_logs = submit_get_logs
)
kube_submssion.execute(context = None)
def def_func2(**kwargs):
print("In func2")
dag_base = airflow_dag_object.get_dag_object()
func1=PythonOperator(
task_id='func1',
provide_context=True,
python_callable=def_func1,
dag=dag_base
)
func2=PythonOperator(
task_id='func2',
provide_context=True,
python_callable=def_func2,
dag=dag_base
)
func1 >> func2
OUTPUT ERROR:-
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py", line 419, in execute
context=context,
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py", line 387, in get_or_create_pod
pod = self.find_pod(self.namespace or pod_request_obj.metadata.namespace, context=context)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py", line 371, in find_pod
label_selector=label_selector,
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api/core_v1_api.py", line 15697, in list_namespaced_pod
return self.list_namespaced_pod_with_http_info(namespace, **kwargs) # noqa: E501
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api/core_v1_api.py", line 15826, in list_namespaced_pod_with_http_info
collection_formats=collection_formats)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api_client.py", line 353, in call_api
_preload_content, _request_timeout, _host)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api_client.py", line 184, in __call_api
_request_timeout=_request_timeout)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api_client.py", line 377, in request
headers=headers)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/rest.py", line 244, in GET
query_params=query_params)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/rest.py", line 234, in request
raise ApiException(http_resp=r)
kubernetes.client.exceptions.ApiException: (400)
Reason: Bad Request
HTTP response headers: HTTPHeaderDict({'Audit-Id': '6ab39ea1-f955-4481-b3eb-7b3abe747a7c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': '8e487991-120d-49d0-940a-ace0b0e64421', 'X-Kubernetes-Pf-Prioritylevel-Uid': '8f6ab0b3-abdf-4782-994c-2f0f247592d2', 'Date': 'Thu, 12 Jan 2023 13:13:20 GMT', 'Content-Length': '169'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"found ',', expected: !, identifier, or 'end of string'","reason":"BadRequest","code":400}

In previous version of airflow < 2.3 KubernetesPodOperator used to work with None context
As mentioned in your question
kube_submssion = KubernetesPodOperator(namespace = namespace,
image = docker_image,
is_delete_operator_pod = is_delete_operator_pod,
image_pull_secrets = docker_image_creds,
cmds = submit_command,
arguments = submit_spark_args,
in_cluster = submit_in_cluster,
affinity = submit_spark_pod_affinity,
tolerations = submit_spark_pod_tolerations,
container_resources = container_resources,
name = application_name,
task_id = application_name,
startup_timeout_seconds = submit_startup_timeout_seconds,
get_logs = submit_get_logs
)
kube_submssion.execute(context = None)
The execute method is expecting the context as mentioned in the documentation at followig link
https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/_modules/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.html#KubernetesPodOperator.execute
You can pass the context from **kwargs to the execute method. You can try by passing kwargs to execute method
kube_submssion.execute(context = kwargs)

how do I find a way to use ssh and rsync

I have troubles using to implement ssh and rsync including a private key in python, including Popen (subprocess).
Basically the rsync syntax to use should be:
$ rsync -az -e --log-file=$logdir/logfile.out \
'ssh -e /home/user/.ssh/id_rsa' user#server:/target-directory
What I have is this:
import subprocess
First, I build my logdir path - with variables :
logdir = [basedir + '/' + 'log' + '/' + today + '/' + 'pfdcopy_' \
+ typ + '_' + ts + '.log.txt']
Then I build the target directory:
target= ['jboss' + '#' + targetsvr + ':' + /data']
Finally, I try to run this code
p1 = subprocess.Popen(['rsync', '-az', '--log-file=%s' % \
logdir/logfile.out , '-e', 'ssh', '-i', \
'/home/user/.ssh/id_rsa', target])
It's quite complex, I know, mainly because of the variables, and the quotation marks.
Running this, I get always different syntax errors with p1.
Any help is highly appreciated. Thanks!
edited (08-10-2018):
here is my complete runnable code snippet -
from datetime import datetime
import subprocess
import os
import fnmatch
now = datetime.now()
today = now.strftime("%Y-%m-%d")
ts = now.strftime("%Y-%m-%d-%H-%M-%S")
sign_output_dir = '/Users/fanta4/Documents/python-files/outgoing'
mandator = 'BTV'
formsize = 'A4'
basedir = '/Users/fanta4/Documents'
pdf_to_send = []
targetsvr = 'nas1'
doktyp = (
'WPGEBUEHR', 'WPDURCHFU', 'WPABR', 'WPABRKF', 'WPABRVK', 'WPABRTILG', 'WPABRERTR', 'WPAMIS', 'WPSTREP',
'WPABLAUF', 'WPAVISO', 'WPAUSZUG', 'WPERTRAEG', 'WPSIKTEST', 'WPTRANS', 'WPANSCHAFF', 'KKKONTOMIT', 'KRKURSUEW',
'WPVERLUSTA', 'WPVERLUSTG')
os.chdir(sign_output_dir)
for file in os.listdir(sign_output_dir):
if fnmatch.fnmatch(file, '*.pdf'):
pdf_to_send.append(file)
os.chdir(sign_output_dir)
print('debug: doktyp ist: {}'.format(formsize))
for typ in doktyp:
if typ in str(pdf_to_send):
ts = now.strftime("%Y-%m-%d-%Hh-%Mm-%Ss")
print('typ: {:12s} exists -> will be transfered to nas1'.format(typ))
logdir = [basedir + '/' + 'log' + '/' + mandator + '/' + today + '/' + 'pfdcopy_' + typ + '_' + ts + '.log.txt']
target = ['jboss' + '#' + targetsvr + '/data' + '/' + mandator + typ]
p1 = subprocess.Popen(
['rsync', '-az', '--log-file=%s' % logdir, '-e', 'ssh', '-i', '/Users/fanta4/.ssh/id_rsa', typ, '-l', target])
p1.returncode
if p1 > 0:
print('debug: Error with rsync of typ: {} to target: {}'.format(typ, targetsvr))
else:
print('debug: rsync mandator: {:3s} with typ: {:12s} succeeded'.format(mandator, typ))
else:
print('debug: typ: {:12s} does not exist'.format(typ))
logfile = ['/data' + '/' + 'log' + '/' + mandator + '/' + ts]
print('debug: pls see logfile in: {}'.format(logfile))
If I run this code, I get:
/Users/fanta4/anaconda3/bin/python "/Users/fanta4/Library/Mobile Documents/com~apple~CloudDocs/entw/python/prog/rsync-test.py"
Traceback (most recent call last):
/Users/fanta4/Documents/python-files/outgoing
debug: doktyp ist: A4
File "/Users/fanta4/Library/Mobile
Documents/com~apple~CloudDocs/entw/python/prog/rsync-test.py", line 37, in <module>
typ: WPGEBUEHR exists -> will be transfered to nas1
['rsync', '-az', '--log-file=%s' % logdir, '-e', 'ssh', '-i', '/Users/fanta4/.ssh/id_rsa', typ, '-l', target])
File "/Users/fanta4/anaconda3/lib/python3.6/subprocess.py", line 709, in __init__
restore_signals, start_new_session)
File "/Users/fanta4/anaconda3/lib/python3.6/subprocess.py", line 1275, in _execute_child
restore_signals, start_new_session, preexec_fn)
TypeError: expected str, bytes or os.PathLike object, not list
Process finished with exit code 1

Your issue is exemplified by the following lines (when used to generate items which are later used as elements inside an argument vector):
logdir = [basedir + '/' + 'log' + '/' + mandator + '/' + today + '/' + 'pfdcopy_' + typ + '_' + ts + '.log.txt']
target = ['jboss' + '#' + targetsvr + '/data' + '/' + mandator + typ]
You're defining logdir and target as lists (with only one string inside them), whereas they need to be strings.
Just take out the square brackets that create a list, and you'll have strings instead:
logdir = basedir + '/log/' + mandator + '/' + today + '/pfdcopy_' + typ + '_' + ts + '.log.txt'
target = 'jboss#' + targetsvr + '/data/' + mandator + typ

You haven't mentioned what syntax errors you are getting. It would really for the benefit of everyone for you to include that information. I am guessing it's missing the missing quotes around one the string parameters.
p1 = subprocess.Popen([
'rsync', '-az', '--log-file=%s' % 'logdir/logfile.out',
'-e', 'ssh', '-i',
'/home/user/.ssh/id_rsa', target
])

syntax error for writing a variable in a script

dump.pbd='pdb' + pdbFile + '_' + 'res' + residMin + '_' residMax + '.pdb'
the program keep giving me syntax error when I run it.
import re
import sys
import os
import time
from sys import argv
import xmltodict
if len(sys.argv) < 3:
message = '\n Get protein file in the form of pdf file from pdb website. \n\n Usage: '+sys.argv[0] + ' [4-letter PDBid] [resid range] \n' + ' Example: ' + sys.argv[0] + ' 2rh1 53-71\n' + ' Output File: pdb2rh1_res53-71.pdb'
print (message)
exit()
pdbID=sys.argv[1]
residRange=sys.argv[2]
residData=residRange.split('-')
residMin=int(residData[0])
residMax=int(residData[1])
twoletter=pdbID[1:3]
xmlfile=pdbID + '.xml'
pdbgzfile=pdbID + '.pdb.gz'
pdbFile=pdbID+'.pdb'
dump.pbd='pdb' + pdbFile + '_' + 'res' + residMin + '_' residMax + '.pdb'
wgetcom='wget https://files.rcsb.org/view/'+pdbFile+' -O '+pdbFile
print(wgetcom)
os.system(wgetcom)
f = open (pdbFile,'r')
k = 0
rc = 0
data = f.readlines()
g = open (dump.pdb, 'w')
for linedata in data:
line=linedata.strip()
words = line.split()
if(words[0] == 'ATOM'):
k=k+1
words[5]=int(line[22:26].strip())
if(words[5] in range(residMin,residMax+1)):
g.write(linedata)
for i in words:
if(i=='CA'):
rc = rc+1
print(rc)
the code is not working because it is giving me a syntax error for line number 22 that states dump.pbd='pdb' + pdbFile + '' + 'res' + residMin + '' residMax + '.pdb'. so can you please help me with that?
Thanks so much on advance!

You've forgotten to add a + sign.
This line should work: dump.pbd='pdb' + pdbFile + '' + 'res' + residMin + '' + residMax + '.pdb'
There must be a + sign between '_' and residMax as this is the Python concatenating strings syntax.

Converting date and time from shell script into python

I am a beginner to python and I have this date and time code from shell script, and how do I make it into python, also how can I take date from last month till now
ToDay=`date "+%Y%m%d"`
CheckDATE=`date "+%Y-%m-%d" --date '1 day ago'`
NOWDATE=`date "+%Y-%m-%d"`
This is the python code
from datetime import datetime, date, timedelta
progToRun = 'python ' + ScriptDir + '/bin/panafapi.py -K ' + secretkey + ' --samples -j -r "{\\"query\\":{\\"operator\\":\\"all\\",\\"children\\":[{\\"field\\":\\"alias.ip_address\\",\\"operator\\":\\"contains\\",\\"value\\":\\"' + ResultFile + '\\"},{\\"operator\\":\\"any\\",\\"children\\":[{\\"field\\":\\"sample.update_date\\",\\"operator\\":\\"is in the range\\",\\"value\\":[\\"' + CheckDATE + 'T00:00:00\\",\\"' + NOWDATE + 'T23:59:59\\"]},{\\"field\\":\\"sample.create_date\\",\\"operator\\":\\"is in the range\\",\\"value\\":[\\"' + CheckDATE + 'T00:00:00\\",\\"' + NOWDATE + 'T23:59:59\\"]},{\\"operator\\":\\"any\\",\\"children\\":[{\\"field\\":\\"sample.malware\\",\\"operator\\":\\"is\\",\\"value\\":1},{\\"field\\":\\"sample.malware\\",\\"operator\\":\\"is\\",\\"value\\":4}]}]}]},\\"scope\\":\\"global\\",\\"size\\":1,\\"from\\":0,\\"sort\\":{\\"create_date\\":{\\"order\\":\\"desc\\"}}}" > ' + ResultDir + 'srciplist-' + ToDay + '.json'
ToDay = datetime.now().strftime('%Y%m%d')
CheckDATE = datetime.strptime("2017-12-01", "%Y-%m-%d").date()
NOWDATE = datetime.now().date()
Error:
progToRun = 'python ' + ScriptDir + '/bin/panafapi.py -K ' + secretkey + ' --samples -j -r "{\\"query\\":{\\"operator\\":\\"all\\",\\"children\\":[{\\"field\\":\\"alias.ip_address\\",\\"operator\\":\\"contains\\",\\"value\\":\\"' + ResultFile + '\\"},{\\"operator\\":\\"any\\",\\"children\\":[{\\"field\\":\\"sample.update_date\\",\\"operator\\":\\"is in the range\\",\\"value\\":[\\"' + CheckDATE + 'T00:00:00\\",\\"' + NOWDATE + 'T23:59:59\\"]},{\\"field\\":\\"sample.create_date\\",\\"operator\\":\\"is in the range\\",\\"value\\":[\\"' + CheckDATE + 'T00:00:00\\",\\"' + NOWDATE + 'T23:59:59\\"]},{\\"operator\\":\\"any\\",\\"children\\":[{\\"field\\":\\"sample.malware\\",\\"operator\\":\\"is\\",\\"value\\":1},{\\"field\\":\\"sample.malware\\",\\"operator\\":\\"is\\",\\"value\\":4}]}]}]},\\"scope\\":\\"global\\",\\"size\\":1,\\"from\\":0,\\"sort\\":{\\"create_date\\":{\\"order\\":\\"desc\\"}}}" > ' + ResultDir + 'srciplist-' + ToDay + '.json'
TypeError: must be str, not datetime.date

>>> import datetime
>>> today = datetime.date.today()
>>> yesterday = today - datetime.timedelta(days=1)
>>> ToDay = today.strftime("%Y%m%d")
>>> CheckDATE = yesterday.strftime("%Y-%m-%d")
>>> NOWDATE = today.strftime("%Y-%m-%d")
>>> print ToDay
20180126
>>> print CheckDATE
2018-01-25
>>> print NOWDATE
2018-01-26

You can convert date using the following code:
CheckDATE = datetime.datetime.strptime("2017-12-01", "%Y-%m-%d").date()
ToDAY = datetime.datetime.now().strftime("%Y-%m-%d")
CheckDATE = CheckDATE.strftime("%Y-%m-%d")
Adding date at the end only outputs the date part and omits time.
I hope this answers your question.
Edit: Updated based on additional information.

os.rename returning winerror 2

I'm trying to a script to rename to the date that it was sent as an email(which is the first part of the script but doesn't matter for this part) then to rename, and sort it into a 'Complete' folder. This is what my code looks like
Edit - I have all the imported stuff way up at the top and i didnt show it, but i assume i have the right stuff imported if you would like to see just ask
dir5 = "C:\\Users\\Michael D\\Documents\\Test\\AmLit"
dir6 = "C:\\Users\\Michael D\\Documents\\Test\\History"
dir7 = "C:\\Users\\Michael D\\Documents\\Test\\MultiLit"
dir8 = "C:\\Users\\Michael D\\Documents\\Test\\Physics"
dir5_final = "C:\\Users\\Michael D\\Documents\\TestMove\\AmLit"
dir6_final = "C:\\Users\\Michael D\\Documents\\TestMove\\History"
dir7_final = "C:\\Users\\Michael D\\Documents\\TestMove\\MultiLit"
dir8_final = "C:\\Users\\Michael D\\Documents\\TestMove\\Physics"
now = datetime.datetime.now()
now1 = (str(now.day) + '/' + str(now.month) + '/' + str(now.year))
dir5_files = os.listdir(dir5)
dir6_files = os.listdir(dir6)
dir7_files = os.listdir(dir7)
dir8_files = os.listdir(dir8)
for f in dir5_files:
if (f.startswith("A") or f.startswith("a")):
os.rename(f, now1 + " " + f)
but i keep getting this error
RESTART: C:/Users/Michael D/Documents/Coding/Schoolwork Email/Email Sender Beta 1.7.21.9.16.py
Traceback (most recent call last):
File "C:/Users/Michael D/Documents/Coding/Schoolwork Email/Email Sender Beta 1.7.21.9.16.py", line 148, in <module>
os.rename(f, now1 + " " + f)
FileNotFoundError: [WinError 2] The system cannot find the file specified: 'A Test.txt' -> '21/9/2016 A Test.txt'
any thoughts as to what I'm doing wrong?

2 errors:
You are not in the current directory
You just cannot have slashes in the names. The filesystem won't allow it as it is (alternately) used to separate path parts.
First, generate the date directly with underscores:
now1 = (str(now.day) + '_' + str(now.month) + '_' + str(now.year))
Then replace
os.rename(f, now1 + " " + f)
by
os.rename(os.path.join(dir5,f), os.path.join(dir5,now1.replace("/","_") + " " + f))
and A Test.txt would be renamed to 21_9_2016 A Test.txt in the directory you specified.