I want to remove the website from further checking process, if my "keyword" on the website got found, so they are not getting still checked multiple times.
How can i do this? Iam still a beginner but i linked you my whole Script at the bottom thanks.
If the keyword "google" got found at the currently checking website, i want to remove this website from further checking.
if "google" in r2.text:
print (bcolors.OKGREEN + "Parameters Found : " +server+ "/" + para1 + "/" + para2 + bcolors.ENDC)
client = server + "," + para1 + "," + para2 + "\n"
f = open('log.txt', 'a')
f.write(client)
f.close()
My whole Script
import os
import sys
from threading import Thread, BoundedSemaphore
from datetime import datetime
import optparse
import requests
import urllib3
os.system("color")
requests.urllib3.disable_warnings()
maxConnections = 10
connection_lock = BoundedSemaphore(maxConnections)
time = datetime.now().time()
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
ENDC = '\033[0m'
def connect(server, para1, para2):
try:
r = requests.request('put', server + para1 + para2, timeout=30, verify=False, headers={'Content-Type':'application/octet-stream'})
r.close()
except Exception as e:
print(e)
r2 = requests.request('get', server + para1 + para2, verify=False, timeout=30, headers={'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'})
r2.close()
if "google" in r2.text:
print (bcolors.OKGREEN + "Parameters Found : " +server+ "/" + para1 + "/" + para2 + bcolors.ENDC)
client = server + "," + para1 + "," + para2 + "\n"
f = open('log.txt', 'a')
f.write(client)
f.close()
def generate_tests(hosts, paras1, paras2):
i = 0
for para1 in paras1:
para1 = para1.strip('\n\r')
for para2 in paras2:
para2 = para2.strip('\n\r')
for host in hosts:
server = host.strip('\n\r')
print (bcolors.OKGREEN + "=" * 60 + bcolors.ENDC)
print (bcolors.BOLD + "Website: " + bcolors.OKBLUE + server + para1 + para2 + bcolors.ENDC)
print (bcolors.BOLD + "Parameter1: " + bcolors.OKBLUE + para1 + bcolors.ENDC)
print (bcolors.BOLD + "Parameter2: " + bcolors.OKBLUE + para2 + bcolors.ENDC)
i += 1
print (bcolors.BOLD + "Attempts: " + bcolors.OKBLUE + str(i) + bcolors.ENDC)
print (bcolors.BOLD + "Time Started: " + bcolors.OKBLUE + str(time) + bcolors.ENDC)
print (bcolors.BOLD + "Time now: " + bcolors.OKBLUE + str(datetime.now().time()) + bcolors.ENDC)
print (bcolors.OKGREEN + "=" * 60 + bcolors.ENDC)
t = Thread(target=connect, args=(server, para1, para2))
t.start()
def read_test_files(hostsfile, paras1file, paras2file):
hosts = open(hostsfile, 'r').readlines()
paras1 = open(paras1file, 'r').readlines()
paras2 = open(paras2file, 'r').readlines()
generate_tests(hosts, paras1, paras2)
def main():
parser = optparse.OptionParser('usage python test.py -H <hosts file> -U <para1 file> -P <para2 file>')
parser.add_option('-H', dest='hostsfile', help="specify host file to test")
parser.add_option('-U', dest='paras1file', help="specify possible parameters1")
parser.add_option('-P', dest='paras2file', help="specify possible parameters2")
(options, args) = parser.parse_args()
if options.hostsfile and options.paras1file and options.paras2file:
hostsfile = options.hostsfile
paras1file = options.paras1file
paras2file = options.paras2file
read_test_files(hostsfile, paras1file, paras2file)
else:
print (parser.usage)
exit(0)
if __name__ == "__main__":
main()
You can create a collection of whitelists, which will be used to save websites that pass the check, and determine whether they are in the whitelist before starting the thread each time, and skip if they exist.
In the code below, the whitelist collection is temporary.
If you want it to take effect every time you start the program, you can save it to a file, read the file every time you start the program, and write it every time it ends.
import os
import sys
from threading import Thread, BoundedSemaphore
from datetime import datetime
import optparse
import requests
import urllib3
os.system("color")
requests.urllib3.disable_warnings()
maxConnections = 10
connection_lock = BoundedSemaphore(maxConnections)
time = datetime.now().time()
white_list = set()
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
ENDC = '\033[0m'
def connect(server, para1, para2):
try:
r = requests.request('put', server + para1 + para2, timeout=30, verify=False,
headers={'Content-Type': 'application/octet-stream'})
r.close()
except Exception as e:
print(e)
r2 = requests.request('get', server + para1 + para2, verify=False, timeout=30, headers={
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'})
r2.close()
if "google" in r2.text:
# add
white_list.add(server)
print(bcolors.OKGREEN + "Parameters Found : " + server + "/" + para1 + "/" + para2 + bcolors.ENDC)
client = server + "," + para1 + "," + para2 + "\n"
f = open('log.txt', 'a')
f.write(client)
f.close()
def generate_tests(hosts, paras1, paras2):
i = 0
for para1 in paras1:
para1 = para1.strip('\n\r')
for para2 in paras2:
para2 = para2.strip('\n\r')
for host in hosts:
server = host.strip('\n\r')
# Determine whether it is in the whitelist
# The `host` in the whitelist has already been subjected to the `strip('\n\r')` operation.
if server in white_list:
continue
print(bcolors.OKGREEN + "=" * 60 + bcolors.ENDC)
print(bcolors.BOLD + "Website: " + bcolors.OKBLUE + server + para1 + para2 + bcolors.ENDC)
print(bcolors.BOLD + "Parameter1: " + bcolors.OKBLUE + para1 + bcolors.ENDC)
print(bcolors.BOLD + "Parameter2: " + bcolors.OKBLUE + para2 + bcolors.ENDC)
i += 1
print(bcolors.BOLD + "Attempts: " + bcolors.OKBLUE + str(i) + bcolors.ENDC)
print(bcolors.BOLD + "Time Started: " + bcolors.OKBLUE + str(time) + bcolors.ENDC)
print(bcolors.BOLD + "Time now: " + bcolors.OKBLUE + str(datetime.now().time()) + bcolors.ENDC)
print(bcolors.OKGREEN + "=" * 60 + bcolors.ENDC)
t = Thread(target=connect, args=(server, para1, para2))
t.start()
t.join()
def read_test_files(hostsfile, paras1file, paras2file):
hosts = open(hostsfile, 'r').readlines()
paras1 = open(paras1file, 'r').readlines()
paras2 = open(paras2file, 'r').readlines()
generate_tests(hosts, paras1, paras2)
def main():
parser = optparse.OptionParser('usage python test.py -H <hosts file> -U <para1 file> -P <para2 file>')
parser.add_option('-H', dest='hostsfile', help="specify host file to test")
parser.add_option('-U', dest='paras1file', help="specify possible parameters1")
parser.add_option('-P', dest='paras2file', help="specify possible parameters2")
(options, args) = parser.parse_args()
if options.hostsfile and options.paras1file and options.paras2file:
hostsfile = options.hostsfile
paras1file = options.paras1file
paras2file = options.paras2file
read_test_files(hostsfile, paras1file, paras2file)
else:
print(parser.usage)
exit(0)
if __name__ == "__main__":
main()
Related
I am using airflow 2.4.3 and running KubernetesPodOperator
Below is the code and error:-
Please help me with creating a KubernetesPosOperator in python. I have tried on both GCP and Azure.
Also adding the kubernetes documentation for reference:-
https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/_api/airflow/providers/cncf/kubernetes/operators/kubernetes_pod/index.html#airflow.providers.cncf.kubernetes.operators.kubernetes_pod.KubernetesPodOperator
I can also share any other info if required.
from kubernetes.client import models as k8s
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
#custom modules
from spark_API.spark_submit import SparkSubmit
#import modules
import json
import datetime
import logging
import airflow_API
import time
airflow_dag_object = AirflowDagUtilities("aaa_test_airflow_api")
def def_func1(**kwargs):
print("In func1")
namespace = "segmentation-pipeline"
docker_image = "****:v6" # name commented out
is_delete_operator_pod = True
docker_image_creds = [k8s.V1LocalObjectReference("****")] # harbor name commented out
submit_command = ["/bin/bash","-c"]
max_cores = 60
driver_memory = "4g"
executor_memory = "4g"
submit_args = "/usr/local/spark/bin/spark-submit --master local[" + str(max_cores) + "] --driver-memory " + \
driver_memory + " --executor-memory " + executor_memory + " "
submit_spark_pipeline_config_conf = "--conf " + '\'' + 'spark.pipelineConfig' + "=" + json.dumps(_infra_config.get_infra_config(),separators=(',',':')) + '\'' + " "
submit_spark_broadcast_timeout = "--conf " + '\"' + "spark.sql.broadcastTimeout" + "=" + str("36000") + '\"' + " "
submit_spark_max_result_size = "--conf " + '\"' + "spark.driver.maxResultSize" + "=" + str("0") + '\"' + " "
final_dependency_jars = ["./resources/mysql_connector_java_5.1.45.jar",\
"./resources/commons_httpclient_3.0.1.jar"]
dependency_jars_string = ','.join(list(set(final_dependency_jars)))
submit_spark_dependency_jars = "--conf " + '\"' + "spark.jars" + "=" + dependency_jars_string + '\"' + " "
extra_conf = []
extra_conf_final = []
for conf in extra_conf:
conf_appended_string = "--conf " + '\"' + conf + '\'' + " "
extra_conf_final.append(conf_appended_string)
extra_conf = " ".join(extra_conf_final) + " "
airflow_task_settings = airflow_API.extract_airflow_task_details(kwargs['task_instance'])
submit_spark_airflow_task_details = "--conf " + '\"' + "spark.airflowTaskDetails" + "=" + json.dumps(airflow_task_settings) + '\'' + " "
common_submit_args_beginning = submit_args + submit_spark_broadcast_timeout + submit_spark_max_result_size + submit_spark_dependency_jars + extra_conf + submit_spark_airflow_task_details
application_resource = "/update_scores.py"
application_arguments = ["test_args"]
string_application_arguments = " "
for i in range(0,len(application_arguments)):
string_application_arguments = string_application_arguments + " " + json.dumps(application_arguments[i])
common_submit_args_end = application_resource + string_application_arguments
platform_utilities = PlatformUtilities(_infra_config)
print("platform_utilities.get_python_modules_path() -> ",str(platform_utilities.get_python_modules_path()))
submit_spark_python_module_path = "--conf " + '\"' + "spark.modulePath" + "=" + str(platform_utilities.get_python_modules_path()) + '\"' + " "
submit_spark_args = [common_submit_args_beginning + submit_spark_pipeline_config_conf + submit_spark_python_module_path + common_submit_args_end]
print("submit_spark_args -> ",submit_spark_args)
submit_in_cluster = True
submit_spark_pod_affinity = k8s.V1Affinity(
node_affinity=k8s.V1NodeAffinity(k8s.V1NodeSelectorTerm(
match_expressions=[
k8s.V1NodeSelectorRequirement(key="****", operator="In", values=["n2-highmem-8"]),
k8s.V1NodeSelectorRequirement(key="deployment", operator="In", values=["dynamic"]),
]
)
)
)
submit_spark_pod_tolerations = [k8s.V1Toleration(key="deployment", operator="Equal", value="dynamic", effect="NoSchedule")]
application_name = "test_airflow_api_test_task_id"
container_resources = k8s.V1ResourceRequirements(
requests={
'memory': str("10Gi"),
'cpu': str("2")
},
limits={
'memory': str("50Gi"),
'cpu': str("5")
}
)
submit_startup_timeout_seconds = 600
submit_get_logs = True
kube_submssion = KubernetesPodOperator(namespace = namespace,
image = docker_image,
is_delete_operator_pod = is_delete_operator_pod,
image_pull_secrets = docker_image_creds,
cmds = submit_command,
arguments = submit_spark_args,
in_cluster = submit_in_cluster,
affinity = submit_spark_pod_affinity,
tolerations = submit_spark_pod_tolerations,
container_resources = container_resources,
name = application_name,
task_id = application_name,
startup_timeout_seconds = submit_startup_timeout_seconds,
get_logs = submit_get_logs
)
kube_submssion.execute(context = None)
def def_func2(**kwargs):
print("In func2")
dag_base = airflow_dag_object.get_dag_object()
func1=PythonOperator(
task_id='func1',
provide_context=True,
python_callable=def_func1,
dag=dag_base
)
func2=PythonOperator(
task_id='func2',
provide_context=True,
python_callable=def_func2,
dag=dag_base
)
func1 >> func2
OUTPUT ERROR:-
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py", line 419, in execute
context=context,
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py", line 387, in get_or_create_pod
pod = self.find_pod(self.namespace or pod_request_obj.metadata.namespace, context=context)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py", line 371, in find_pod
label_selector=label_selector,
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api/core_v1_api.py", line 15697, in list_namespaced_pod
return self.list_namespaced_pod_with_http_info(namespace, **kwargs) # noqa: E501
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api/core_v1_api.py", line 15826, in list_namespaced_pod_with_http_info
collection_formats=collection_formats)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api_client.py", line 353, in call_api
_preload_content, _request_timeout, _host)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api_client.py", line 184, in __call_api
_request_timeout=_request_timeout)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/api_client.py", line 377, in request
headers=headers)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/rest.py", line 244, in GET
query_params=query_params)
File "/home/airflow/.local/lib/python3.7/site-packages/kubernetes/client/rest.py", line 234, in request
raise ApiException(http_resp=r)
kubernetes.client.exceptions.ApiException: (400)
Reason: Bad Request
HTTP response headers: HTTPHeaderDict({'Audit-Id': '6ab39ea1-f955-4481-b3eb-7b3abe747a7c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': '8e487991-120d-49d0-940a-ace0b0e64421', 'X-Kubernetes-Pf-Prioritylevel-Uid': '8f6ab0b3-abdf-4782-994c-2f0f247592d2', 'Date': 'Thu, 12 Jan 2023 13:13:20 GMT', 'Content-Length': '169'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"found ',', expected: !, identifier, or 'end of string'","reason":"BadRequest","code":400}
In previous version of airflow < 2.3 KubernetesPodOperator used to work with None context
As mentioned in your question
kube_submssion = KubernetesPodOperator(namespace = namespace,
image = docker_image,
is_delete_operator_pod = is_delete_operator_pod,
image_pull_secrets = docker_image_creds,
cmds = submit_command,
arguments = submit_spark_args,
in_cluster = submit_in_cluster,
affinity = submit_spark_pod_affinity,
tolerations = submit_spark_pod_tolerations,
container_resources = container_resources,
name = application_name,
task_id = application_name,
startup_timeout_seconds = submit_startup_timeout_seconds,
get_logs = submit_get_logs
)
kube_submssion.execute(context = None)
The execute method is expecting the context as mentioned in the documentation at followig link
https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/_modules/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.html#KubernetesPodOperator.execute
You can pass the context from **kwargs to the execute method. You can try by passing kwargs to execute method
kube_submssion.execute(context = kwargs)
I am running this code to list the IP addresses on my network along with the mac addresses but i ran into this problem. it says invalid syntax but i can't seem to find what is wrong with it
I've tried removing spaces and replacing them with tabs but it doesn't fix it. i also tried moving them one up or down but still doesn't work. Any help?
The Whole code:
from getmac import get_mac_address
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
minr = int(input("Starting Ip: "))
maxr = int(input("Ending Ip: "))
while True:
for num in range(minr, maxr + 1): #plus one is to include the last digit entered
ip = "192.168.2." + str(num)
from getmac import getmac
exit_code = os.system("ping -n 1 -w 1 " + ip + " > nul") # Windows
#exit_code = os.system("ping -c 1 -W 1 " + ip + " > /dev/null") # Linux
getmac.PORT = 44444 # Default: 55555
if exit_code == 0:
print(ip, bcolors.OKGREEN + "ONLINE" + bcolors.ENDC + get_mac_address(ip=ip, network_request=True)
elif (ip == '192.168.2.' + str(maxr + 1) and exit_code == 0):
print('192.168.2.' + str(maxr), bcolors.OKGREEN + "ONLINE" + bcolors.ENDC + get_mac_address(ip=ip, network_request=True))
print("")
print(bcolors.HEADER + "Beginning" + bcolors.ENDC)
print("")
elif (ip == '192.168.2.' + str(maxr)):
print('192.168.2.' + str(maxr), bcolors.FAIL + "OFFLINE" + bcolors.ENDC)
print("")
print(bcolors.HEADER + "Refreshed" + bcolors.ENDC)
print("")
else:
print(ip, bcolors.FAIL + "OFFLINE" + bcolors.ENDC)
I am supposed to see the IP addressees along with the mac but i get this error code:
$ python test.py
File "test.py", line 34
elif (ip == '192.168.2.' + str(maxr + 1) and exit_code == 0):
^
SyntaxError: invalid syntax
I just forgot to add the ) at the end of the line above. Thanks to #depperm he showed me my mistake. Where it says ''' print(ip, bcolors.OKGREEN + "ONLINE" + bcolors.ENDC + get_mac_address(ip=ip, network_request=True)
at the end of the bracket add one more. '''
I am trying to scrape data from soccerway.com and checking whether the page is a completed game/game to be played with each instance being written to a seperate csv file. I am running through 10,000 pages and so have written it using Pools. However, I am getting empty lists from the append function and cannot write anything to the csv files.
I tried writing straight to the file instead of list appending however this gave incomplete files
import requests
from bs4 import BeautifulSoup
import time
import numpy as np
import uuid
import time
from multiprocessing import Pool
import sys, os
fixturesA = []
linksA = []
statsA = []
def parse(url):
try:
#print(url)
delays = [0.25,0.5,0.75,1]
delay = np.random.choice(delays)
#time.sleep(delay)
#r = requests.get(url)
r = requests.get(url, timeout = 10)
soup = BeautifulSoup(r.content, "html.parser")
teams = soup.findAll('h3', attrs = {'class' : 'thick'})
homeTeam = teams[0].text.strip()
awayTeam = teams[2].text.strip()
middle = teams[1].text.strip()
dds = soup.findAll('dd')
date = dds[1].text.strip()
gameWeek = dds[2].text.strip()
if ':' not in middle:
middle = middle.split(" - ")
homeGoals = 0
awayGoals = 0
homeGoals = middle[0]
try:
awayGoals = middle[1]
except Exception as e:
homeGoals = "-1"
awayGoals = "-1"
matchGoals = int(homeGoals) + int(awayGoals)
if(matchGoals >= 0):
if(int(homeGoals) > 0 and int(awayGoals) > 0):
btts = "y"
else:
btts = "n"
halfTimeScore = dds[4].text.strip().split(" - ")
firstHalfHomeGoals = halfTimeScore[0]
firstHalfAwayConc = halfTimeScore[0]
firstHalfAwayGoals = halfTimeScore[1]
firstHalfHomeConc = halfTimeScore[1]
firstHalfTotalGoals = int(firstHalfHomeGoals) + int(firstHalfAwayGoals)
secondHalfHomeGoals = int(homeGoals) - int(firstHalfHomeGoals)
secondHalfAwayConc = int(homeGoals) - int(firstHalfHomeGoals)
secondHalfAwayGoals = int(awayGoals) - int(firstHalfAwayGoals)
secondHalfHomeConc = int(awayGoals) - int(firstHalfAwayGoals)
secondHalfTotalGoals = matchGoals - firstHalfTotalGoals
homeTeamContainers = soup.findAll('div', attrs = {'class' : 'container left'})
homeTeamStarting = homeTeamContainers[2]
homeTeamBench = homeTeamContainers[3]
homeTeamYellows = len(homeTeamStarting.findAll('img', attrs = {'src' : 'https://s1.swimg.net/gsmf/700/img/events/YC.png' })) + len(homeTeamBench.findAll('img', attrs = {'src' : 'https://s1.swimg.net/gsmf/699/img/events/YC.png' }))
homeTeamReds = len(homeTeamStarting.findAll('img', attrs = {'src' : 'https://s1.swimg.net/gsmf/700/img/events/RC.png' })) + len(homeTeamBench.findAll('img', attrs = {'src' : 'https://s1.swimg.net/gsmf/699/img/events/RC.png' }))
homeTeamCards = homeTeamYellows + homeTeamReds
awayTeamContainers = soup.findAll('div', attrs = {'class' : 'container right'})
awayTeamStarting = awayTeamContainers[2]
awayTeamBench = awayTeamContainers[3]
awayTeamYellows = len(awayTeamStarting.findAll('img', attrs = {'src' : 'https://s1.swimg.net/gsmf/700/img/events/YC.png' })) + len(awayTeamBench.findAll('img', attrs = {'src' : 'https://s1.swimg.net/gsmf/699/img/events/YC.png' }))
awayTeamReds = len(awayTeamStarting.findAll('img', attrs = {'src' : 'https://s1.swimg.net/gsmf/700/img/events/RC.png' })) + len(awayTeamBench.findAll('img', attrs = {'src' : 'https://s1.swimg.net/gsmf/699/img/events/RC.png' }))
awayTeamCards = awayTeamYellows + awayTeamReds
matchCards = homeTeamCards + awayTeamCards
try:
iframe = soup.findAll('iframe')
iframeSrc = iframe[1]['src']
url = 'https://us.soccerway.com/' + iframeSrc
c = requests.get(url,timeout = 10)
soupC = BeautifulSoup(c.content, "html.parser")
cornerContainer = soupC.findAll('td', attrs = {'class' : 'legend left value'})
homeCorners = cornerContainer[0].text.strip()
awayCornersConc = homeCorners
cornerContainer = soupC.findAll('td', attrs = {'class' : 'legend right value'})
awayCorners = cornerContainer[0].text.strip()
homeCornersConc = awayCorners
matchCorners = int(homeCorners) + int(awayCorners)
print("Got Score . " + homeTeam + " vs " + awayTeam+" . " + gameWeek )
statsA.append(homeTeam + "," + awayTeam + "," + gameWeek + "," + homeGoals + "," + awayGoals + "," + str(matchGoals) + "," + btts + "," + firstHalfHomeGoals + "," + firstHalfHomeConc + "," + firstHalfAwayGoals + "," + firstHalfAwayConc + "," + str(firstHalfTotalGoals) + "," + str(secondHalfHomeGoals) + "," + str(secondHalfHomeConc) + "," + str(secondHalfAwayGoals) + "," + str(secondHalfAwayConc) + "," + str(secondHalfTotalGoals) + "," + str(homeTeamCards) + "," + str(awayTeamCards) + "," + str(matchCards) + "," + homeCorners + "," + awayCorners + "," + homeCornersConc + "," + awayCornersConc + "," + str(matchCorners)+","+dds[0].text.strip() + "\n")
return None
except Exception as e:
print("Got Score no corners. " + homeTeam + " vs " + awayTeam+" . " + gameWeek + " NO FRAME")
statsA.append(homeTeam + "," + awayTeam + "," + gameWeek + "," + homeGoals + "," + awayGoals + "," + str(matchGoals) + "," + btts + "," + firstHalfHomeGoals + "," + firstHalfHomeConc + "," + firstHalfAwayGoals + "," + firstHalfAwayConc + "," + str(firstHalfTotalGoals) + "," + str(secondHalfHomeGoals) + "," + str(secondHalfHomeConc) + "," + str(secondHalfAwayGoals) + "," + str(secondHalfAwayConc) + "," + str(secondHalfTotalGoals) + "," + str(homeTeamCards) + "," + str(awayTeamCards) + "," + str(matchCards) + "," + "" + "," + "" + "," + "" + "," + "" + "," + ""+","+dds[0].text.strip() + "\n")
return None
else:
fixturesA.append(homeTeam + "," + awayTeam + "," + gameWeek + "," + date + "\n")
linksA.append(url + "\n")
print(homeTeam + " vs " + awayTeam + " at " + middle + " GW:" + gameWeek)
return None
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
linksA.append(url + "\n")
print(url)
return None
stats = open('Statsv2.csv','a',encoding='utf-8')
fixtures = open('fixturesv2.csv','w',encoding='utf-8')
with open('links.txt') as f:
content = f.readlines()
content = [x.strip() for x in content]
links = open('links.txt','w')
if __name__ == '__main__':
start_time = time.time()
p = Pool(20) # Pool tells how many at a time
records = p.map(parse, content)
p.terminate()
p.join()
print("--- %s seconds ---" % (time.time() - start_time))
I assume you are running Windows? Then the answer is that multi-processing in Windows creates copies instead of forks. So you have your main process with the lists and you get your working processes (from pool) with their own separate set of lists.
The workers most likely fill their list correctly, but the lists in the main-process don't get any data and so are staying empty. And the workers do not return anything. So, as you write your files in the main-process, you get empty files.
An easy way to solve this is creating pipes or queues between the main process and the workers to allow communication between the threads. You could also use shared arrays like they are provided by the multiprocessing class, but than you would need to know the length during creation.
see documentation: Multiprocessing
as pointed out by #RaJa you're not actually doing anything the parent/controlling process can see. the easiest is just to return values from the mapped function
for example, parse() could return tuple at the end like:
def parse(url):
# do work
return url, homeTeam, awayTeam, gameWeek, homeGoals, awayGoals # ...
then the parent process can receive the values and do useful things like saving them to a CSV file:
import csv
with Pool(20) as pool:
records = pool.map(parse, content)
with open('stats.csv', 'w') as fd:
out = csv.writer(fd)
out.writerow([
'url', 'hometeam', 'awayteam',
# and the remaining column names for the header
])
out.writerows(records)
I am trying to get the System User's Login Time using Python 3.7. I have tried win32net and platform module for Python but, functions are not defined in platform module and Win32net is not compatible with Python 3 and more. I have tried following code:
import platform
platform.uname()
import platform
os_name = platform.uname()[0].lower()
if os_name == "windows":
get_win_login_time()
elif os_name.endswith("nix"):
get_nix_login_time()
Try These ( install win32com.client and subprocess modules first ):
import win32com.client, time
strComputer = "."
objWMIService = win32com.client.Dispatch("WbemScripting.SWbemLocator")
objSWbemServices = objWMIService.ConnectServer(strComputer,"root\cimv2")
colItems = objSWbemServices.ExecQuery("SELECT * FROM Win32_NetworkLoginProfile")
def Convert_to_human_time(dtmDate):
strDateTime = ""
if dtmDate[4] == 0:
strDateTime = dtmDate[5] + '/'
else:
strDateTime = dtmDate[4] + dtmDate[5] + '/'
if dtmDate[6] == 0:
strDateTime = strDateTime + dtmDate[7] + '/'
else:
strDateTime = strDateTime + dtmDate[6] + dtmDate[7] + '/'
strDateTime = strDateTime + dtmDate[0] + dtmDate[1] + dtmDate[2] + dtmDate[3] + " " + dtmDate[8] + dtmDate[9] + ":" + dtmDate[10] + dtmDate[11] +':' + dtmDate[12] + dtmDate[13]
return strDateTime
for objItem in colItems:
if objItem.Name is not None:
print("Name: " + str(objItem.Name))
if objItem.LastLogon is not None:
print("Last Logon (Normal Format): " + str(objItem.LastLogon))
print("Last Logon (Human Readable Format): " + Convert_to_human_time(objItem.LastLogon))
if objItem.LastLogoff is not None:
print("Last Logoff (Normal Format): " + str(objItem.LastLogoff))
print("Last Logoff (Human Readable Format): " + Convert_to_human_time(objItem.LastLogoff))
if objItem.LogonHours is not None:
print("Logon Hours: " + str(objItem.LogonHours))
if objItem.LogonServer is not None:
print("Logon Server: " + str(objItem.LogonServer))
if objItem.NumberOfLogons is not None:
print("Number Of Logons: " + str(objItem.NumberOfLogons))
Another way :
from subprocess import check_output
import sys
get_result = check_output("wmic netlogin get name, fullname, lastlogon", shell=True, stderr=False)
print(get_result)
clean_result = str(get_result).lstrip("b'").rstrip("'").replace("\\r\\r\\n", "\n").replace('\n\n', '\n').split('\n')[2:-1]
for items in clean_result:
print(items.lstrip().rstrip())
Good Luck ...
I have two python modules called main.py and black.py :
Main.py content as follows :
import os,sys,string,time
import subprocess,commands,re
import random
import black
class bcolors:
BOLD = '\033[1m'
UNBOLD = '033[0m'
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
def disable(self):
self.BOLD = ''
self.UNBOLD = ''
self.HEADER = ''
self.OKBLUE = ''
self.OKGREEN = ''
self.WARNING = ''
self.FAIL = ''
self.ENDC = ''
def print_menu():
#subprocess.call("tput clear")
print " "
#print bcolors.OKBLUE + (35 * '-') + bcolors.OKBLUE
print bcolors.OKBLUE + '\t\t\t\t' + (52 * '*') + bcolors.OKBLUE
print ("\t\t\t\t\t\tM A I N - M E N U")
print '\t\t\t\t' + (52 * '*')
print ("\t\t\t\t 0. Enter POD Name(s).")
print ("\t\t\t\t 1. QUIT")
print '\t\t\t\t' + (52 * '*') + bcolors.ENDC
is_valid=0
while not is_valid:
try :
choice = int ( raw_input('Enter your choice [0-24] : ') )
is_valid = 1
except ValueError, e :
print ("'%s' is not a valid integer." % e.args[0].split(": ")[1])
return choice
########## M A I N #####################
rUser=commands.getoutput("id -un").strip()
if rUser != "oracle":
print bcolors.FAIL + " Login as mc_admin to run the script. Exiting .." + bcolors.ENDC
exit ()
else:
os.system("tput clear")
choice = print_menu()
###################
global podList #
podList = None #
###################
while choice >= 0 and choice < 1:
if choice == 0:
pPODName()
black.blackout()
and black.py content as follows:
def blackout():
def pPODName():
global podList
podList = str(raw_input('Enter pipe separated list of PODS : ')).upper().strip()
if podList:
try:
cmd = "/fsnadmin/f516478/romeshar/myfolder/PodDetails.sh -p " + "\"" + podList + "\""
prodP = commands.getoutput(cmd).strip()
if prodP:
print bcolors.FAIL + "Production Pods [Status: Customer Active|Dev Post Provisioning|Cloud Ops Provisioning]: \n" + bcolors.OKBLUE + prodP + bcolors.ENDC
cmd = "/fsnadmin/f516478/romeshar/myfolder/PodDetails.sh -t " + "\"" + podList + "\""
stagP = commands.getoutput(cmd).strip()
if stagP:
print bcolors.FAIL + "Stage/Test Pods [Status: Customer Active|Dev Post Provisioning|Cloud Ops Provisioning]: \n" + bcolors.OKBLUE + stagP + bcolors.ENDC
cmd = "/fsnadmin/f516478/romeshar/myfolder/PodDetails.sh -d " + "\"" + podList + "\""
devP = commands.getoutput(cmd).strip()
if devP:
print bcolors.FAIL + "Dev/Upgrade Pods [Status: Customer Active|Dev Post Provisioning|Cloud Ops Provisioning]: \n" + bcolors.OKBLUE + devP + bcolors.ENDC
cmd = "/fsnadmin/f516478/romeshar/myfolder/PodDetails.sh -i " + "\"" + podList + "\""
intstgP = commands.getoutput(cmd).strip()
if intstgP:
print bcolors.FAIL + "Internal Staging Pods [Status: Customer Active|Dev Post Provisioning|Cloud Ops Provisioning]: \n" + bcolors.OKBLUE + intstgP + bcolors.ENDC
cmd = "/fsnadmin/f516478/romeshar/myfolder/PodDetails.sh -r " + "\"" + podList + "\""
prtnP = commands.getoutput(cmd).strip()
if prtnP:
print bcolors.FAIL + "Partner Pods [Status: Customer Active|Dev Post Provisioning|Cloud Ops Provisioning]: \n" + bcolors.OKBLUE + prtnP + bcolors.ENDC
cmd = "/fsnadmin/f516478/romeshar/myfolder/PodDetails.sh -s " + "\"" + podList + "\""
stbyP = commands.getoutput(cmd).strip()
if stbyP:
print bcolors.FAIL + "DR/Standby Pods [Status: Customer Active|Dev Post Provisioning|Cloud Ops Provisioning]: \n" + bcolors.OKBLUE + stbyP + bcolors.ENDC
cmd = "/fsnadmin/f516478/romeshar/myfolder/PodDetails.sh -u " + "\"" + podList + "\""
unalloP = commands.getoutput(cmd).strip()
if unalloP:
print bcolors.FAIL + "Unallocated Pods [Status: Dev Post Provisioning|Cloud Ops Provisioning]: \n" + bcolors.OKBLUE + unalloP + bcolors.ENDC
cmd = "/fsnadmin/f516478/romeshar/myfolder/PodDetails.sh -l " + "\"" + podList + "\""
trlP = commands.getoutput(cmd).strip()
if trlP:
print bcolors.FAIL + "Trial Pods [Status: Customer Active|Dev Post Provisioning|Cloud Ops Provisioning]: \n" + bcolors.OKBLUE + trlP + bcolors.ENDC
except OSError:
print bcolors.FAIL + "Could not invoke Pod Details Script. " + bcolors.ENDC
podList = vPodName(podList)
print bcolors.FAIL + "\nYou Have Entered PodList as: " + podList + "\n" + bcolors.ENDC
uResp = str(raw_input('Do You Want To Continue [YES|Y|NO|N] : ')).upper().strip()
#if uResp == "NO" or uResp == "N":
if uResp != "YES" and uResp != 'Y':
pPODName ()
if __name__ == '__main__':
blackout()
I want black.py module to be called in main.py but when I am running main.py I am getting the following error:
[oracle#localhost tmp]$ ./main.py
****************************************************
M A I N - M E N U
****************************************************
0. Enter POD Name(s).
1. QUIT
Enter your choice [0-24] : 0
Traceback (most recent call last):
File "./main.py", line 131, in <module>
pPODName()
NameError: name 'pPODName' is not defined
pPODName has not been defined as far as main.py can tell.
Firstly
pPODName is defined within another function, so only that function is aware of it. This is referred to as being in local scope. Basically anything defined within a function is only accessible to that function, and nothing else.
For an example:
def test():
def test_local():
return 3
return test_local()
Running the function test gives you
>>> test()
3
However running the function test_local
>>> test_local()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
NameError: name 'test_local' is not defined
Secondly
Secondly, when importing modules, you need to either qualify them with the module name, like you have done with black.blackout(), of you need to use:
from black import *
Although personally I don't like that, it makes it hard to see where methods are coming from.
Fix
For main.py to be able to call pPODName, it must be defined within the module as a whole, e.g.:
def blackout()
#do stuff here
def pPODNAME()
#do other stuff here
#note the indentation level is the same as blackout
and you must call it with
black.pPODName()
From main.py
You have some programming hierarchy level mistake
why pPODName is inner function of blackout ?
and has James said- if you are using outer and inner function the outer needs to call the inner.
I suggest you to use regular function in black.py
blackout function does nothing except containing the pPODName so you don't need it
Shlomy