I have set up the below in Apache Airflow Admin --> Connections.
How do I read these values programmatically inside my DAG?
def check_email_requests():
conn = Connection(conn_id="artnpics_api_calls")
print(conn)
hostname = conn.host
login_name = conn.login
login_password = conn.password
port_number = conn.port
print("hostname = " + hostname + "; Login name: " + login_name + "; password = " + login_password + " ; port number = " + port_number)
request_api = hostname + ":" + port_number
print("request api " + request_api)
result = requests.get(request_api, auth=(login_name, login_password)).json()
print(result)
print("done with check_email_requests")
return False
The above obviously did not work, and I couldn't find any information on how to read from the connections (there is numerous article on how to create one programmatically). My objective is to read API connection and authentication information programmatically and invoke the call, rather than hard coding them.
Rhonald
You can do:
from airflow.hooks.base import BaseHook
conn = BaseHook.get_connection("artnpics_api_calls")
hostname = conn.host
login_name = conn.login
login_password = conn.password
port_number = conn.port
Related
I'm setting up a little server with commands and other kind of stuff, but ,I don't get the thread functionement, when I connect my server it seems like everything is okay, I can connect a first client "without" problems, but when I want to connect another client it never get conected , the code runs but I can send anything I want it never shows up on the other client or the server.
I've already read the Threading documentation but even with the exemples, I don't get it, can someone give me some clues about how to proceed to handle more than just one client?
the server code :
#!/usr/bin/python3+x
import socket
import sys
from time import sleep
import threading
import random
HOST = "localhost"
PORT = 33700
MSG_SIZE = 32768
serveur_on = True
CLIENT_NICK_CHAN = {} #clients" nicks dict like {nickname : channel} -> needed to easily know who is where
CLIENT_NICK_SOCKET = {} #clients" dict like {nickname : socket} -> needed to send private message to the nickname's socket easily
CLIENT_NICK_THREAD = {} #clients" dict with like {nick : thread}
Rand_disconnection_msg = [" has drown in the abyss.", " is disconnected.", " is now in a better place.", " is now a part of our past", " passed away, in really tragic circumstances..."]
CHANNELS = ["main lobby", "test"]
CMD_LIST = [b"HELP",b"NICK",b"JOIN",b"CHANNELS",b"LEAVE"]
COMMANDS = ["/NICK <nickname>: Use only when you\'re connecting, allow you to choose a unique nickname",
"/JOIN <channel_name>: Allow you to join or create a channel, you can\'t use this command if your are already in another channel than the" + CHANNELS[0],
"/CHANNELS : Allow you to see every channels on the server with every connected people",
"/LEAVE : You leave the channel your within and get bringed back to the" + CHANNELS[0],
"/HELP : Gives you the whole command list",
"/BYE : Disconnect ou from the server, you have to in the " + CHANNELS[0] + " to use this command"
]
class Colors:
Blue, Cyan, Green, Red, Magenta, Yellow, White =b"\033[94m", b"\033[96m", b"\033[92m", b"\033[91m", b"\033[95m", b"\033[93m", b"\033[0m"
Normal, Bold, Italics, Thin = b"\033[0m", b"\033[1m", b"\x1B[3m", b"\033[2m"
class thread_client(threading.Thread):
def __init__(self,conn):
self.nom = ""
if(self.nom == ""):
nickname_input(connexion, self)
print("nom : " + self.nom.decode("utf8"))
self.channel = CHANNELS[0]
self.admin = False
self.adress = ""
threading.Thread.__init__(self)
self.connexion = conn
print("init done")
def run(self):
while True:
msgClient = self.connexion.recv(MSG_SIZE)
if not msgClient or msgClient == b"BYE":
break
print(type(self.nom))
print(type(msgClient))
str_name = self.nom.decode("utf8")
msg = str_name + " > " + msgClient.decode("utf8")
print("string type name is : " + str_name + "\n")
str_msg = msgClient.decode("utf8")
print("{} > {}".format(str_name, str_msg))
for clients in CLIENT_NICK_SOCKET:
if clients != self.nom:
CLIENT_NICK_SOCKET[clients].send(bytes(str_msg,"utf8"))
self.connexion.send(b"You are now disconnected.\n")
self.connexion.close()
del CLIENT_NICK_SOCKET[self.nom.decode("utf8")]
del CLIENT_NICK_CHAN[self.nom.decode("utf8")]
rand_leave = random.randint(0, len(Rand_disconnection_msg)-1)
leaving_msg = Rand_disconnection_msg[rand_leave]
print(str_name + leaving_msg + "\n")
def nickname_input(client_socket, thread):
print("now in input nickname")
msg_nom = client_socket.recv(MSG_SIZE)
print("msg_nom = " + msg_nom.decode("utf8"))
msg_nom_arr = msg_nom.split()
if not msg_nom_arr[0]:
client_socket.send(b"Please send a non void message")
nickname_input(client_socket, thread)
print("msg_nom_arr[0] = " + str(msg_nom_arr[0]))
if(msg_nom_arr[0] == b"NICK"):
if(len(msg_nom_arr)== 1):
client_socket.send(b"Please do not just enter '/NICK' -> you have to type '/NICK <your_nickname>', please proceed again :\n")
nickname_input(client_socket, thread)
else:
thread.nom = msg_nom_arr[1]
else:
client_socket.send(b"It seems like you forgot to use '/NICK' before entering your nickname, please proceed again:\n")
nickname_input(client_socket, thread)
return
def print_channels(client_socket, thread):
client_socket.send(b"Here\'s the current channel list :\n\n")
for chan in CHANNELS:
sleep(0.70)
client_socket.send( bytes(chan,"utf8") + b":\n current members :\n")
for chan_user in CLIENT_NICK_CHAN:
if(CLIENT_NICK_CHAN[chan_user] == chan):
sleep(0.35)
if(chan_user == thread.nom):
if(thread.admin):
client_socket.send(b" " +Colors.Bold + Colors.Yellow + b"#"+ thread.nom + b"#" + Colors.Normal + b"\n")
else:
client_socket.send(b" " +Colors.Bold + Colors.Yellow + thread.nom + Colors.Normal + b"\n")
else:
client_socket.send(b" " +bytes(chan_user,"utf8") + b"#\n")
client_socket.send(b"\n")
client_socket.send(b"\n")
return
def join_channel(client_socket, thread, data, data_array):
if(not data_arr[1]):
connexion.send(b"Please select a channel you want to join using '/JOIN <channel_name>'\nNote that if the channel you asked for doesn\'t exists a new channel <channel_name> will be created and you will be the administrator of this channel")
return
else:
asked_channel = data_arr[1]
if( not (asked_channel in CHANNELS)):
thread.channel = asked_channel
thread.admin = True
connexion.send(b"Welcome in " + asked_channel + b" channel, since you\'re the on who created this channel you are granted as administrator for this channel")
connexion.send(b"Note that being administrator allow you tu use some new commands as '/GRANT', '/REVOKE' or '/REN', for more information please use '/HELP'")
else:
thread.channel = asked_channel
connexion.send(b"Welcome in " + asked_channel + b" channel !")
return
SERVER = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
SERVER.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
try:
SERVER.bind((HOST,PORT))
except socket.error:
print("Server connexion failed")
sys.exit()
print("Server is now connected\nWaiting for connexions...\n")
SERVER.listen(5)
connexion, adresse = SERVER.accept()
thread = thread_client(connexion)
thread.start()
print("thread type = " +str(type(thread)) +"\n")
print("thread = ")
print(thread)
connexion.send(bytes("Welcome ","utf8") + Colors.Yellow + Colors.Bold + thread.nom + Colors.Normal)
nick = thread.nom #type -> bytes
str_nick = nick.decode("utf8")
CLIENT_NICK_CHAN[str_nick] = thread.channel
CLIENT_NICK_SOCKET[str_nick] = connexion
CLIENT_NICK_THREAD[str_nick] = thread
print("client list : ")
print(CLIENT_NICK_CHAN)
print("\n")
print("CLIENT_NICK_SOCKET = ")
print(CLIENT_NICK_SOCKET)
print("\n")
while serveur_on:
conn_msg = str_nick + " joined the chat\n"
print(conn_msg)
connexion.send(b"hello world 3\n\n")
connexion.send(b"*" * 80 + b"\n")
connexion.send(Colors.Red + Colors.Bold + b"\nWELCOME IN THE MAIN LOBBY \n" + Colors.Normal+b"\nTo enter a channel use '/JOIN <channel_name>'\nthe <channel_name> have to be composed by one world or use underscores to join words\nIf the channel does not exists a new one will be created\n\nNote that you have to be in another channel than the main lobby to chat\n")
print_channels(connexion, thread)
connexion.send(b"*" * 80 + b"\n\n")
while True:
print("thread list = ")
print(CLIENT_NICK_THREAD)
data = connexion.recv(MSG_SIZE) #receiving data from client
data_arr= data.split() #convert data into an array to check if the first word in the message is "MSG" or not
print(str_nick +" is now in -> " + thread.channel + "\n")
if(data_arr[0] in CMD_LIST):
if(data.startswith(b"HELP")): #HELP CMD
for command in COMMANDS:
connexion.send(bytes(command,"utf") + b"\n")
if(data.startswith(b"CHANNELS")): #Channels + current members CMD
connexion.send(b"\n")
print_channel(connexion, thread)
connexion.send(b"\n")
if(data.startswith(b"JOIN")):
join_channel(connexion, thread, data, data_arr)
connexion.send(b"\n")
else:
if ((thread.channel != CHANNELS[0]) and (data.startswith("MSG"))):
for chan_user in thread.channel:
chan_user.send(nick + b" > " + bytes(data,"utf8"))
print("data = " + data)
elif (thread.channel == CHANNELS[0]):
connexion.send(b"You have to be in another channel than the " + bytes(CHANNELS[0], "utf8") + b" to start chating !\nPlease use '/JOIN <channel_name>' or '/HELP' to learn how to join another channel.\n\n")
and the client code:
#!/usr/bin/python3+x
host = ''
port = 33700
MSG_SIZE = 32768
emission_stop = False
import socket
import sys
import threading
import time
def Tsend():
while True:
msg_envoi = input("> ")
if msg_envoi.startswith("/"):
msg_envoi = msg_envoi.replace("/","",1)
else:
msg_envoi = msg_envoi
CLIENT.send(bytes(msg_envoi,"utf8"))
if emission_stop:
CLIENT.close()
def Trecv():
while True:
msg_recu = CLIENT.recv(MSG_SIZE).decode("utf8")
print("\n" + msg_recu)
if not msg_recu:
break
emission_stop = True
print("connexion lost\n")
CLIENT.close()
CLIENT = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
try:
CLIENT.connect((host,port))
except socket.error:
print("connexion failed\n")
sys.exit()
print("Now connected to the server on port: {}\n".format(port))
print("Please now enter your nickname using '/NICK'\n")
thread_emission = threading.Thread(target = Tsend)
thread_reception = threading.Thread(target = Trecv)
thread_emission.start()
thread_reception.start()
What I want is just to have multiple clients that are allowed to talk to each other but I can't even get two clients.
The biggest problem I can see is that you are only calling SERVER.accept() once. This means you will only ever accept 1 client connection. When using blocking sockets as you are, a typical approach is to keep doing SERVER.accept() inside a loop so that you can keep accepting all client sockets. After you accept() a new socket, you create new thread(s) dedicated to sending/receiving for that socket, so that you don't block the accepting thread. And then you continue accepting more connections. Something like this:
#SERVER:
while serveur_on:
connexion, adresse = SERVER.accept()
# Possibly do some limited IO with client socket here, but be careful not
# to block this thread too long because that will prevent more clients from
# connecting.
thread = thread_client(connexion)
thread.start()
# No more client IO on this thread, it's the client thread's job now.
You seem to have code that communicates with the client (receiving messages and sending responses) in 2 different places: on the main thread after you SERVER.accept(), and up in thread_client.run(). It doesn't make sense that way, it should all be in thread_client.run().
I'm a networking student who has little programming experience and I have simple project on my hands where I need to make a simple notification that notifies the user when an email arrives in gmail. I have been trying to get this to work for the last few weeks and I am honestly stuck and struggling. I'm using the code below and I think the problem is the authentication bit, where I can't connect to the server. All help is appreciated.
import imaplib
import re
import time
import subprocess
## enter your account details bellow!
imapServer = "imap.gmail.com"
port = "993"
username = "example#gmail.com"
password = "password"
##how often to check ? give interval in seconds! Checking too often might performance and stability.
checkInterval = 120
Mailbox = imaplib.IMAP4_SSL(imapServer, port)
rc,resp = Mailbox.login(username,password)
if rc == 'OK':
print("Connected to mail-server " + imapServer)
rc, message = Mailbox.status('INBOX', "(UNSEEN)")
unreadCount = int(re.search("UNSEEN (\d+)",str( message[0])).group(1))
oldValue = 0
file = open("%sytemdrive%\Windows\Temp\mailnotify.tmp", "w+")
file.write(str(unreadCount))
file.close
while(1):
rc, message = Mailbox.status('INBOX', "(UNSEEN)")
unreadCount = int(re.search("UNSEEN (\d+)",str( message[0])).group(1))
file = open("%sytemdrive%\Windows\Temp\mailnotify.tmp", "w+")
oldValue = int(file.readline())
file.close()
if (unreadCount>oldValue):
subprocess.call(["notify-send", "-u", "low", "low", "t", "5000", "New email!", "New email!",
"You have " + str(unreadCount) + " unread " + "emails!" if unreadCount > 1 else "email!",
"--icon=email"])
if oldValue != unreadCount:
file = open("%sytemdrive%\Windows\Temp\mailnotify.tmp", "w+")
file.write(str(unreadCount))
file.close()
time.sleep(checkInterval)
else :
print('Fail to connect')
Mailbox.logout()
file.remove()
I am trying to make my IRC bot handle multiple messages at a time, but it's not sending back messages.
Behavior: Process(target=func) is called, func() calls a function that has socket.socket().send(message) in it, but the message doesn't send. Suspect is that the socket isn't passed to the sending function.
Code:
import socket
import re
import requests
import urllib
import config # just my file of variables
import math
import time
import sys
import winsound
import string
import random
import multiprocessing
# import traceback
# CONNECTION COMMANDS
ircsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server = config.server # Server
password = config.password # Password
botnick = config.botnick # Your bots nick
adminname = config.adminname # Your IRC nickname
exitcode = config.exitcode
ircsock.settimeout(300)
def connection(host, port, password, nick, realname):
ircsock.connect((host, port))
ircsock.send(bytes("PASS " + password + "\n", "UTF-8"))
ircsock.send(bytes("USER " + botnick + " " + botnick + " " + botnick + " " + botnick + "\n", "UTF-8"))
ircsock.send(bytes("NICK " + botnick + "\n", "UTF-8"))
def ping(): # respond to server Pings.
ircsock.send(bytes("PONG :pingis\n", "UTF-8"))
print("Ponged after " + str(time.time() - last_ping) + " seconds from last ping!")
def sendmsg(msg, target): # sends messages to the target.
# it enters here, no problem
ircsock.send(bytes("PRIVMSG " + target + " :" + msg + "\n", "UTF-8")) ### At this point, when using multiprocessing, the bot fails ###
print("Sending: [" + str(msg) + "] to: " + str(target))
# MAIN
if __name__ == '__main__':
connection(server, 6667, password, botnick, botnick)
# joinchan(channel)
while 1:
# print("Connected!")
ircmsg = ircsock.recv(1024).decode("UTF-8")
ircmsg = ircmsg.strip('\n\r')
if ircmsg.find("PRIVMSG") != -1:
try:
# “:[Nick]!~[hostname]#[IP Address] PRIVMSG [channel] :[message]”
name = ircmsg.split('PRIVMSG', 1)[0].split(':')[-1].split("!")[0] # ircmsg.split('!', 1)[0][1:]
message = ircmsg.split('PRIVMSG', 1)[1].split(':', 1)[1].splitlines()[0] # .strip()[0]
me = ircmsg.split('PRIVMSG', 1)[1].split(':', 1)[0].split()[0]
# print(me)
print("name: " + name + ", message: " + message)
if len(name) < 17:
if me == botnick:
if message.find("Hi!") != -1:
process1 = multiprocessing.Process(target=sendmsg, args=("Hello!", name))
process1.daemon = True
process1.start()
if name.lower() == adminname.lower() and message.rstrip() == exitcode:
sendmsg("Bot is quitting.", name)
ircsock.send(bytes("QUIT \n", "UTF-8"))
sys.exit()
time.sleep(1)
except:
pass
elif ircmsg.find("PING") != -1:
ping()
Please word your answers as simply as possible, since I am not that experienced in Python. The code above can be run with a correct config.py file.
Format:
password = "" # password to open the server
exitcode = "" # What is typed to stop the bot
server = "" # Server
botnick = "" # Your bots nick
adminname = "" # Your IRC nickname
I wanted to create a Python program that does several things. Ping all addresses in a predefined network, gather the DNS information, write a file with IP address, DNS name, ping fail or pass, date. Then run and email the resulting file to myself once a week, every Friday. I have created this program and will post my own answer. I am new to Python and was able to get this written with the help from other answers posted on this site. Thanks to all those who contributed answers on this site. Hope the answer I post will help someone else.
#!/usr/bin/python3.4
#Above statement makes sure you are using version 3.4
#when multiple versions are installed. has to be the 1st line.
# Import modules
import subprocess
import socket
import errno
import time
import datetime
import ipaddress
today = datetime.date.today()
# define DNS lookup and error handling
# return none,none,none needed otherwise if no DNS record
# the routine errors out and the program stops
def lookup(addr):
try:
return socket.gethostbyaddr(addr)
except socket.herror:
return None, None, None
# Prompt the user to input a network address
# commented out the prompt for input so it can run unattended
# net_addr = input("Enter a network address in CIDR
format(ex.192.168.1.0/24): ")
net_addr = ('192.168.1.0/24')
# Create the network
ip_net = ipaddress.ip_network(net_addr)
# Get all hosts on that network
all_hosts = list(ip_net.hosts())
# Configure subprocess to hide the console window
# removed code due to errors not windows linux
# setup online and offline count variables
offCnt = 0
onCnt = 0
# Open file and or create if it doesn't exist.
# file to be overwritten each time the program is run.
file = open("lab-ip.doc","w")
# For each IP address in the subnet,
# run the ping command with subprocess.popen interface
# Grab the DNS information for each IP address
# Print to console add counters and write to file.
for i in range(len(all_hosts)):
output = subprocess.Popen(['ping', '-c', '2', str(all_hosts[i])],
stdout=subprocess.PIPE).communicate()[0]
name,alias,addresslist = lookup(str(all_hosts[i]))
if "Destination Host Unreachable" in output.decode('utf-8'):
print(str(all_hosts[i]), " Ping Fail", str(name), today)
file.write(str(all_hosts[i]) + " Ping Fail - " + str(name) + " " + str(today) + "\n")
offCnt = offCnt + 1
elif "Request timed out" in output.decode('utf-8'):
print(str(all_hosts[i]), " Ping Fail", str(name), today)
file.write(str(all_hosts[i]) + " Ping Fail - " + str(name) + " " + str(today) + "\n")
offCnt = offCnt + 1
else:
print(str(all_hosts[i]), " Ping Pass", str(name), today)
file.write(str(all_hosts[i]) + " Ping Pass - " + str(name) + " " + str(today) + "\n")
onCnt = onCnt + 1
print ("Pass count = ", str(onCnt))
file.write("Pass count = " + str(onCnt))
print ("Fail count = ", str(offCnt))
file.write(" Fail count = " + str(offCnt))
file.close()
# Import yagmail for the actual sending function
import yagmail
yag = yagmail.SMTP('Gmail-id', 'gmail-pswd')
yag.send('email#email.com', subject = "Lab-ip List",contents = 'lab-ip.doc')
yag.send('email2#email2.com', subject = "Lab-ip List",contents = 'lab-ip.doc')
#end
The script below works great when logged in as root and run from the
command line, but when run at first boot using /etc/rc.local in Ubuntu
10.04, it fails about 25% of the time- the system root, mysql root and
some mysql user passwords are set correctly, but one will fail with
console log reporting standard mysql login error: "ERROR 1045 (28000):
Access denied for user 'root' # 'localhost' (using password: YES)"
Is there something about running python scripts from init jobs that I
should account for, such as an environment variable?
#!/usr/bin/env python
# Randomizes and outputs to files the system root and mysql user passwords
files = ['/home/ubuntu/passwords','/opt/data1/alfresco/extensions/
extension/alfresco-global.properties','/opt/data/etc/mysql/
debian.cnf','/home/ubuntu/duncil']
userpasswords = {'root':'ROOTPASSWORD'}
mysqlpasswords =
{'root':'MYSQLPASSWORD','alfresco':'alfrescoPASSWORD','debian-sys-
maint':'debian-sys-maintPASSWORD'}
otherpasswords = ['OTHERPASSWORD']
log = '/var/log/firstrun'
import random, string
import crypt
import re
from subprocess import PIPE, Popen
def getsalt(chars = string.letters + string.digits):
# generate a random 2-character 'salt'
return random.choice(chars) + random.choice(chars)
def getpwd(chars = string.letters + string.digits, len = 12):
retval = "";
for i in range(0, len):
# generate 12 character alphanumeric password
retval += random.choice(chars)
return retval
def replace_pass(filename):
handle = open(filename, 'r')
hbuf = handle.read()
handle.close()
for placeholder, password in pdict.iteritems():
hbuf = re.sub(placeholder, password, hbuf)
try:
# Output file
handle = open(filename, 'w')
handle.write(hbuf)
handle.close()
except:
pass
#logh.write('failed to update ' + filename + "\n")
#logh.write('maybe you don\'t have permision to write to it?\n')
logh = open(log, "a")
logh.write("Starting...\n")
# Generate passwords
pdict = {}
for user, placeholder in userpasswords.iteritems():
syspass = getpwd()
Popen(['usermod', '--password', crypt.crypt(syspass, getsalt()), user])
logh.write(placeholder + ": User " + user + " --> " + syspass + "\n")
pdict[placeholder] = syspass
# Whats the MySQL Root password placeholder?
mplace = mysqlpasswords['root']
for user, placeholder in mysqlpasswords.iteritems():
mpass = getpwd()
if (("root" in mysqlpasswords) and (mysqlpasswords['root'] in pdict)):
mrootpass = pdict[mysqlpasswords['root']]
else:
mrootpass = ""
Popen(['mysql', '-uroot', "--password=" + mrootpass, "-e", "UPDATE user SET Password = PASSWORD('" + mpass + "') WHERE User = '" + user + "';FLUSH PRIVILEGES;","mysql"])
logh.write(placeholder + ": MySQL " + user + " --> " + mpass + "\n")
pdict[placeholder] = mpass
for placeholder in otherpasswords:
opass = getpwd()
logh.write(placeholder + ": " + opass + "\n")
pdict[placeholder] = opass
# Update passwords
for file in files:
logh.write("Replacing placeholders in " + file + "\n")
replace_pass(file)
logh.write("Finished\n")
logh.close
Doesn't Popen execute asynchronously?
It seems that during boot, the load is high and you are getting a race condition between setting the root password and using it to set the next password (next command).
Try
p = Popen(['mysql', '-uroot', "--password=" + mrootpass, "-e", "UPDATE user SET Password = PASSWORD('" + mpass + "') WHERE User = '" + user + "';FLUSH PRIVILEGES;","mysql"])
p.wait()
and see if that does it.