Develop Reference

why do I get [Errno 2]?

so I have created a server that as for now does a very simple task, it suppose to send to the browser a packet whitch contains the string "datatatata".
but it does'nt work, when running it and sending a request from browser, I keep getting [Errno 2]. the request is of course HTTP request
the code:
import socket
# functions
def get_file_data(filename):
pass
def handle_client_request(file_path, client_socket):
file_to_return = open(file_path, 'rb')
file_content = file_to_return.read()
client_socket.send(file_content)
def validate_client_request(client_request):
# checks validation of request. for now validation is set to true
validation = True
# checks for requested URL
desired_file_name = ""
for i in client_request[5::]:
if i != " ":
desired_file_name += i
else:
break
# checking file type
file_type = ""
if desired_file_name == "favicon":
file_type = "ico"
elif desired_file_name == "abstruct" or desired_file_name == "":
file_type = "jpg"
elif desired_file_name == "loading":
file_type = "gif"
if file_type == "":
validation = False
file_path = "C:\\Users\\hadad\\Desktop\\webroot\\imgs\\" + desired_file_name + "." + file_type
return validation, file_path
def handle_client(client_socket):
client_request = client_socket.recv(1024).decode()
valid_request, file_path = validate_client_request(client_request)
if valid_request:
handle_client_request(file_path, client_socket)
else:
client_socket.send("unvalid request. connection ended".encode())
client_socket.close()
def main():
server_socket = socket.socket()
server_socket.bind(("0.0.0.0", 80))
server_socket.listen()
# everlasting loop
while True:
client_socket, client_address = server_socket.accept()
handle_client(client_socket)
if __name__ == '__main__':
main()
the error:
Traceback (most recent call last):
File "C:/Networks/hed/good_server.py", line 64, in <module>
main()
File "C:/Networks/hed/good_server.py", line 60, in main
handle_client(client_socket)
File "C:/Networks/hed/good_server.py", line 46, in handle_client
handle_client_request(file_path, client_socket)
File "C:/Networks/hed/good_server.py", line 11, in handle_client_request
file_to_return = open(file_path, 'rb')
FileNotFoundError: [Errno 2] No such file or directory: 'C:\\Users\\hadad\\Desktop\\webroot\\imgs\\abstruct.jpg'

trying to send data from one script in python to another script

I am trying to write a script so when an IP address can't be seen a message gets sent with a telegram letting me know which computer is offline
I have been able to get the telegram side working but i have not been able to pass the data from the main script where it is testing the ip address , at the moment i have test data in there but i would like it to send the error with the computer name
main.py
import socket
import ssl
from datetime import datetime
import pickle
import subprocess
import platform
class Server:
def __init__(self, name, port, connection, priority):
self.name = name
self.port = port
self.connection = connection.lower()
self.priority = priority.lower()
self.history = []
self.alert = False
def check_connection(self):
msg = ""
success = False
now = datetime.now().strftime("%d-%m-%Y %H:%M")
try:
if self.connection == "plain":
socket.create_connection((self.name, self.port), timeout=10)
msg = f"{self.name} is up. On port {self.port} with {self.connection}"
success = True
self.alert = False
elif self.connection == "ssl":
ssl.wrap_socket(socket.create_connection((self.name, self.port), timeout=10))
msg = f"{self.name} is up. On port {self.port} with {self.connection}"
success = True
self.alert = False
else:
if self.ping():
msg = f"{self.name} is up. On port {self.port} with {self.connection}"
success = True
self.alert = False
except socket.timeout:
msg = f"server: {self.name} timeout. On port {self.port}"
except (ConnectionRefusedError, ConnectionResetError) as e:
msg = f"server: {self.name} {e}"
except Exception as e:
msg = f"No Clue??: {e}"
if success == False and self.alert == False:
# Send Alert
self.alert = True
import tg_start
tg_start.send_message("Happy days")
self.create_history(msg, success, now)
def create_history(self, msg, success, now):
history_max = 100
self.history.append((msg, success, now))
while len(self.history) > history_max:
self.history.pop(0)
def ping(self):
try:
output = subprocess.check_output("ping -{} 1 {}".format('n' if platform.system().lower(
) == "windows" else 'c', self.name), shell=True, universal_newlines=True)
if 'unreachable' in output:
return False
else:
return True
except Exception:
return False
if __name__ == "__main__":
try:
servers = pickle.load(open("servers.pickle", "rb"))
except:
servers = [
# Server("ifmc-repserver", 80, "plain", "high"),
# Server("ifmc-repserver", 80, "plain", "high"),
# Server("ifmc-repserver", 465, "ssl", "high"),
# Server("ifmc-repserver", 80, "ping", "high"),
Server("ifmc-repserver", 80, "ping", "high")
]
for server in servers:
server.check_connection()
print(len(server.history))
print(server.history[-1])
pickle.dump(servers, open("servers.pickle", "wb"))
and tg_start.py
import requests
message = "global"
alert = ""
def send_message(text):
global alert
global message
print ("this is text messsage" + " " + text)
#text = "Superman"
alert = text
print("Sending ALERT ...")
token = "token"
chat_id = "chat_id"
print("test message" + " " + alert)
url_req = "https://api.telegram.org/bot" + token + "/sendMessage" + "?chat_id=" + chat_id + "&text=" + alert
print(url_req)
#results = requests.get(url_req)
results = requests.post(url_req) # this request is a post, not a get
print(results.json())
# text = "my name" + text
send_message(alert)

You code worked with a slight change, sendMessage require a POST request, not a GET request.
def send_message(text):
global alert
global message
print ("this is text messsage" + " " + text)
alert = text
print("Sending ALERT ...")
token = "token"
chat_id = "chat_id"
print("test message" + " " + alert)
url_req = f"https://api.telegram.org/bot{token}/sendMessage?chat_id={chat_id}&text={alert}"
print(url_req)
results = requests.post(url_req) # this request is a post, not a get
print(results.json())
# text = "my name" + text

How realtime capture logs of query from HiveServer2 with python client?

I use modified version of pyhs2 (https://pypi.python.org/pypi/pyhs2) with ability run async queries and additional methods from TCLIService.Client (GetLog, send_GetLog, recv_GetLog) in sources of Hue (https://github.com/cloudera/hue/blob/master/apps/beeswax/gen-py/TCLIService/TCLIService.py#L739)
But when I run TCLIService.Client.GetLog method, there is an error:
$ python example.py
Traceback (most recent call last):
File "example.py", line 85, in <module>
rq = client.GetLog(lq)
File "/Users/toly/hive_streaming/libs/pyhs4/TCLIService/TCLIService.py", line 757, in GetLog
return self.recv_GetLog()
File "/Users/toly/hive_streaming/libs/pyhs4/TCLIService/TCLIService.py", line 773, in recv_GetLog
raise x
thrift.Thrift.TApplicationException: Invalid method name: 'GetLog'
In script I use HiveServer2 from Cloudera VM. Same server, as I quess, used by Hue and it successfully works. In addition I try client_protocol in range from 0 to 7 for creating session.
import time
import sasl
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from thrift.transport.TSocket import TSocket
from thrift.transport.TTransport import TBufferedTransport
from libs.pyhs4.cloudera.thrift_sasl import TSaslClientTransport
from libs.pyhs4.TCLIService import TCLIService
from libs.pyhs4.TCLIService.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,\
TStatusCode, TGetResultSetMetadataReq, TGetColumnsReq, TType, TTypeId, \
TExecuteStatementReq, TGetOperationStatusReq, TFetchOrientation, TCloseOperationReq, \
TCloseSessionReq, TGetSchemasReq, TCancelOperationReq, TGetLogReq
auth = 'PLAIN'
username = 'apanin'
password = 'none'
host = 'cloudera'
port = 10000
test_hql1 = 'select count(*) from test_text'
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("username", username)
saslc.setAttr("password", password)
saslc.init()
return saslc
def get_type(typeDesc):
for ttype in typeDesc.types:
if ttype.primitiveEntry is not None:
return TTypeId._VALUES_TO_NAMES[ttype.primitiveEntry.type]
elif ttype.mapEntry is not None:
return ttype.mapEntry
elif ttype.unionEntry is not None:
return ttype.unionEntry
elif ttype.arrayEntry is not None:
return ttype.arrayEntry
elif ttype.structEntry is not None:
return ttype.structEntry
elif ttype.userDefinedTypeEntry is not None:
return ttype.userDefinedTypeEntry
def get_value(colValue):
if colValue.boolVal is not None:
return colValue.boolVal.value
elif colValue.byteVal is not None:
return colValue.byteVal.value
elif colValue.i16Val is not None:
return colValue.i16Val.value
elif colValue.i32Val is not None:
return colValue.i32Val.value
elif colValue.i64Val is not None:
return colValue.i64Val.value
elif colValue.doubleVal is not None:
return colValue.doubleVal.value
elif colValue.stringVal is not None:
return colValue.stringVal.value
sock = TSocket(host, port)
transport = TSaslClientTransport(sasl_factory, "PLAIN", sock)
client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = client.OpenSession(TOpenSessionReq(username=username, password=password))
session = res.sessionHandle
query1 = TExecuteStatementReq(session, statement=test_hql1, confOverlay={}, runAsync=True)
response1 = client.ExecuteStatement(query1)
opHandle1 = response1.operationHandle
while True:
time.sleep(1)
q1 = TGetOperationStatusReq(operationHandle=opHandle1)
res1 = client.GetOperationStatus(q1)
lq = TGetLogReq(opHandle1)
rq = client.GetLog(lq)
if res1.operationState == 2:
break
req = TCloseOperationReq(operationHandle=opHandle1)
client.CloseOperation(req)
req = TCloseSessionReq(sessionHandle=session)
client.CloseSession(req)
How realtime capture logs of hive query from HiveServer2?
UPD Hive version - 1.2.1

For getting logs of operation used method FetchResults with param fetchType=1 - returning logs.
Example usage:
query1 = TExecuteStatementReq(session, statement=test_hql1, confOverlay={}, runAsync=True)
response1 = client.ExecuteStatement(query1)
opHandle1 = response1.operationHandle
while True:
time.sleep(1)
q1 = TGetOperationStatusReq(operationHandle=opHandle1)
res1 = client.GetOperationStatus(q1)
request_logs = TFetchResultsReq(operationHandle=opHandle1, orientation=0, maxRows=10, fetchType=1)
response_logs = client.FetchResults(request_logs)
print response_logs.results
if res1.operationState == 2:
break

Check server status on Twisted

While I was writing simple message-based fileserver and client, I got the idea about checking fileserver status, but don't know how to realize this: just try to connect and disconnect from server (and how disconnect immediately, when server is not running, if using this way?) or maybe twisted/autobahn have some things, which help to get server status without creating "full connection"?
a) fileserver.py
import os
import sys
import json
from twisted.internet import reactor
from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol, listenWS
CONFIG_TEMPLATE = ''
CONFIG_DATA = {}
class MessageBasedServerProtocol(WebSocketServerProtocol):
"""
Message-based WebSockets server
Template contains some parts as string:
[USER_ID:OPERATION_NAME:FILE_ID] - 15 symbols for USER_ID,
10 symbols for OPERATION_NAME,
25 symbols for FILE_ID
other - some data
"""
def __init__(self):
path = CONFIG_DATA['path']
base_dir = CONFIG_DATA['base_dir']
# prepare to working with files...
if os.path.exists(path) and os.path.isdir(path):
os.chdir(path)
if not os.path.exists(base_dir) or not os.path.isdir(base_dir):
os.mkdir(base_dir)
os.chdir(base_dir)
else:
os.makedir(path)
os.chdir(path)
os.mkdir(base_dir)
os.chdir(base_dir)
# init some things
self.fullpath = path + '/' + base_dir
def __checkUserCatalog(self, user_id):
# prepare to working with files...
os.chdir(self.fullpath)
if not os.path.exists(user_id) or not os.path.isdir(user_id):
os.mkdir(user_id)
os.chdir(user_id)
else:
os.chdir(self.fullpath + '/' + user_id)
def onOpen(self):
print "[USER] User with %s connected" % (self.transport.getPeer())
def connectionLost(self, reason):
print '[USER] Lost connection from %s' % (self.transport.getPeer())
def onMessage(self, payload, isBinary):
"""
Processing request from user and send response
"""
user_id, cmd, file_id = payload[:54].replace('[', '').replace(']','').split(':')
data = payload[54:]
operation = "UNK" # WRT - Write, REA -> Read, DEL -> Delete, UNK -> Unknown
status = "C" # C -> Complete, E -> Error in operation
commentary = 'Succesfull!'
# write file into user storage
if cmd == 'WRITE_FILE':
self.__checkUserCatalog(user_id)
operation = "WRT"
try:
f = open(file_id, "wb")
f.write(data)
except IOError, argument:
status = "E"
commentary = argument
except Exception, argument:
status = "E"
commentary = argument
raise Exception(argument)
finally:
f.close()
# read some file
elif cmd == 'READU_FILE':
self.__checkUserCatalog(user_id)
operation = "REA"
try:
f = open(file_id, "rb")
commentary = f.read()
except IOError, argument:
status = "E"
commentary = argument
except Exception, argument:
status = "E"
commentary = argument
raise Exception(argument)
finally:
f.close()
# delete file from storage (and in main server, in parallel delete from DB)
elif cmd == 'DELET_FILE':
self.__checkUserCatalog(user_id)
operation = "DEL"
try:
os.remove(file_id)
except IOError, argument:
status = "E"
commentary = argument
except Exception, argument:
status = "E"
commentary = argument
raise Exception(argument)
self.sendMessage('[%s][%s]%s' % (operation, status, commentary), isBinary=True)
if __name__ == '__main__':
if len(sys.argv) < 2:
print "using python fileserver_client.py [PATH_TO_config.json_FILE]"
else:
# read config file
CONFIG_TEMPLATE = sys.argv[1]
with open(CONFIG_TEMPLATE, "r") as f:
CONFIG_DATA = json.load(f)
# create server
factory = WebSocketServerFactory("ws://localhost:9000")
factory.protocol = MessageBasedServerProtocol
listenWS(factory)
reactor.run()
b) client.py
import json
import sys
import commands
from twisted.internet import reactor
from autobahn.twisted.websocket import WebSocketClientFactory, WebSocketClientProtocol, connectWS
CONFIG_TEMPLATE = ''
CONFIG_DATA = {}
class MessageBasedClientProtocol(WebSocketClientProtocol):
"""
Message-based WebSockets client
Template contains some parts as string:
[USER_ID:OPERATION_NAME:FILE_ID] - 15 symbols for USER_ID,
10 symbols for OPERATION_NAME,
25 symbols for FILE_ID
other - some data
"""
def onOpen(self):
user_id = CONFIG_DATA['user']
operation_name = CONFIG_DATA['cmd']
file_id = CONFIG_DATA['file_id']
src_file = CONFIG_DATA['src_file']
data = '[' + str(user_id) + ':' + str(operation_name) + ':' + str(file_id) + ']'
if operation_name == 'WRITE_FILE':
with open(src_file, "r") as f:
info = f.read()
data += str(info)
self.sendMessage(data, isBinary=True)
def onMessage(self, payload, isBinary):
cmd = payload[1:4]
result_cmd = payload[6]
if cmd in ('WRT', 'DEL'):
print payload
elif cmd == 'REA':
if result_cmd == 'C':
try:
data = payload[8:]
f = open(CONFIG_DATA['src_file'], "wb")
f.write(data)
except IOError, e:
print e
except Exception, e:
raise Exception(e)
finally:
print payload[:8] + "Successfully!"
f.close()
else:
print payload
reactor.stop()
if __name__ == '__main__':
if len(sys.argv) < 2:
print "using python fileserver_client.py [PATH_TO_config.json_FILE]"
else:
# read config file
CONFIG_TEMPLATE = sys.argv[1]
with open(CONFIG_TEMPLATE, "r") as f:
CONFIG_DATA = json.load(f)
# connection to server
factory = WebSocketClientFactory("ws://localhost:9000")
factory.protocol = MessageBasedClientProtocol
connectWS(factory)
reactor.run()

Find solution this issue: using callLater or deferLater for disconnect from server, if can't connect, but when all was 'OK', just take server status, which he says.
import sys
from twisted.internet.task import deferLater
from twisted.internet import reactor
from autobahn.twisted.websocket import WebSocketClientFactory, WebSocketClientProtocol, connectWS
CONFIG_IP = ''
CONFIG_PORT = 9000
def isOffline(status):
print status
class StatusCheckerProtocol(WebSocketClientProtocol):
def __init__(self):
self.operation_name = "STATUS_SRV"
self.user_id = 'u00000000000000'
self.file_id = "000000000000000000000.log"
def onOpen(self):
data = '[' + str(self.user_id) + ':' + str(self.operation_name) + ':' + str(self.file_id) + ']'
self.sendMessage(data, isBinary=True)
def onMessage(self, payload, isBinary):
cmd = payload[1:4]
result_cmd = payload[6]
data = payload[8:]
print data
reactor.stop()
if __name__ == '__main__':
if len(sys.argv) < 3:
print "using python statuschecker.py [IP] [PORT]"
else:
# read preferences
CONFIG_IP = sys.argv[1]
CONFIG_PORT = int(sys.argv[2])
server_addr = "ws://%s:%d" % (CONFIG_IP, CONFIG_PORT)
# connection to server
factory = WebSocketClientFactory(server_addr)
factory.protocol = StatusCheckerProtocol
connectWS(factory)
# create special Deffered, which disconnect us from some server, if can't connect within 3 seconds
d = deferLater(reactor, 3, isOffline, 'OFFLINE')
d.addCallback(lambda ignored: reactor.stop())
# run all system...
reactor.run()

TypeError: cannot concatenate 'str'

I got this error TypeError: cannot concatenate 'str' and 'builtin_function_or_method' objects when i tried to change directory in the ftp.
ftp.py
from ftplib import FTP
ftp = FTP()
class FTPClient():
connection_id = None
login_ok = False
message_array = []
def __init__(self):
pass
def log_message(self, message, clear=True):
if clear:
self.message_array = message
def get_message(self):
return self.message_array
def connect(self, server, ftp_user, ftp_password, is_passive = False):
self.connection_id = ftp.connect(server)
login_result = ftp.login(user=ftp_user, passwd=ftp_password)
ftp.set_pasv(is_passive)
if not self.connection_id or not login_result:
self.log_message("FTP Connection has Failed")
self.log_message("Attempted to connect to {0} for {1}".format(server, ftp_user))
return False
else:
self.log_message("Connected to {0} for {1}".format(server, ftp_user))
self.login_ok = True
return True
def make_dir(self, directory):
if ftp.mkd(directory):
self.log_message("Directory {0} created successfully".format(directory))
return True
else:
self.log_message("Failed creating directory")
return False
def change_directory(self, directory):
if ftp.cwd(directory):
self.log_message("Current Directory is now {0}".format(ftp.pwd))
else:
self.log_message("Can't change Directory")
def upload_file(self, file_from, file_to):
if file_from.endswith(('.csv', '.txt')):
with open(file_from, 'r') as f:
self.connection_id.storelines("Uploaded from {0} to {1}".format(file_from, file_to), f)
else:
with open(file_from, 'rb') as f:
self.connection_id.storebinary('Uploaded from {0} to {1}'.format(file_from, file_to), f)
ftp_obj = FTPClient()
FTP_HOST = "yyy"
FTP_USER = "xxx"
FTP_PASS = "zzz"
ftp_obj.connect(FTP_HOST, FTP_USER, FTP_PASS)
print(ftp_obj.get_message())
FILE_FROM = "config.txt"
FILE_TO = ftp_obj.change_directory(dir)
ftp_obj.upload_file(FILE_FROM, FILE_TO)
print(ftp_obj.get_message())
Full traceback:
Connected to yyy for xxx
Traceback (most recent call last):
File "C:/Users/Ajay/PycharmProjects/database/test.py", line 67, in <module>
FILE_TO = ftp_obj.change_directory(dir)
File "C:/Users/Ajay/PycharmProjects/database/test.py", line 44, in change_directory
if ftp.cwd(directory):
File "C:\Python27\lib\ftplib.py", line 552, in cwd
cmd = 'CWD ' + dirname
TypeError: cannot concatenate 'str' and 'builtin_function_or_method' objects
Whats happening there ? and why this error ?

Error is here:
FILE_TO = ftp_obj.change_directory(dir)
you have not declared dir var, builtin function dir passed instead.