Develop Reference

Sending and receiving package in python

This is a console chat app on a TCP socket server. The client will send the request/message to the server and the server will distribute the message to the target user or provide requested information.I am currently running into a problem regarding the recv package on the server side. I received the package and was able to print it out. However the system still give me a syntax error for some reason.
Thanks.
This is my client:
import socket
import select
import errno
import sys, struct
import pickle
HEADER_LENGTH = 1024
IP = "127.0.0.1"
PORT = 9669
def send_login_request(username):
package = [1]
length = len(username)
if length > 1019:
print ("Error: Username too long")
sys.exit()
package += struct.pack("I", length)
package += username
return package
def send_message(recv_id, message):
package = [2]
length = len(message)
if length > 1015:
print('message too long')
sys.exit()
package += recv_id
package += struct.pack('I', length)
package += message
return package
def send_con_request(conv_id):
package = [3]
length = len(id)
if length > 1015:
print('id too long')
sys.exit()
package += struct.pack("I", length)
package += conv_id
return package
# Create a socket
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Connect to a given ip and port
client_socket.connect((IP, PORT))
client_socket.setblocking(False)
my_username = input("Username: ")
request = send_login_request(my_username)
user_request = str(request)
client_socket.send(user_request.encode())
username_conf = client_socket.recv(HEADER_LENGTH).decode()
if username_conf == "Welcome to the server":
con_id = input("Please enter conversation's id, if don't have one, please enter no ")
if con_id == 'no':
con_request = send_con_request(con_id)
con_request = str(con_request)
client_socket.send(con_request.encode())
else:
con_request = send_con_request(con_id)
con_request = str(con_request)
client_socket.send(con_request.encode())
conversation = client_socket.recv(HEADER_LENGTH).decode()
recv_id = input("Please enter receiver's id")
while True:
# Wait for user to input a message
message = input(f'{my_username} > ').encode()
# If message is not empty - send it
if message:
send_message = send_message(recv_id,message)
client_socket.send(bytes(send_message))
try:
while True:
message_receiver = client_socket.recv(HEADER_LENGTH).decode()
x = message_receiver.split('|')
print(x)
username = x[0]
message = x[1]
# Print message
print(f'{username} > {message}')
except IOError as e:
if e.errno != errno.EAGAIN and e.errno != errno.EWOULDBLOCK:
print('Reading error: {}'.format(str(e)))
sys.exit()
# We just did not receive anything
continue
except Exception as e:
# Any other exception - something happened, exit
print('Reading error: {}'.format(str(e)))
sys.exit()
This is my server:
import socket
import select
import struct
import sys
import pickle
HEADER_LENGTH = 1024
conversation ={}
users = [
{
'username': 'user1',
'user_id': 1
},
{
'username': 'user2',
'user_id': 2
},
{
'username': 'user3',
'user_id': 3
},
{
'username': 'user4',
'user_id': 4
},
{
'username': 'user5',
'user_id': 5
}
]
def login(username):
for user in users:
if user['username'] == username:
return user
else:
return False
IP = "127.0.0.1"
PORT = 9669
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((IP, PORT))
server_socket.listen()
# List of sockets for select.select()
sockets_list = [server_socket]
# List of connected clients - socket as a key, user header and name as data
clients_socket = {}
sessions = {
(1,2) : '1.txt',
(3,4) : '2.txt'
}
def getRecvSocket(user_id):
try:
return sessions[user_id]
except:
return None
def sendErrorMes(socketid, mes):
package = [9]
length = len(mes)
if length > 1019:
length = 1019
package += struct.pack("I", length)
package += mes
print(f'Listening for connections on {IP}:{PORT}...')
# Handles message receiving
def receive_message(client_socket):
try:
receive_message = client_socket.recv(HEADER_LENGTH)
return receive_message
except:
return False
while True:
read_sockets, _, exception_sockets = select.select(sockets_list, [], sockets_list)
# Iterate over notified sockets
for notified_socket in read_sockets:
# If notified socket is a server socket - new connection, accept it
if notified_socket == server_socket:
client_socket, client_address = server_socket.accept()
sockets_list.append(client_socket)
else:
# Receive message
package = receive_message(notified_socket)
print(package)
package_recv = eval(package.decode())
print(package_recv)
print(type(package_recv))
package_type = package_recv[0]
if package_type == 1:
size = struct.unpack("I", package[1:5])
if size[0] > 1019:
continue
username = package[5:5+size[0]]
username = username.decode()
# username = package_recv[1]
user = login(username)
if user == False:
notified_socket.send("no user found".encode())
else:
sessions[user["user_id"]] = notified_socket
notified_socket.send(("Welcome to the server").encode())
elif package_type == 2:
recv_id = struct.unpack("I", package[1:5])
size = struct.unpack("I", package[5:9])
if size[0] > 1015:
continue
# recv_id = package_recv[1]
if getRecvSocket(recv_id) == None:
sendErrorMes(notified_socket, "User is offline")
else:
message = package[9:9+size[0]]
# message = package_recv[2]
for socket in sessions.values():
if socket == notified_socket:
user = sessions[notified_socket]
# print(f'Received message from {user}, {message}')
# fIterate over connected clients and broadcast message
for client_socket in clients_socket:
# if clients[client_socket] == receive_user and client_socket != notified_socket:
# But don't sent it to sender
if client_socket != notified_socket and clients_socket[client_socket] == recv_id:
# Send user and message (both with their headers)
# We are reusing here message header sent by sender, and saved username header send by user when he connected
a = sessions[notified_socket]
b = recv_id
with open(f"{conversation[a,b]}.txt", "w"):
f.write(user + message)
client_socket.send((user + "|" + message).encode())
if message is False:
# print('Closed connection from: {}'.format(user))
# Remove from list for socket.socket()
sockets_list.remove(notified_socket)
# Remove from our list of users
del clients_socket[notified_socket]
continue
elif package_type == 3:
size = struct.unpack("I", package[1:5])
if size[0] > 1019:
continue
convo_id = package[5:5+size[0]]
convo_id = convo_id.decode()
# convo_id = package_recv[2]
if convo_id in conversation:
with open(conversation[convo_id], 'rb') as file_to_send:
for data in file_to_send:
notified_socket.sendall(data)
print('send successful')
else:
f = open(f"{len(conversation)+1}.txt", "w+")
This is the error in the server side which I am having a problem to locate and solve:
Listening for connections on 127.0.0.1:9669...
b"[1, 5, 0, 0, 0, 'u', 's', 'e', 'r', '1']"
[1, 5, 0, 0, 0, 'u', 's', 'e', 'r', '1']
<class 'list'>
b''
Traceback (most recent call last):
File "c:/Users/Duong Dang/Desktop/bai 2.3/server.py", line 134, in <module>
package_recv = eval(package.decode())
File "<string>", line 0
^
SyntaxError: unexpected EOF while parsing

The sending code doesn't make a lot of sense. You're creating a python list which is a most strange way to implement a protocol. You're then taking python's string representation of that list and sending it to the server. You're not doing anything on the server side to ensure that you got the entire message. Then you're using eval to interpret the string you created on the client. That is a very dangerous practice, as your peer can essentially instruct your python interpreter to do literally anything.
Also, your send_con_request is calling len(id) which won't work at all because id is a python built-in that doesn't supply a __len__ method. I assume that was supposed to be len(conv_id)?
Anyway, you should rework your protocol. Use the struct tools to create the correct binary string you want. There are tons of possible ways to structure this but here's one. On the client side, create a fixed-length header that identifies which request type you're sending and the length of the remaining "payload" bytes. You'll convert your string (username or whatever) into bytes first with str.encode.
import struct
# ProtoHeader encodes a 16 bit request identifer, plus a 32 bit payload
# length. A protocol data unit consists of this 6-byte header followed by
# payload bytes (which will vary according to the request)
ProtoHeader = struct.Struct("!HI")
LoginRequest = 1
SomeOtherRequest = 2
...
def format_login_request(username):
""" Create a protocol block containing a user login request.
Return the byte string containing the encoded request """
username_bytes = username.encode()
proto_block = ProtoHeader.pack(LoginRequest, len(username_bytes)) + username_bytes
return proto_block
...
conn.sendall(format_login_request(username))
On the server side, you will first receive the fixed-length header (which tells you what the request type was and how many other payload bytes are present). Then receive those remaining bytes ensuring that you get exactly that many. socket.recv does not guarantee that you will receive exactly the number of bytes sent in any particular send from the peer. It does guarantee that you will get them in the right order so you must keep receiving until you got exactly the number you expected. That's why it's important to have fixed length byte strings as a header and to encode the number of bytes expected in variable length payloads.
The server would look something like this:
import struct
ProtoHeader = struct.Struct("!HI")
LoginRequest = 1
def receive_bytes(conn, count):
""" General purpose receiver:
Receive exactly #count bytes from #conn """
buf = b''
remaining = count
while remaining > 0:
# Receive part or all of data
tbuf = conn.recv(remaining)
tbuf_len = len(tbuf)
if tbuf_len == 0:
# Really you probably want to return 0 here if buf is empty and
# allow the higher-level routine to determine if the EOF is at
# a proper message boundary in which case, you silently close the
# connection. You would normally only raise an exception if you
# EOF in the *middle* of a message.
raise RuntimeError("end of file")
buf += tbuf
remaining -= tbuf_len
return buf
def receive_proto_block(conn):
""" Receive the next protocol block from #conn. Return a tuple of
request_type (integer) and payload (byte string) """
proto_header = receive_bytes(conn, ProtoHeader.size)
request_type, payload_length = ProtoHeader.unpack(proto_header)
payload = receive_bytes(conn, payload_length)
return request_type, payload
...
request_type, payload = receive_proto_block(conn)
if request_type == LoginRequest:
username = payload.decode()

Python: Bloomberg API Failed to get Token. No Authorisation

Trying to use the IntradayTickExample code enclosed below. However I kept getting the following message. It says about authorization so I guess its getting stuck in the main. But I have no idea why this wouldn't work as its a BBG example?
My thoughts:
- It exits with code 0 so everything works well.
- It seems to go debug through the code until the authorisation section
- I cant find any way to change anything else r./e. the authorisation
C:\Users\ME\MyNewEnv\Scripts\python.exe F:/ME/BloombergWindowsSDK/Python/v3.12.2/examples/IntradayTickExample.py
IntradayTickExample
Connecting to localhost:8194
TokenGenerationFailure = {
reason = {
source = "apitkns (apiauth) on ebbdbp-ob-596"
category = "NO_AUTH"
errorCode = 12
description = "User not in emrs userid=DBG\ME firm=8787"
subcategory = "INVALID_USER"
}
}
Failed to get token
No authorization
Process finished with exit code 0
And full code below
# IntradayTickExample.py
from __future__ import print_function
from __future__ import absolute_import
import blpapi
import copy
import datetime
from optparse import OptionParser, Option, OptionValueError
TICK_DATA = blpapi.Name("tickData")
COND_CODE = blpapi.Name("conditionCodes")
TICK_SIZE = blpapi.Name("size")
TIME = blpapi.Name("time")
TYPE = blpapi.Name("type")
VALUE = blpapi.Name("value")
RESPONSE_ERROR = blpapi.Name("responseError")
CATEGORY = blpapi.Name("category")
MESSAGE = blpapi.Name("message")
SESSION_TERMINATED = blpapi.Name("SessionTerminated")
TOKEN_SUCCESS = blpapi.Name("TokenGenerationSuccess")
TOKEN_FAILURE = blpapi.Name("TokenGenerationFailure")
AUTHORIZATION_SUCCESS = blpapi.Name("AuthorizationSuccess")
TOKEN = blpapi.Name("token")
def authOptionCallback(option, opt, value, parser):
vals = value.split('=', 1)
if value == "user":
parser.values.auth = "AuthenticationType=OS_LOGON"
elif value == "none":
parser.values.auth = None
elif vals[0] == "app" and len(vals) == 2:
parser.values.auth = "AuthenticationMode=APPLICATION_ONLY;"\
"ApplicationAuthenticationType=APPNAME_AND_KEY;"\
"ApplicationName=" + vals[1]
elif vals[0] == "userapp" and len(vals) == 2:
parser.values.auth = "AuthenticationMode=USER_AND_APPLICATION;"\
"AuthenticationType=OS_LOGON;"\
"ApplicationAuthenticationType=APPNAME_AND_KEY;"\
"ApplicationName=" + vals[1]
elif vals[0] == "dir" and len(vals) == 2:
parser.values.auth = "AuthenticationType=DIRECTORY_SERVICE;"\
"DirSvcPropertyName=" + vals[1]
else:
raise OptionValueError("Invalid auth option '%s'" % value)
def checkDateTime(option, opt, value):
try:
return datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
except ValueError as ex:
raise OptionValueError(
"option {0}: invalid datetime value: {1} ({2})".format(
opt, value, ex))
class ExampleOption(Option):
TYPES = Option.TYPES + ("datetime",)
TYPE_CHECKER = copy.copy(Option.TYPE_CHECKER)
TYPE_CHECKER["datetime"] = checkDateTime
def parseCmdLine():
parser = OptionParser(description="Retrieve intraday rawticks.",
epilog="Notes: " +
"1) All times are in GMT. " +
"2) Only one security can be specified.",
option_class=ExampleOption)
parser.add_option("-a",
"--ip",
dest="host",
help="server name or IP (default: %default)",
metavar="ipAddress",
default="localhost")
parser.add_option("-p",
dest="port",
type="int",
help="server port (default: %default)",
metavar="tcpPort",
default=8194)
parser.add_option("-s",
dest="security",
help="security (default: %default)",
metavar="security",
default="IBM US Equity")
parser.add_option("-e",
dest="events",
help="events (default: TRADE)",
metavar="event",
action="append",
default=[])
parser.add_option("--sd",
dest="startDateTime",
type="datetime",
help="start date/time (default: %default)",
metavar="startDateTime",
default=None)
parser.add_option("--ed",
dest="endDateTime",
type="datetime",
help="end date/time (default: %default)",
metavar="endDateTime",
default=None)
parser.add_option("--cc",
dest="conditionCodes",
help="include condition codes",
action="store_true",
default=False)
parser.add_option("--auth",
dest="auth",
help="authentication option: "
"user|none|app=<app>|userapp=<app>|dir=<property>"
" (default: %default)",
metavar="option",
action="callback",
callback=authOptionCallback,
type="string",
default="user")
(options, args) = parser.parse_args()
if not options.host:
options.host = ["localhost"]
if not options.events:
options.events = ["TRADE"]
return options
def authorize(authService, identity, session, cid):
tokenEventQueue = blpapi.EventQueue()
session.generateToken(eventQueue=tokenEventQueue)
# Process related response
ev = tokenEventQueue.nextEvent()
token = None
if ev.eventType() == blpapi.Event.TOKEN_STATUS or \
ev.eventType() == blpapi.Event.REQUEST_STATUS:
for msg in ev:
print(msg)
if msg.messageType() == TOKEN_SUCCESS:
token = msg.getElementAsString(TOKEN)
elif msg.messageType() == TOKEN_FAILURE:
break
if not token:
print("Failed to get token")
return False
# Create and fill the authorization request
authRequest = authService.createAuthorizationRequest()
authRequest.set(TOKEN, token)
# Send authorization request to "fill" the Identity
session.sendAuthorizationRequest(authRequest, identity, cid)
# Process related responses
startTime = datetime.datetime.today()
WAIT_TIME_SECONDS = 10
while True:
event = session.nextEvent(WAIT_TIME_SECONDS * 1000)
if event.eventType() == blpapi.Event.RESPONSE or \
event.eventType() == blpapi.Event.REQUEST_STATUS or \
event.eventType() == blpapi.Event.PARTIAL_RESPONSE:
for msg in event:
print(msg)
if msg.messageType() == AUTHORIZATION_SUCCESS:
return True
print("Authorization failed")
return False
endTime = datetime.datetime.today()
if endTime - startTime > datetime.timedelta(seconds=WAIT_TIME_SECONDS):
return False
def printErrorInfo(leadingStr, errorInfo):
print("%s%s (%s)" % (leadingStr, errorInfo.getElementAsString(CATEGORY),
errorInfo.getElementAsString(MESSAGE)))
def processMessage(msg):
data = msg.getElement(TICK_DATA).getElement(TICK_DATA)
print("TIME\t\t\t\tTYPE\tVALUE\t\tSIZE\tCC")
print("----\t\t\t\t----\t-----\t\t----\t--")
for item in data.values():
time = item.getElementAsDatetime(TIME)
timeString = item.getElementAsString(TIME)
type = item.getElementAsString(TYPE)
value = item.getElementAsFloat(VALUE)
size = item.getElementAsInteger(TICK_SIZE)
if item.hasElement(COND_CODE):
cc = item.getElementAsString(COND_CODE)
else:
cc = ""
print("%s\t%s\t%.3f\t\t%d\t%s" % (timeString, type, value, size, cc))
def processResponseEvent(event):
for msg in event:
print(msg)
if msg.hasElement(RESPONSE_ERROR):
printErrorInfo("REQUEST FAILED: ", msg.getElement(RESPONSE_ERROR))
continue
processMessage(msg)
def sendIntradayTickRequest(session, options, identity = None):
refDataService = session.getService("//blp/refdata")
request = refDataService.createRequest("IntradayTickRequest")
# only one security/eventType per request
request.set("security", options.security)
# Add fields to request
eventTypes = request.getElement("eventTypes")
for event in options.events:
eventTypes.appendValue(event)
# All times are in GMT
if not options.startDateTime or not options.endDateTime:
tradedOn = getPreviousTradingDate()
if tradedOn:
startTime = datetime.datetime.combine(tradedOn,
datetime.time(15, 30))
request.set("startDateTime", startTime)
endTime = datetime.datetime.combine(tradedOn,
datetime.time(15, 35))
request.set("endDateTime", endTime)
else:
if options.startDateTime and options.endDateTime:
request.set("startDateTime", options.startDateTime)
request.set("endDateTime", options.endDateTime)
if options.conditionCodes:
request.set("includeConditionCodes", True)
print("Sending Request:", request)
session.sendRequest(request, identity)
def eventLoop(session):
done = False
while not done:
# nextEvent() method below is called with a timeout to let
# the program catch Ctrl-C between arrivals of new events
event = session.nextEvent(500)
if event.eventType() == blpapi.Event.PARTIAL_RESPONSE:
print("Processing Partial Response")
processResponseEvent(event)
elif event.eventType() == blpapi.Event.RESPONSE:
print("Processing Response")
processResponseEvent(event)
done = True
else:
for msg in event:
if event.eventType() == blpapi.Event.SESSION_STATUS:
if msg.messageType() == SESSION_TERMINATED:
done = True
def getPreviousTradingDate():
tradedOn = datetime.date.today()
while True:
try:
tradedOn -= datetime.timedelta(days=1)
except OverflowError:
return None
if tradedOn.weekday() not in [5, 6]:
return tradedOn
def main():
options = parseCmdLine()
# Fill SessionOptions
sessionOptions = blpapi.SessionOptions()
sessionOptions.setServerHost(options.host)
sessionOptions.setServerPort(options.port)
sessionOptions.setAuthenticationOptions(options.auth)
print("Connecting to %s:%s" % (options.host, options.port))
# Create a Session
session = blpapi.Session(sessionOptions)
# Start a Session
if not session.start():
print("Failed to start session.")
return
identity = None
if options.auth:
identity = session.createIdentity()
isAuthorized = False
authServiceName = "//blp/apiauth"
if session.openService(authServiceName):
authService = session.getService(authServiceName)
isAuthorized = authorize(
authService, identity, session,
blpapi.CorrelationId("auth"))
if not isAuthorized:
print("No authorization")
return
try:
# Open service to get historical data from
if not session.openService("//blp/refdata"):
print("Failed to open //blp/refdata")
return
sendIntradayTickRequest(session, options, identity)
print(sendIntradayTickRequest(session, options, identity))
# wait for events from session.
eventLoop(session)
finally:
# Stop the session
session.stop()
if __name__ == "__main__":
print("IntradayTickExample")
try:
main()
except KeyboardInterrupt:
print("Ctrl+C pressed. Stopping...")
__copyright__ = """
Copyright 2012. Bloomberg Finance L.P.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions: The above
copyright notice and this permission notice shall be included in all copies
or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
"""

I realized I have been trying to connect to the Server API instead of connectiong to the local terminal. For example, the following code works perfectly:
# SimpleHistoryExample.py
from __future__ import print_function
from __future__ import absolute_import
import blpapi
from optparse import OptionParser
def parseCmdLine():
parser = OptionParser(description="Retrieve reference data.")
parser.add_option("-a",
"--ip",
dest="host",
help="server name or IP (default: %default)",
metavar="ipAddress",
default="localhost")
parser.add_option("-p",
dest="port",
type="int",
help="server port (default: %default)",
metavar="tcpPort",
default=8194)
(options, args) = parser.parse_args()
return options
def main():
options = parseCmdLine()
# Fill SessionOptions
sessionOptions = blpapi.SessionOptions()
sessionOptions.setServerHost(options.host)
sessionOptions.setServerPort(options.port)
print("Connecting to %s:%s" % (options.host, options.port))
# Create a Session
session = blpapi.Session(sessionOptions)
# Start a Session
if not session.start():
print("Failed to start session.")
return
try:
# Open service to get historical data from
if not session.openService("//blp/refdata"):
print("Failed to open //blp/refdata")
return
# Obtain previously opened service
refDataService = session.getService("//blp/refdata")
# Create and fill the request for the historical data
request = refDataService.createRequest("HistoricalDataRequest")
request.getElement("securities").appendValue("EURUSD Curncy")
request.getElement("fields").appendValue("PX_LAST")
request.set("periodicityAdjustment", "ACTUAL")
request.set("periodicitySelection", "DAILY")
request.set("startDate", "20000102")
request.set("endDate", "20191031")
request.set("maxDataPoints", 100000)
print("Sending Request:", request)
# Send the request
session.sendRequest(request)
# Process received events
while(True):
# We provide timeout to give the chance for Ctrl+C handling:
ev = session.nextEvent(500)
for msg in ev:
print(msg)
if ev.eventType() == blpapi.Event.RESPONSE:
# Response completly received, so we could exit
break
finally:
# Stop the session
session.stop()
if __name__ == "__main__":
print("SimpleHistoryExample")
try:
main()
except KeyboardInterrupt:
print("Ctrl+C pressed. Stopping...")

RYU Controller with ARP Loop (Python API Mininet)

TOpology Loop
I have a problem with the ARP loop. I have attached my python API code and the Simple_Swicth.py code that I used. As you can see the picture, I cannot able to ping h1 to h2. can able to help?
Here is my topology code:
#!/usr/bin/python
from mininet.node import Controller, RemoteController, OVSController
from mininet.net import Mininet
from mininet.node import OVSKernelSwitch, UserSwitch
from mininet.cli import CLI
from mininet.link import TCLink, Intf
from mininet.log import setLogLevel, info
def Topo1():
net = Mininet(controller=RemoteController, switch=OVSKernelSwitch)
info('*** Adding controll\n')
c0 = net.addController(name='c0', controller=RemoteController, ip='127.0.0.1', protocol='tcp', port=6633)
info('*** Add host\n')
h1 = net.addHost('h1')
h2 = net.addHost('h2')
info('*** Add switches\n')
s1 = net.addSwitch('s1')
s2 = net.addSwitch('s2')
s3 = net.addSwitch('s3')
s4 = net.addSwitch('s4')
s5 = net.addSwitch('s5')
s1.linkTo( h1 )
s1.linkTo( s2 )
s2.linkTo( s3 )
s2.linkTo( s4 )
s3.linkTo( s4 )
s4.linkTo( s5 )
s5.linkTo( h2 )
net.build()
c0.start
info('***Starting switches\n')
net.get('s1').start([c0])
net.get('s2').start([c0])
net.get('s3').start([c0])
net.get('s4').start([c0])
net.get('s5').start([c0])
info('***Post configure switches and hosts\n')
net.start()
net.pingAll()
CLI(net)
net.stop()
if __name__ == '__main__':
setLogLevel('info')
Topo1()
here is the simple_switch code for ryu remote controller. Does anyone know which part of the simple_Switch code that I need add extra code to manage ARP loop. And since I do not really good at it don you mind to let me know what kind of code should I use?
from ryu.base import app_manager
from ryu.controller import ofp_event
from ryu.controller.handler import CONFIG_DISPATCHER, MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_3
from ryu.lib.packet import packet
from ryu.lib.packet import ethernet
from ryu.lib.packet import ether_types
class SimpleSwitch13(app_manager.RyuApp):
OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]
def __init__(self, *args, **kwargs):
super(SimpleSwitch13, self).__init__(*args, **kwargs)
self.mac_to_port = {}
#set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER)
def switch_features_handler(self, ev):
datapath = ev.msg.datapath
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
# install table-miss flow entry
#
# We specify NO BUFFER to max_len of the output action due to
# OVS bug. At this moment, if we specify a lesser number, e.g.,
# 128, OVS will send Packet-In with invalid buffer_id and
# truncated packet data. In that case, we cannot output packets
# correctly. The bug has been fixed in OVS v2.1.0.
match = parser.OFPMatch()
actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER,
ofproto.OFPCML_NO_BUFFER)]
self.add_flow(datapath, 0, match, actions)
def add_flow(self, datapath, priority, match, actions, buffer_id=None):
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
inst = [parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS,
actions)]
if buffer_id:
mod = parser.OFPFlowMod(datapath=datapath, buffer_id=buffer_id,
priority=priority, match=match,
instructions=inst)
else:
mod = parser.OFPFlowMod(datapath=datapath, priority=priority,
match=match, instructions=inst)
datapath.send_msg(mod)
#set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
def _packet_in_handler(self, ev):
# If you hit this you might want to increase
# the "miss_send_length" of your switch
if ev.msg.msg_len < ev.msg.total_len:
self.logger.debug("packet truncated: only %s of %s bytes",
ev.msg.msg_len, ev.msg.total_len)
msg = ev.msg
datapath = msg.datapath
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
in_port = msg.match['in_port']
pkt = packet.Packet(msg.data)
eth = pkt.get_protocols(ethernet.ethernet)[0]
if eth.ethertype == ether_types.ETH_TYPE_LLDP:
# ignore lldp packet
return
dst = eth.dst
src = eth.src
dpid = datapath.id
self.mac_to_port.setdefault(dpid, {})
self.logger.info("packet in %s %s %s %s", dpid, src, dst, in_port)
# learn a mac address to avoid FLOOD next time.
self.mac_to_port[dpid][src] = in_port
if dst in self.mac_to_port[dpid]:
out_port = self.mac_to_port[dpid][dst]
else:
out_port = ofproto.OFPP_FLOOD
actions = [parser.OFPActionOutput(out_port)]
# install a flow to avoid packet_in next time
if out_port != ofproto.OFPP_FLOOD:
match = parser.OFPMatch(in_port=in_port, eth_dst=dst)
# verify if we have a valid buffer_id, if yes avoid to send both
# flow_mod & packet_out
if msg.buffer_id != ofproto.OFP_NO_BUFFER:
self.add_flow(datapath, 1, match, actions, msg.buffer_id)
return
else:
self.add_flow(datapath, 1, match, actions)
data = None
if msg.buffer_id == ofproto.OFP_NO_BUFFER:
data = msg.data
out = parser.OFPPacketOut(datapath=datapath, buffer_id=msg.buffer_id,
in_port=in_port, actions=actions, data=data)
datapath.send_msg(out)

How realtime capture logs of query from HiveServer2 with python client?

I use modified version of pyhs2 (https://pypi.python.org/pypi/pyhs2) with ability run async queries and additional methods from TCLIService.Client (GetLog, send_GetLog, recv_GetLog) in sources of Hue (https://github.com/cloudera/hue/blob/master/apps/beeswax/gen-py/TCLIService/TCLIService.py#L739)
But when I run TCLIService.Client.GetLog method, there is an error:
$ python example.py
Traceback (most recent call last):
File "example.py", line 85, in <module>
rq = client.GetLog(lq)
File "/Users/toly/hive_streaming/libs/pyhs4/TCLIService/TCLIService.py", line 757, in GetLog
return self.recv_GetLog()
File "/Users/toly/hive_streaming/libs/pyhs4/TCLIService/TCLIService.py", line 773, in recv_GetLog
raise x
thrift.Thrift.TApplicationException: Invalid method name: 'GetLog'
In script I use HiveServer2 from Cloudera VM. Same server, as I quess, used by Hue and it successfully works. In addition I try client_protocol in range from 0 to 7 for creating session.
import time
import sasl
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from thrift.transport.TSocket import TSocket
from thrift.transport.TTransport import TBufferedTransport
from libs.pyhs4.cloudera.thrift_sasl import TSaslClientTransport
from libs.pyhs4.TCLIService import TCLIService
from libs.pyhs4.TCLIService.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,\
TStatusCode, TGetResultSetMetadataReq, TGetColumnsReq, TType, TTypeId, \
TExecuteStatementReq, TGetOperationStatusReq, TFetchOrientation, TCloseOperationReq, \
TCloseSessionReq, TGetSchemasReq, TCancelOperationReq, TGetLogReq
auth = 'PLAIN'
username = 'apanin'
password = 'none'
host = 'cloudera'
port = 10000
test_hql1 = 'select count(*) from test_text'
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("username", username)
saslc.setAttr("password", password)
saslc.init()
return saslc
def get_type(typeDesc):
for ttype in typeDesc.types:
if ttype.primitiveEntry is not None:
return TTypeId._VALUES_TO_NAMES[ttype.primitiveEntry.type]
elif ttype.mapEntry is not None:
return ttype.mapEntry
elif ttype.unionEntry is not None:
return ttype.unionEntry
elif ttype.arrayEntry is not None:
return ttype.arrayEntry
elif ttype.structEntry is not None:
return ttype.structEntry
elif ttype.userDefinedTypeEntry is not None:
return ttype.userDefinedTypeEntry
def get_value(colValue):
if colValue.boolVal is not None:
return colValue.boolVal.value
elif colValue.byteVal is not None:
return colValue.byteVal.value
elif colValue.i16Val is not None:
return colValue.i16Val.value
elif colValue.i32Val is not None:
return colValue.i32Val.value
elif colValue.i64Val is not None:
return colValue.i64Val.value
elif colValue.doubleVal is not None:
return colValue.doubleVal.value
elif colValue.stringVal is not None:
return colValue.stringVal.value
sock = TSocket(host, port)
transport = TSaslClientTransport(sasl_factory, "PLAIN", sock)
client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = client.OpenSession(TOpenSessionReq(username=username, password=password))
session = res.sessionHandle
query1 = TExecuteStatementReq(session, statement=test_hql1, confOverlay={}, runAsync=True)
response1 = client.ExecuteStatement(query1)
opHandle1 = response1.operationHandle
while True:
time.sleep(1)
q1 = TGetOperationStatusReq(operationHandle=opHandle1)
res1 = client.GetOperationStatus(q1)
lq = TGetLogReq(opHandle1)
rq = client.GetLog(lq)
if res1.operationState == 2:
break
req = TCloseOperationReq(operationHandle=opHandle1)
client.CloseOperation(req)
req = TCloseSessionReq(sessionHandle=session)
client.CloseSession(req)
How realtime capture logs of hive query from HiveServer2?
UPD Hive version - 1.2.1

For getting logs of operation used method FetchResults with param fetchType=1 - returning logs.
Example usage:
query1 = TExecuteStatementReq(session, statement=test_hql1, confOverlay={}, runAsync=True)
response1 = client.ExecuteStatement(query1)
opHandle1 = response1.operationHandle
while True:
time.sleep(1)
q1 = TGetOperationStatusReq(operationHandle=opHandle1)
res1 = client.GetOperationStatus(q1)
request_logs = TFetchResultsReq(operationHandle=opHandle1, orientation=0, maxRows=10, fetchType=1)
response_logs = client.FetchResults(request_logs)
print response_logs.results
if res1.operationState == 2:
break

Python weird code error

I have an IRC bot and I'm trying to get information for game server (GTA SA Multiplayer).
I have ready-to-use query, but I can't implement it into my bot. It works if I try to load the same script, but without getting it into bot's structure. The error it gives me is
NameError: name 'ip' is not defined
I've tried adding the ip address as argument in def(inp,say=None), but it still didn't work. That's the code:
from util import Query
from util import hook
import sys
#hook.command
def serverinfo(inp,ip="",port="",say=None):
ip = "78.129.221.58"
port = 7777
if len(sys.argv) >= 3:
ip = str(sys.argv[1])
port = int(sys.argv[2])
query = Query(ip,port)
info = query.GetInformation()
say(info)
if info['players'] <= 100:
say(query.GetPlayers())
say(query.GetDetailedPlayers())
else: say('can\' get players because players is above 100')
say(query.Ping())
query.Close()
That's Query that I import:
import socket, struct, random, datetime
from cStringIO import StringIO
class Query:
def __init__(self, ip, port):
self.ip, self.port = socket.gethostbyname(ip), port
self.data = StringIO("")
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.connect((ip, port))
self.sock.settimeout(1)
def CreatePacket(self, opcode):
ips = self.ip.split('.');
packet = "SAMP{0}{1}{2}{3}{4}{5}{6}".format(chr(int(ips[0])), chr(int(ips[1])), chr(int(ips[2])), chr(int(ips[3])), chr(self.port & 0xFF), chr(self.port >> 8 & 0xFF), opcode)
if opcode == 'p':
packet += struct.pack("BBBB", random.randint(0, 255), random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
return packet
def GetInformation(self):
try:
self.sock.send(self.CreatePacket('i'))
info = {}
self.data = StringIO(self.sock.recv(2048))
self.data.read(11)
info['passworded'] = struct.unpack('?', self.data.read(1))[0]
info['players'] = struct.unpack('h', self.data.read(2))[0]
info['maxplayers'] = struct.unpack('h', self.data.read(2))[0]
info['hostname'] = self.data.read(struct.unpack('i', self.data.read(4))[0])
info['gamemode'] = self.data.read(struct.unpack('i', self.data.read(4))[0])
info['mapname'] = self.data.read(struct.unpack('i', self.data.read(4))[0])
except socket.timeout:
info['error'] = 1
return info
def GetRules(self):
try:
self.sock.send(self.CreatePacket('r'))
rules = {}
self.data = StringIO(self.sock.recv(2048))
self.data.read(11)
rulecount = struct.unpack('h', self.data.read(2))[0]
for i in range(rulecount):
name = self.data.read(struct.unpack('b', self.data.read(1))[0])
rules[name] = self.data.read(struct.unpack('b', self.data.read(1))[0])
except socket.timeout:
rules['error'] = 1
return rules
def GetPlayers(self):
try:
self.sock.send(self.CreatePacket('c'))
players = []
self.data = StringIO(self.sock.recv(2048))
self.data.read(11)
playercount = struct.unpack('h', self.data.read(2))[0]
for i in range(playercount):
name = self.data.read(struct.unpack('b', self.data.read(1))[0])
players.append([name, struct.unpack('i', self.data.read(4))[0]])
except socket.timeout:
players = {'error': 1}
return players
def GetDetailedPlayers(self):
try:
self.sock.send(self.CreatePacket('d'))
players = []
self.data = StringIO(self.sock.recv(2048))
self.data.read(11)
playercount = struct.unpack('h', self.data.read(2))[0]
for i in range(playercount):
playerid = struct.unpack('b', self.data.read(1))[0]
name = self.data.read(struct.unpack('b', self.data.read(1))[0])
score = struct.unpack('i', self.data.read(4))[0]
ping = struct.unpack('i', self.data.read(4))[0]
players.append([playerid, name, score, ping])
except socket.timeout:
players = {'error': 1}
return players
def Close(self):
self.sock.close()
def Ping(self):
packet = self.CreatePacket('p')
a = datetime.datetime.now()
self.sock.send(packet)
self.sock.recv(2048)
b = datetime.datetime.now()
c = b - a
return int((c.days * 24 * 60 * 60 + c.seconds) * 1000 + c.microseconds / 1000.0)
class Rcon:
def __init__(self, ip, port, password):
self.ip, self.port, self.password = socket.gethostbyname(ip), port, password
self.data = StringIO("")
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.connect((ip, port))
self.sock.settimeout(0.5)
def CreatePacket(self, opcode, password, command):
ips = self.ip.split('.');
packet = "SAMP{0}{1}{2}{3}{4}{5}{6}{7}{8}{9}{10}{11}{12}".format(chr(int(ips[0])), chr(int(ips[1])), chr(int(ips[2])), chr(int(ips[3])), chr(self.port & 0xFF), chr(self.port >> 8 & 0xFF), opcode, chr(len(password) & 0xFF), chr(len(password) >> 8 & 0xFF), password, chr(len(command) & 0xFF), chr(len(command) >> 8 & 0xFF), command)
return packet
def Send(self, command):
self.sock.send(self.CreatePacket('x', self.password, command))
output = []
while 1:
try:
self.data = StringIO(self.sock.recv(2048))
self.data.read(11)
strlen = struct.unpack('h', self.data.read(2))[0]
if strlen == 0: break
output += [self.data.read(strlen)]
except: break;
return output
def Close(self):
self.sock.close()
Any ideas?
Edit: After some changes I did, gives me the following error:
query = Query(ip,port)
TypeError: 'module' object is not callable
I've basically changed the location of ip and port, moved it out of the serverinfo command.
from util import Query
from util import hook
ip = "78.129.221.58"
port = 7777
query = Query(ip,port)
info = query.GetInformation()
#hook.command
def serverinfo(inp,ip="",port="",say=None):
say(info)
if info['players'] <= 100:
say(query.GetPlayers())
say(query.GetDetailedPlayers())
else: say('can\' get players because players are above 100')
say(query.Ping())
query.Close()

Try adding this to the start of the python code :
ip = "78.129.221.58"
port = 7777
Try
query = Query.Query(ip, port)

The problematic line is this one:
query = Query(ip,port)
If the initialization using the command line parameters fail or if you pass in insufficient arguments, ip will not be initialized resulting in the said error.
Also, a better way of testing out scripts is something like this:
def main():
if len(sys.argv) >= 3:
ip = str(sys.argv[1])
port = int(sys.argv[2])
query = Query(ip,port)
# start using query
else:
raise Exception("Insufficient args passed in")
if __name__ == '__main__':
main()
Raising an exception/error or printing out something guarantees that you know that insufficient args are passed in. Also, maybe move all that code out in the open to some function?