Develop Reference

Python - Send Integer or String to Spark-Streaming

I can send my data through CSV file. First, write my random numbers into CSV file then send it, but is it possible to send it directly?
my socket code:
import socket
host = 'localhost'
port = 8080
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
s.listen(1)
while True:
print('\nListening for a client at',host , port)
conn, addr = s.accept()
print('\nConnected by', addr)
try:
print('\nReading file...\n')
while 1:
out = "test01"
print('Sending line', line)
conn.send(out)
except socket.error:
print ('Error Occured.\n\nClient disconnected.\n')
conn.close()
spark streaming code:
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
sc = SparkContext("local[2]","deneme")
ssc = StreamingContext(sc, 10)
socket_stream = ssc.socketTextStream("localhost",8080)
random_integers = socket_stream.window( 30 )
digits = random_integers.flatMap(lambda line: line.split(" ")).map(lambda digit: (digit, 1))
digit_count = digits.reduceByKey(lambda x,y:x+y)
digit_count.pprint()
ssc.start()

This is because socket blocks sending the data and never moves on. The most basic solution is to send some amount of data and close the connection:
import socket
import time
host = 'localhost'
port = 50007
i = 0
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
s.listen(1)
try:
while True:
conn, addr = s.accept()
try:
for j in range(10):
conn.send(bytes("{}\n".format(i), "utf-8"))
i += 1
time.sleep(1)
conn.close()
except socket.error: pass
finally:
s.close()
To get something more interesting check non-blocking mode with timeouts.

Continuous listening to TCP port

I've made a code which is able to receive data from the port over TCP protocol. I receive data from ESP8266 every 15 minutes, and then ESP goes to a deepSleep mode. How to change it to make it work continuosly? I wanted to create a new connection in while loop, but it doesn't work.
My code
import sys
import socket
TCP_IP = '192.168.42.1'
TCP_PORT = 8888
BUFFER_SIZE = 1024
param = []
i=0
#s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.bind((TCP_IP,TCP_PORT))
#s.listen(1)
#print 'Listening for client...'
#conn, addr = s.accept()
#print 'Connection address:', addr
while 1:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP,TCP_PORT))
s.listen(1)
print 'Listening for client...'
conn, addr = s.accept()
print 'Connection address:', addr
data = conn.recv(BUFFER_SIZE)
if data == ";" :
conn.close()
print "Received all the data"
i=0
for x in param:
print x
#break
elif data:
print "received data: ", data
param.insert(i,data)
i+=1
#print "End of transmission"
EDIT:
My code after modification.
import sys
import socket
TCP_IP = '192.168.42.1'
TCP_PORT = 8888
BUFFER_SIZE = 1024
param = []
i=0
#s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.bind((TCP_IP,TCP_PORT))
#s.listen(1)
#print 'Listening for client...'
#conn, addr = s.accept()
#print 'Connection address:', addr
while 1:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP,TCP_PORT))
s.listen(1)
while 1:
print 'Listening for client...'
conn, addr = s.accept()
print 'Connection address:', addr
data = conn.recv(BUFFER_SIZE)
if data == ";" :
conn.close()
print "Received all the data"
i=0
for x in param:
print x
#break
elif data:
print "received data: ", data
param.insert(i,data)
i+=1
#print "End of transmission"
s.close()
I created second while loop. I can listen continuously now, but I receive only one packet from the ESP (ESP send 9 packets). How to solve that issue?

If you want to continuously listen for connections and data from your remote end, you can achieve this using select()
A modified version of your code that uses select() is shown below. This will also handle the remote end closing the connection:
import sys
import socket
import select
TCP_IP = '127.0.0.1'
TCP_PORT = 8888
BUFFER_SIZE = 1024
param = []
print 'Listening for client...'
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind((TCP_IP,TCP_PORT))
server.listen(1)
rxset = [server]
txset = []
while 1:
rxfds, txfds, exfds = select.select(rxset, txset, rxset)
for sock in rxfds:
if sock is server:
conn, addr = server.accept()
conn.setblocking(0)
rxset.append(conn)
print 'Connection from address:', addr
else:
try:
data = sock.recv(BUFFER_SIZE)
if data == ";" :
print "Received all the data"
for x in param:
print x
param = []
rxset.remove(sock)
sock.close()
else:
print "received data: ", data
param.append(data)
except:
print "Connection closed by remote end"
param = []
rxset.remove(sock)
sock.close()
NB I've replaced your IP address with the loopback but you get the idea.
Hope this may be helpful.

Python socket.error: [Errno 104] Connection reset by peer

I have a code where there are 13 clients that have to connect to the server. Then the server does some counting on the data given by the client. After that the roles turns around - the server becomes a client and clients become servers to receive the data.
The thing is that when trying to do the first connection, that is when the 13 clients try to connect to the server I keep getting this error: [Errno 104] Connection reset by peer. I tried some workarounds for example trying to connect 5 times in a second interval but nothing works.
Here my code:
server.py
import socket, pickle, numpy as np
import struct
import math
while 1:
HOST = ''
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(13)
adresses = []
ports = []
i = 0
print("receiving...")
while i < 13:
i += 1
#wait to accept a connection - blocking call
conn, addr = s.accept()
print ('Connected with ', addr)
adresses.append(addr[0])
buf = b''
while len(buf) < 4:
buf += conn.recv(4 - len(buf))
length = struct.unpack('>I', buf)[0]
data = b''
l = length
while l > 0:
d = conn.recv(l)
l -= len(d)
data += d
if not data: break
M = np.loads(data)
if i == 1:
L = M[0]
else:
L += M[0]
ports.append(M[1])
conn.close()
s.close()
L /= 993040
packet = pickle.dumps(L)
length = struct.pack('>I', len(packet))
packet = length + packet
print("sending...")
for kl, addr in enumerate(adresses):
HOST = addr
PORT = 50007 + ports[kl]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.sendall(packet)
s.close()
client.py
def connection(centers, kl):
HOST = "192.168.143.XX"
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(3600)
try:
s.connect((HOST, PORT)) # HERE IS AN ERROR
s.settimeout(None)
packet = pickle.dumps([centers, kl]) ## ???
length = struct.pack('>I', len(packet))
packet = length + packet
s.sendall(packet) # OR HERE IS AN ERROR
s.close()
except Exception as e:
print(e)
print('error ', kl)
s.close()
return np.zeros(centers.shape)
HOST = ''
PORT = 50007 + kl
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(2)
i = 0
while i < 1:
#wait to accept a connection - blocking call
conn, addr = s.accept()
i += 1
print ('Connected with ', addr)
buf = b''
while len(buf) < 4:
buf += conn.recv(4 - len(buf))
length = struct.unpack('>I', buf)[0]
data = b''
l = length
while l > 0:
d = conn.recv(l)
l -= len(d)
data += d
if not data: break
new_centers = np.loads(data)
conn.close()
s.close()
return new_centers
aa = 0
for k in range(99):
print(k)
centers = some_function(centers)
time.sleep(60)
centers1 = connection(centers, i)
aa = 0
while not (centers1.any()) and aa < 5:
time.sleep(1)
centers1 = connection(centers, i)
aa += 1
centers = centers1
The thing is all of the 13 clients HAVE TO connect to the server or it won't proceed to the next iteration.
I'm using Python 3.4.
Please help.
Update:
I have added threads but the error remains:
[Errno 104] Connection reset by peer
server.py
import socket, pickle, numpy as np
import struct
import math
from multiprocessing.pool import ThreadPool
def clientthread(conn, L):
buf = b''
while len(buf) < 4:
buf += conn.recv(4 - len(buf))
length = struct.unpack('>I', buf)[0]
data = b''
l = length
while l > 0:
d = conn.recv(l)
l -= len(d)
data += d
M = np.loads(data)
return(M)
j = 0
while 1:
HOST = ''
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(2)
#print('0')
adresses = []
ports = []
i = 0
print("receiving...")
while i < 13:
i += 1
#wait to accept a connection - blocking call
conn, addr = s.accept()
print ('Connected with ', addr)
adresses.append(addr[0])
pool = ThreadPool(processes=13)
async_result = pool.apply_async(clientthread, (conn, i,))
M = async_result.get()
conn.close()
if i == 1:
L = M[0]
else:
L += M[0]
ports.append(M[1])
s.close()
L /= 993040
packet = pickle.dumps(L)
length = struct.pack('>I', len(packet))
packet = length + packet
for kl, addr in enumerate(adresses):
HOST = addr
PORT = 50007 + ports[kl]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.sendall(packet)
s.close()

It seems that the clients were connected to the server but they encountered with " [Errno 104] Connection reset by peer" exception when they tried to send data. For the first time, Python raises "[Errno 104] Connection reset by peer" exception, then for the second time and more you would get "[Errno 32] Broken pipe" exception on the client side.
This can mean that the server is up and listening on the port (otherwise, you would get "[Errno 111] Connection refused" exception on the client side". This also means that the server is crashed before closing the connection since if the connection was closed on the server side before sending data on the client side, the client would encounter with "[Errno 32] Broken pipe" exception.
"Connection reset by peer" is the TCP/IP equivalent of slamming the phone back on the hook. It's more polite than merely not replying, leaving one hanging. But it's not the FIN-ACK expected of the truly polite TCP/IP converseur. (From another stackoverflow answer)

You are going to have to use threading on the server.
For a simple explanation see: http://www.binarytides.com/python-socket-server-code-example/
The guts of the simple example given there is:
import socket
import sys
from thread import *
HOST = '' # Symbolic name meaning all available interfaces
PORT = 8888 # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print 'Socket created'
#Bind socket to local host and port
try:
s.bind((HOST, PORT))
except socket.error as msg:
print 'Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1]
sys.exit()
print 'Socket bind complete'
#Start listening on socket
s.listen(10)
print 'Socket now listening'
#Function for handling connections. This will be used to create threads
def clientthread(conn):
#Sending message to connected client
conn.send('Welcome to the server. Type something and hit enter\n') #send only takes string
#infinite loop so that function do not terminate and thread do not end.
while True:
#Receiving from client
data = conn.recv(1024)
reply = 'OK...' + data
if not data:
break
conn.sendall(reply)
#came out of loop
conn.close()
#now keep talking with the client
while 1:
#wait to accept a connection - blocking call
conn, addr = s.accept()
print 'Connected with ' + addr[0] + ':' + str(addr[1])
#start new thread takes 1st argument as a function name to be run, second is the tuple of arguments to the function.
start_new_thread(clientthread ,(conn,))
s.close()

I had this exact error when trying to connect to a remote redis instance from python, because I had mistakenly left Fiddler running and it was interfering with the requests.

Server that echos the client number?

Trying to make a server that tells the client what number he/she is. For example, once you connect it should say something like "Welcome client #5" or something along those lines. Right now I'm just trying to write it so that it simply reads a line in and echos it back. Im stuck on as far as getting it to show the clients number.
import socket
import sys
host = ''
port = 37373
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(1)
while 1:
s, address = s.accept()
data = s.recv(1024)
if data:
s.send(data)
s.close()

that is
import socket
import sys
Clinet_number = 0
host = ''
port = 37373
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(10) # number of queued connections
while 1:
Client_number += 1
s, address = s.accept()
data = s.recv(1024)
if data:
s.send(str(Client_number))
s.close()

Cannot Get Strings Across in a Socket Program

I have been looking at some code for a small chat program that I found online. It was originally written for 2.7, but it seems to work with 3.2. The only problem is that I cannot send strings, only numbers:
The chat.py file source code:
from socket import *
HOST = ''
PORT = 8000
s = socket(AF_INET, SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print ('Connected by ' + str(addr))
i = True
while i is True:
data = conn.recv(1024)
print ("Received " + repr(data))
reply = str(input("Reply: "))
conn.send(reply)
conn.close()
And the client.py source file:
from socket import *
HOST = ''
PORT = 8000
s = socket(AF_INET, SOCK_STREAM)
s.connect((HOST, PORT)) # client-side, connects to a host
while True:
message = str(input("Your Message: "))
s.send(message)
print ("Awaiting reply...")
reply = s.recv(1024) # 1024 is max data that can be received
print ("Received " + repr(reply))
s.close()
When I run these using two separate terminals, they work, but do not send strings.
Thank you

When you work with sockets, the message you're passing around should probably be in bytes, b'bytes'. In Python 2.x, a str is actually what a bytes is in Python 3.x
So your message should be something like:
message = b'Message I want to pass'
Check here http://docs.python.org/3.3/library/stdtypes.html for more information.
According to http://docs.python.org/3/library/functions.html#input input returns a str, which means you'll have to encode message into bytes as such:
message = message.encode()
Do verify that this is the right approach to convert str to bytes by checking the type of message.

Your socket code is correct, it was just failing due to an unrelated error due to raw_input vs input. You probably intended to read a string from the shell instead of reading a string and trying to evaluate it as Python code which is what input does.
Try this instead:
chat.py
from socket import *
HOST = ''
PORT = 8000
s = socket(AF_INET, SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print ('Connected by ' + str(addr))
i = True
while i is True:
data = conn.recv(1024)
print ("Received " + repr(data))
reply = str(raw_input("Reply: "))
conn.send(reply)
conn.close()
client.py
from socket import *
HOST = ''
PORT = 8000
s = socket(AF_INET, SOCK_STREAM)
s.connect((HOST, PORT)) # client-side, connects to a host
while True:
message = str(raw_input("Your Message: "))
s.send(message)
print ("Awaiting reply...")
reply = s.recv(1024) # 1024 is max data that can be received
print ("Received " + repr(reply))
s.close()