Develop Reference

I cannot send an ack confirmation from my client after receiving an ACK through socket

I have a server that's always listening and waiting for a connection. When a client connect then sends a file to it, it receives it, saves it and then send back an ACK file, my client send back a confirmation that he received the ACK and finally my server close the connection and my client too.
I had an issue while attempting to receive the ACK from my client, but now it's resolved.
BUT, now that I shutdown my socket connection with SHUT_WR (that's telling that we will stop sending after this, if I'm right) I cannot resend my confirmation.
I can't figure out how that's working and how can I:
From my client
Send a file to my server
Receive the ACK from my server
Resend a confirmation
From my server
Receive a file
Send an ACK
Receive the confirmation
I'm stuck. My server is working unless I try to receive or send something.
The following code blocks are my actual files
client.py
import socket
import os
import random
from socket import SHUT_WR
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 8192
HOST = "127.0.0.1"
PORT = 8000
files = ["test1.HL7","test2.HL7","test3.HL7","test4.HL7","test5.HL7","test6.HL7","test7.HL7","test8.HL7"]
fileName = f".\\ClientFiles\\{files[random.randrange(1,8)]}"
filesize = os.path.getsize(fileName)
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print(f"[+] Connecting to {HOST}:{PORT}")
socket.connect((HOST, PORT))
print("[+] Connected.")
socket.send(f"{fileName}{SEPARATOR}{filesize}".encode())
# opening file
with open(fileName, "rb") as f:
print("[*] Reading")
while True:
# reading bytes
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
# Transmitting is done
print("[+] File reading is done.")
break
# send all the buffer
socket.sendall(bytes_read)
print(f"[+] File {fileName} sent")
socket.shutdown(SHUT_WR)
print("[*] Waiting for an acknowledgment")
data = socket.recv(BUFFER_SIZE)
print("[+] Acknoledgment recieved")
print(data.decode())
socket.sendall(data.decode().split('|')[9].encode())
print("[+] Acknowledgment confirmation sent")
print("[*] Closing")
socket.close()
server.py
import itertools
import socket
import signal
import ntpath
from pathlib import Path
from consts import *
from helper import *
# Setting Dicionnaries for HL7 ACK Requirements
MSH = {0:'MSH',2:'', 3:'', 4:'', 5:'', 6:'', 7:'', 8:'', 9:'', 10:'', 11:'',12:'', 13:'', 14:'', 15:'NE', 16:'AL', 17:'', 18:'', 19:'', 20:'', 21:''}
MSA = {0:'MSA', 1:'AA', 2:''}
ACK = {'MSH':{}, 'MSA':{}}
def hl7_ack_generator():
"""Generate the Acknowledgement file and sends it to the client
Returns:
String: Returns the Acknowledgment filename
"""
content = ""
# Switch sender-receiver
MSH[3], MSH[5], MSH[4], MSH[6] = MSH[5], MSH[3], MSH[6], MSH[4]
# Set the message type.
# If possible get the message trigger to return the same trigger
try:
MSH[9] = f"ACK^{check_msh_9_trigger_event(MSH[9].decode().split('^'))}^ACK"
except:
MSH[9] = "ACK"
# Set MSH values
for param in MSH:
ACK['MSH'][param] = MSH.get(param)
# Set MSA values
for param in MSA:
ACK['MSA'][param] = MSA.get(param)
# Create the ACK message
# Handle integers & bytes in each fields
# Get MSH values
for i in range(0,21) :
if i != 1:
if ACK['MSH'][i]:
# Generate a message id based on recieved message timestamp and message id
# not exceeding 20 chars
if i == 10:
ACK['MSH'][10] = ACK['MSH'][7].decode() + ACK['MSH'][10].decode()
if len(ACK['MSH'][10]) > 20:
ACK['MSH'][10] = ACK['MSH'][10][:20]
content += ACK['MSH'][10]
else:
try:
content += ACK['MSH'][i].decode()
except:
if not ACK['MSH'][i] == None:
content += ACK['MSH'][i]
content += '|'
content += "\r"
# Get MSA values
for i in range(0,3):
try:
content += ACK['MSA'][i].decode()
except:
if not ACK['MSA'][i] == None:
content += ACK['MSA'][i]
content += "|"
# Create the ACK filename
filename = ACK['MSH'][10] + "_ACK.HL7"
# create the ACK file and write its content
with open(Path(SERVER_ACK_FILES_FOLDER + filename), "w") as f:
f.write(content)
f.close()
return filename
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# test the binding
try:
socket.bind((SERVER_HOST, SERVER_PORT))
except socket.error as error:
print('Bind failed. Error Code : '
+ str(error[0]) + ' Message '
+ error[1])
exit()
# Handle ctrl+c
def signal_handler(sign, frame):
print('[*] Shutting down')
exit(0)
while signal.signal(signal.SIGINT, signal_handler):
# connection limit(5 connection try then deny)
socket.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# accept the connection if there is any
client_socket, address = socket.accept()
# Below code is executed if sender is connected
print(f"[+] {address} is connected.")
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE)
filename, bytes_read = received.split(SEPARATOR.encode())
# get the file size
fileSize = bytes(itertools.takewhile(lambda i: bytes((i,)).isdigit(), bytes_read))
bytes_read = bytes_read[len(fileSize):]
#convert to integer
fileSize = int(fileSize)
filename = filename.decode()
# remove absolute path if there is
filename = ntpath.basename(filename)
# start receiving the file from the socket and writing to the file stream
with open(Path(SERVER_FILES_FOLDER + filename), "wb") as f:
print("[+] File received")
while True:
# write to the file the bytes we just received
f.write(bytes_read)
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if bytes_read.startswith(b'MSH'):
messageHeader = bytes_read.partition(b'\r')[0].split(b'|')
j = 0
i = 2
for i in range(2,17):
j+=1
# Exclude MSH fields(constant and/or unwanted)
if i not in (15,16):
MSH[i]= messageHeader[j]
#Get message ID
if i == 10:
MSA[2] = messageHeader[j]
ackFilename = hl7_ack_generator()
if not bytes_read:
# file transmitting is done
print("[+] File transfert is done")
break
with open(Path(SERVER_ACK_FILES_FOLDER + ackFilename), "rb") as f:
while True:
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
print("[+] Acknoledgment Sent")
break
client_socket.sendall(bytes_read)
confirmation = client_socket.recv(BUFFER_SIZE)
print(confirmation)
print("[+] Confirmation received")
print("[*] Closing conneciton")
client_socket.close()
socket.close
testX.hl7
MSH|^~\&|ADT1|MCM|LABADT|MCM|198808181126|SECURITY|ADT^A04|MSG00001|P|2.4
EVN|A01-|198808181123
PID|||PATID1234^5^M11||JONES^WILLIAM^A^III||19610615|M-||2106-3|1200 N ELM STREET^^GREENSBORO^NC^27401-1020|GL|(919)379-1212|(919)271-3434~(919)277-3114||S||PATID12345001^2^M10|123456789|9-87654^NC
NK1|1|JONES^BARBARA^K|SPO|||||20011105
NK1|1|JONES^MICHAEL^A|FTH
PV1|1|I|2000^2012^01||||004777^LEBAUER^SIDNEY^J.|||SUR||-||1|A0-
AL1|1||^PENICILLIN||PRODUCES HIVES~RASH
AL1|2||^CAT DANDER
DG1|001|I9|1550|MAL NEO LIVER, PRIMARY|19880501103005|F||
PR1|2234|M11|111^CODE151|COMMON PROCEDURES|198809081123
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^SMITH^ELLEN|199505011201
GT1|1122|1519|BILL^GATES^A
IN1|001|A357|1234|BCMD|||||132987
IN2|ID1551001|SSN12345678
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^ELLEN|199505011201
Thanks for the attention!

I changed my server into this:
final_size = b''
while True:
# write to the file the bytes we just received
f.write(bytes_read)
final_size += bytes_read
if len(final_size) >= fileSize:
# file transmitting is done
print("[+] File transfert is done")
break
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if bytes_read.startswith(b'MSH'):
messageHeader = bytes_read.partition(b'\r')[0].split(b'|')
j = 0
i = 2
for i in range(2,17):
j += 1
# Exclude MSH fields(constant and/or unwanted)
if i not in (15,16):
MSH[i]= messageHeader[j]
#Get message ID
if i == 10:
MSA[2] = messageHeader[j]
ackFilename = hl7_ack_generator()
And my client to this:
print(f"[+] File {fileName} sent")
print("[*] Waiting for an acknowledgment")
data = sock.recv(BUFFER_SIZE)
print("[+] Acknoledgment recieved")
sock.sendall(data.decode().split('|')[9].encode())
print("[+] Acknowledgment confirmation sent")
print("[*] Closing")
sock.close()

Decrypt message with XOR and respond over a socket

I am attempting to complete a challenge that states:
We've noticed that the aliens are sending messages between their ships, we think they're using XOR to encrypt the messages, and we've intercepted a key.
Set up a server listening on ("localhost", 10000) to intercept one of the alien messages. When you do perform a bitwise XOR on the message with the key "attackthehumans" and then respond with the encrypted data.
Tip: Read the response to get the flag.
After some research, I was able to come up with the following code. However, when I run it in the challenge's code editor, the only feedback I receive is "Error trying to connect to your server and recieving message back."
import socket
# Function to xor strings
def xor_string(string):
key = "attackthehumans"
bit_key = ''.join(format(ord(i), 'b') for i in key)
bit_data = ''.join(format(ord(i), 'b') for i in string)
xor_string = str(0)
for i in range(len(bit_key)):
if bit_data[i] == bit_key[i]:
xor_string = xor_string + str(0)
else:
xor_string = xor_string + str(1)
return xor_string
# Sets up server on localhost and port 10000
print("Setting up server...")
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(("localhost", 10000))
print("Server set up. Listening for connections...")
server.listen()
conn, addr = server.accept()
print("Connected by: {}".format(addr))
# Once connection is established, server receives data, XOR's it, sends it back, and
# waits for response to get the flag.
with conn:
print("Receiving data from connection... ")
data = conn.recv()
data = data.decode()
print("Data received: {}".format(data.decode()))
xor_data = xor_string(data)
conn.sendall(xor_data.encode())
response = conn.recv()
response = response.decode()
print(response)
I am not sure what I'm doing wrong. I can't figure out if it's the socket or the xor_string function. Any help would be appreciated.

This should be enough to do your XORing:
def xor_string(string):
key = b"attackthehumans"
key = key * (len(string)//len(key)+1)
res = bytes([k^s for (k,s) in zip(key,string)])
return res
Then your main code becomes:
print("Receiving data from connection... ")
data = conn.recv()
print("Data received:", data)
xor_data = xor_string(data)
conn.sendall(xor_data)
print(conn.recv())

How do I troubleshoot this error: OSError: [Errno 9] Bad file descriptor

I am trying to figure out where the problem is coming from between my client and server files. The client receives the correct calculation done by the TCP server. However, the TCP server continues to throw an error after performing the task.
add_server.py
# This is add_server.py script
import socket
host = socket.gethostname()
port = 8096
s = socket.socket()
s.bind((host, port))
s.listen(5)
print('Waiting for connection...')
conn, addr = s.accept()
while True:
data = conn.recv(1024) # Receive data in bytes
print(data)
decoded_data = data.decode('utf-8') # Decode data from bystes to string
print(data)
d = decoded_data.split(",") # Split the received string using ',' as separator and store in list 'd'
add_data = int(d[0]) + int(d[1]) # add the content after converting to 'int'
conn.sendall(str(add_data).encode('utf-8')) # Stringify results and convert to bytes for transmission (String conversion is a must)
conn.close() # Close connection
add_client.py
# This add_client.py script
import socket
host = socket.gethostname()
port = 8096
s = socket.socket()
s.connect((host, port))
a = input('Enter first number: ')
b = input('Enter second number: ')
c = a + ', ' + b # Generate string from numbers
print('Sending string {} to sever for processing...'.format(c))
s.sendall(c.encode('utf-8')) # Converst string to bytes for transmission
data = s.recv(1024).decode('utf-8') # Receive server response (addition results) and convert from bystes to string
print(data) # Convert 'string' data to 'int'
s.close() # close connection
Full traceback
Traceback (most recent call last):
File "/home/sharhan/AWS/AWS-PERSONAL-COURSES/linux-networking-and-troubleshooting/python-networking/add_server.py", line 17, in <module>
data = conn.recv(1024) # Receive data in bytes
OSError: [Errno 9] Bad file descriptor

You are closing the socket inside the while loop in this line
while True:
data = conn.recv(1024)
# Rest of the code
conn.close()
So the next time you try to receive the data with conn.recv it results in an error.
To fix this simply close the connection after you're done receiving all the data.
while True:
data = conn.recv(1024)
# Rest of the code
conn.close()

tcp lost packet python [duplicate]

This question already exists:
TCP socket python
Closed 2 years ago.
i am using tcp socket for forwarding data (as class with type of request, and data) between server and client. When i send 6 object of order from client, the server get only 5 object of order, 1 is lost, but tcp is reliable protocol... (sometimes the server get 6 but usually get 5) my question is why an object lost in forwarding by tcp?
Here's the client code
PORT = 3000
SIZE = 4096
HOST = '127.0.0.1'
soc = None
ing_map = None
def connect_to_client():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
global soc
soc = s
def send_object(data):
if soc is None:
raise NotImplementedError # There is no connection
else:
# make data as bytes
msg = pickle.dumps(data)
msg = bytes(f"{len(msg):<{HEADERSIZE}}", 'utf-8') + msg
print(sys.getsizeof(msg))
soc.send(msg)
def get_object(req):
if soc is None:
raise NotImplementedError # There is no connection
else:
# unpickle the data
send_object(req)
if soc is None:
raise NotImplementedError # There is no connection
else:
# unpickle the data
data = b''
while True:
part = soc.recv(SIZE)
data += part
if len(part) < SIZE:
break
full_msg = data
try:
data = pickle.loads(full_msg[HEADERSIZE:])
except EOFError:
data = None
return data
def send_order(order):
if order is not None:
send_object(Soc_request(Request.P_ORDER,order))
def main():
global ing_map
connect_to_client()
ing_map = get_object(Soc_request(Request.G_ING_MAP, None))
#send order
burger = Burger(ing_map)
salad = Salad(ing_map)
burger.add_ingredient('ham')
o1 = Order(Priority.NORMAL)
o2 = Order(Priority.NORMAL)
o3 = Order(Priority.VIP)
o4 = Order(Priority.VIP)
o5 = Order(Priority.PLUS)
o6 = Order(Priority.PLUS)
o1.meals_lst.append(burger)
o1.meals_lst.append(burger)
o1.meals_lst.append(burger)
o3.meals_lst.append(burger)
send_order(o1)
send_order(o2)
send_order(o3)
send_order(o4)
send_order(o5)
send_order(o6)
soc.close()
Here's the server code
HOST = '127.0.0.1'
PORT = 3000
SIZE = 1000
HEADERSIZE = 10
WORKERS = 2
conn = None
addr = None
soc = None
order_m = None
def create_connection():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen()
global conn, addr
global soc
soc = s
conn, addr = s.accept()
def send_object(data):
if soc is None:
raise NotImplementedError # There is no connection
else:
# make data as bytes
global conn
msg = pickle.dumps(data)
msg = bytes(f"{len(msg):<{HEADERSIZE}}", 'utf-8') + msg
conn.send(msg)
def get_object():
global conn
if conn is None:
raise NotImplementedError # There is no connection
else:
# unpickle the data
data = b''
while True:
part = conn.recv(SIZE)
data += part
if len(part) < SIZE:
break
full_msg = data
try:
data = pickle.loads(full_msg[HEADERSIZE:])
except EOFError:
data = None
return data
def main():
create_connection()
# Initialize objects
global order_m
ing_map = Ingredient_map()
order_m = OrderManager()
while True:
msg = get_object()
if msg is None:
pass
elif msg.req == Request.G_ING_MAP: # get ingredient map
send_object(ing_map.instance.map)
elif msg.req == Request.P_ORDER:
order_m.add_order(msg.data)
print(msg.data)
# end while
soc.close()
if __name__ == "__main__":
main()
The output of the server is
order 0
order 1
order 3
order 4
order 5
but order 2 lost!
why? what should i fix that all the orders will arrive ?
Request is an enum for request type.

Let's visualize the 6 pickled objects on their way from the client to the server:
server <- 1112222233334445555566 <- client
A TCP connection is one stream of data. There are no boundaries. The server has to split the data to individual parts and unpickle each part:
111 22222 3333 444 55555 66
In order to be able to do that, you must introduce some high level structure to the data. I can't run your code, but I don't see anything like that there. Instead the server just reads into a fixed size buffer. The reality is little bit complicated, but I think this is sometimes happening:
1112 2222 3333 4445 5555 66
The objects 1, 3, 4 and 6 were transferred in one piece, but 2 and 5 were divided into two parts and these parts cannot be separately unpickled.
I think you need to put the length of the pickled data to the header. Then the server should first read the header (fixed size), extract the length and then read and unpickle as many following bytes (not less, not more).
Also this fragment:
try:
data = pickle.loads(full_msg[HEADERSIZE:])
except EOFError:
data = None
hides errors. Do not blame TCP for losing packets when you silently discard data.

As #VPfB mentioned, you are sending the header size but never parsing the size. You really should restructure your code, however, here is a working example using your code.
In your client code, change your get_object function to this.
def get_object(req):
if soc is None:
raise NotImplementedError # There is no connection
else:
# unpickle the data
send_object(req)
if soc is None:
raise NotImplementedError # There is no connection
else:
# unpickle the data
# this initial recv only receives the header length
length = int(soc.recv(HEADERSIZE).decode('utf-8'))
data = b''
while True:
# here we receive a minimum of the standard recv size
# and the diff between your already received data
# and the full length, whichever is smaller
part = soc.recv(min(SIZE, length - len(data)))
data += part
# if the length that was specifed in the header has been received
# we know we have everything
if length == len(data):
break
full_msg = data
try:
data = pickle.loads(full_msg)
except EOFError:
data = None
return data
Now we'll make almost identical changes in your server code.
def get_object():
global conn
if conn is None:
raise NotImplementedError # There is no connection
else:
# unpickle the data
# the only difference here is
# when we receive nothing, we know it's time to wait for another connect
length = conn.recv(HEADERSIZE).decode('utf-8')
if len(length):
length = int(length)
data = b''
while True:
part = conn.recv(min(SIZE, length - len(data)))
data += part
if length == len(data):
break
full_msg = data
try:
data = pickle.loads(full_msg)
except EOFError:
data = None
return data
else:
# wait for another connection
conn, addr = soc.accept()
In reality a server typically behaves like this:
# do this forever
while True:
# wait for an incoming connection
conn, addr = soc.accept()
# get the length of the data the connection is sending
length = int(soc.recv(HEADERSIZE).decode('utf-8'))
# get the data
data = b''
while True:
part = soc.recv(min(SIZE, length - len(data)))
data += part
if length == len(data):
break
# do some stuff with the data
The difference is we wait for a fresh connection after each object received. This gives other clients attempting to connect a turn as well.

Socket Programming; File corrupts when transferring over multiple devices

The problem I'm having is to get a file from the server to client across devices. Everything works fine on localhost.
Lets say I want to "get ./testing.pdf" which sends the pdf from the server to the client. It sends but it is always missing bytes. Is there any problems with how I am sending the data. If so how can I fix it? I left out the code for my other functionalities since they are not used for this function.
sending a txt file with "hello" in it works perfectly
server.py
import socket, os, subprocess # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
#host = ''
port = 5000 # Reserve a port for your service.
bufsize = 4096
s.bind((host, port)) # Bind to the port
s.listen(5) # Now wait for client connection.
while True:
c, addr = s.accept() # Establish connection with client.
print 'Got connection from', addr
while True:
userInput = c.recv(1024)
.... CODE ABOUT OTHER FUNCTIONALITY
elif userInput.split(" ")[0] == "get":
print "inputed get"
somefile = userInput.split(" ")[1]
size = os.stat(somefile).st_size
print size
c.send(str(size))
bytes = open(somefile).read()
c.send(bytes)
print c.recv(1024)
c.close()
client.py
import socket, os # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
#host = '192.168.0.18'
port = 5000 # Reserve a port for your service.
bufsize = 1
s.connect((host, port))
print s.recv(1024)
print "Welcome to the server :)"
while 1 < 2:
userInput = raw_input()
.... CODE ABOUT OTHER FUNCTIONALITY
elif userInput.split(" ")[0] == "get":
print "inputed get"
s.send(userInput)
fName = os.path.basename(userInput.split(" ")[1])
myfile = open(fName, 'w')
size = s.recv(1024)
size = int(size)
data = ""
while True:
data += s.recv(bufsize)
size -= bufsize
if size < 0: break
print 'writing file .... %d' % size
myfile = open('Testing.pdf', 'w')
myfile.write(data)
myfile.close()
s.send('success')
s.close

I can see two problems right away. I don't know if these are the problems you are having, but they are problems. Both of them relate to the fact that TCP is a byte stream, not a packet stream. That is, recv calls do not necessarily match one-for-one with the send calls.
size = s.recv(1024) It is possible that this recv could return only some of the size digits. It is also possible that this recv could return all of the size digits plus some of the data. I'll leave it for you to fix this case.
data += s.recv(bufsize) / size -= bufsize There is no guarantee that that the recv call returns bufsize bytes. It may return a buffer much smaller than bufsize. The fix for this case is simple: datum = s.recv(bufsize) / size -= len(datum) / data += datum.