I am using docker and redis to learn how multi container in docker work.I am using below simple flask application which has a publish and subscribe code.However I don't see any message received by the subscriber
import redis
from flask import Flask
app = Flask(__name__)
client = redis.Redis(host="redis-server", decode_responses=True)
def event_handler(msg):
print("Handler", msg, flush=True)
#app.route("/")
def index():
print("Request received",flush=True)
print(client.ping(), flush=True)
client.publish("insert", "This is a test 1")
client.publish("insert", "This is a test 2")
client.publish("insert", "This is a test 3")
ps = client.pubsub()
ps.subscribe("insert",)
print(ps.get_message(), flush=True)
print(ps.get_message(), flush=True)
print(ps.get_message(), flush=True)
return "Hello World"
if __name__ == '__main__':
app.run(host="0.0.0.0", port="5000")
and below is my docker-compose
version: "3"
services:
redis-server:
image: "redis"
python-app:
build: .
ports:
- "4001:5000"
and docker file
# Specify the base image
FROM python:alpine
# specify the working directory
WORKDIR /usr/app
# copy the requiremnts file
COPY ./requirements.txt ./
RUN pip install -r requirements.txt
COPY ./ ./
CMD ["python","./app.py"]
And below is the output I am getting
Can someone please help me in identifying what I am doing wrong.Thanks
I was able to fix above issue by creating two separate scripts one for publisher and other for subscriber and starting the subscriber first.
Related
I am having trouble getting my docker-compose.yml file to run, I am running the command docker-compose -f docker-compose.yml and am getting the error "TypeError: You must specify a directory to build in path
[7264] Failed to execute script docker-compose"
The file I have this in is called Labsetup and inside the file I have
docker-compose.yml,dockerClient.yml,dokcerServer.yml,and
A file called Volumes that have Server.py and Client.py
To this point I have edit my docker-compose.yml file and tried running "docker-compose -f docker-compose.yml up" to run the yml below is my docker-compose.yml
version: "3"
services:
# new code to execute python scripts in /volumes
server:
image: 85345aa61801 # from DockerServer.yml
container_name: server
build: DockerServer.yml
command: python3 ./volumes/Server.py
ports:
- 8080:8080
client:
image: 700c22b0d9d6 # from DockerClient.yml
container_name: client
build: DockerClient.yml
command: python3 ./volumes/Client.py
network_mode: host
depends_on:
- server
# below is original code for containers
malicious-router:
image: handsonsecurity/seed-ubuntu:large
container_name: malicious-router-10.9.0.111
tty: true
cap_add:
- ALL
sysctls:
- net.ipv4.ip_forward=1
- net.ipv4.conf.all.send_redirects=0
- net.ipv4.conf.default.send_redirects=0
- net.ipv4.conf.eth0.send_redirects=0
privileged: true
volumes:
- ./volumes:/volumes
networks:
net-10.9.0.0:
ipv4_address: 10.9.0.111
command: bash -c "
ip route add 192.168.60.0/24 via 10.9.0.11 &&
tail -f /dev/null "
HostB1:
image: handsonsecurity/seed-ubuntu:large
container_name: host-192.168.60.5
tty: true
cap_add:
- ALL
volumes:
- ./volumes:/volumes
networks:
net-192.168.60.0:
ipv4_address: 192.168.60.5
command: bash -c "python3 Server.py"
HostB2:
image: handsonsecurity/seed-ubuntu:large
container_name: host-192.168.60.6
tty: true
cap_add:
- ALL
volumes:
- ./volumes:/volumes
networks:
net-192.168.60.0:
ipv4_address: 192.168.60.6
command: bash -c "python3 Client.py"
HostB3:
image: handsonsecurity/seed-ubuntu:large
container_name: host-192.168.60.7
tty: true
cap_add:
- ALL
volumes:
- ./volumes:/volumes
networks:
net-192.168.60.0:
ipv4_address: 192.168.60.7
command: bash -c "python3 Client.py 8080"
Router:
image: handsonsecurity/seed-ubuntu:large
container_name: router
tty: true
cap_add:
- ALL
sysctls:
- net.ipv4.ip_forward=1
volumes:
- ./volumes:/volumes
networks:
net-10.9.0.0:
ipv4_address: 10.9.0.11
net-192.168.60.0:
ipv4_address: 192.168.60.11
command: bash -c "
ip route del default &&
ip route add default via 10.9.0.1 &&
tail -f /dev/null "
networks:
net-192.168.60.0:
name: net-192.168.60.0
ipam:
config:
- subnet: 192.168.60.0/24
net-10.9.0.0:
name: net-10.9.0.0
ipam:
config:
- subnet: 10.9.0.0/24
My DockerServer.yml
# Dockerfile.
# FROM ubuntu:20.04 # FROM python:latest
# ... most recent "image" created since DockerServer build = f48ea80eae5a <-- needed in FROM
# after running command: docker build -f DockerServer.yml .
# ... it output this upon completion ...
# ---> Running in 533f2cbd3135
# Removing intermediate container 533f2cbd3135 ---> 85345aa61801 Successfully built 85345aa61801
# FROM requires this syntax:
# FROM [--platform=<platorm>] <image> [AS <name>]
FROM python3:3.8.5 85345aa61801 AS serverImg
ADD Server.py /volumes/
WORKDIR /volumes/
My DokcerClient.yml
# Dockerfile.
# FROM ubuntu:20.04 server # python:latest #FROM ubuntu:latest ?
# step1/3 image --> f48ea80eae5a ... Successfully built 700c22b0d9d6
# image needed for 2nd arg
# after running command: docker build -f DockerClient.yml
# ... it output this upon completion ...
# Removing intermediate container 29037f3d23fb ---> 700c22b0d9d6 Successfully built 700c22b0d9d6
# FROM requires this syntax:
# FROM [--platform=<platorm>] <image> [AS <name>]
FROM python3:3.8.5 700c22b0d9d6 AS clientImg
ADD Client.py /volumes/
WORKDIR /volumes/
I am creating a project for a networking class that connects users to chat on a terminal. I have two python codes one being called Server.py that starts the server and tells who is in the chat room and another called Client.py that asks the user to put in a name and they will be able to chat. I am running this all on a virtual box Ubuntu20.04.
Code for Server.py
import socket
import _thread
import threading
from datetime import datetime
class Server():
def __init__(self):
# For remembering users
self.users_table = {}
# Create a TCP/IP socket and bind it the Socket to the port
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_address = ('localhost', 8080)
self.socket.bind(self.server_address)
self.socket.setblocking(1)
self.socket.listen(10)
print('Starting up on {} port {}'.format(*self.server_address))
self._wait_for_new_connections()
def _wait_for_new_connections(self):
while True:
connection, _ = self.socket.accept()
_thread.start_new_thread(self._on_new_client, (connection,))
def _on_new_client(self, connection):
try:
# Declare the client's name
client_name = connection.recv(64).decode('utf-8')
self.users_table[connection] = client_name
print(f'{self._get_current_time()} {client_name} joined the room !!')
while True:
data = connection.recv(64).decode('utf-8')
if data != '':
self.multicast(data, owner=connection)
else:
return
except:
print(f'{self._get_current_time()} {client_name} left the room !!')
self.users_table.pop(connection)
connection.close()
def _get_current_time(self):
return datetime.now().strftime("%H:%M:%S")
def multicast(self, message, owner=None):
for conn in self.users_table:
data = f'{self._get_current_time()} {self.users_table[owner]}: {message}'
conn.sendall(bytes(data, encoding='utf-8'))
if __name__ == "__main__":
Server()
Code for Client.py
import sys
import socket
import threading
class Client():
def __init__(self, client_name):
client_name = input("Choose Your Nickname:")
# Create a TCP/IP socket and connect the socket to the port
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_address = ('localhost', 8080)
self.socket.connect(self.server_address)
self.socket.setblocking(1)
print("{} is connected to the server, you may now chat.".format(client_name))
self.client_name = client_name
send = threading.Thread(target=self._client_send)
send.start()
receive = threading.Thread(target=self._client_receive)
receive.start()
def _client_send(self):
self.socket.send(bytes(self.client_name, encoding='utf-8'))
while True:
try:
c = input()
sys.stdout.write("\x1b[1A\x1b[2K") # Delete previous line
self.socket.send(bytes(c, encoding='utf-8'))
except:
self.socket.close()
return
def _client_receive(self):
while True:
try:
print(self.socket.recv(1024).decode("utf-8"))
except:
self.socket.close()
return
if __name__ == "__main__":
client_name = sys.argv[1]
Client(client_name)
Edit: I found a example online and followed it and changed a few
[11/26/21]seed#VM:~/.../Labsetup$ docker build -f DockerServer.yml .
Sending build context to Docker daemon 33.28kB
Step 1/3 : FROM python:latest
latest: Pulling from library/python
647acf3d48c2: Pull complete
b02967ef0034: Pull complete
e1ad2231829e: Pull complete
5576ce26bf1d: Pull complete
a66b7f31b095: Pull complete
05189b5b2762: Pull complete
af08e8fda0d6: Pull complete
287d56f7527b: Pull complete
dc0580965fb6: Pull complete
Digest: sha256:f44726de10d15558e465238b02966a8f83971fd85a4c4b95c263704e6a6012e9
Status: Downloaded newer image for python:latest
---> f48ea80eae5a
Step 2/3 : ADD Server.py /volumes/
---> df60dab46969
Step 3/3 : WORKDIR /volumes/
---> Running in 533f2cbd3135
Removing intermediate container 533f2cbd3135
---> 85345aa61801
Successfully built 85345aa61801
[11/26/21]seed#VM:~/.../Labsetup$ docker build -f DockerClient.yml .
Sending build context to Docker daemon 33.28kB
Step 1/3 : FROM python:latest
---> f48ea80eae5a
Step 2/3 : ADD Client.py /volumes/
---> 76af351c1c7a
Step 3/3 : WORKDIR /volumes/
---> Running in 29037f3d23fb
Removing intermediate container 29037f3d23fb
---> 700c22b0d9d6
Successfully built 700c22b0d9d6
I am trying to run a docker-compose app that has two services. One to build a web server and the other to run the tests on it.
docker-compose.yml
version: "3.7"
services:
web:
build: .
ports:
- "127.0.0.1:5000:5000"
expose:
- 5000
test:
# expose:
# - 5000
depends_on:
- web
build: test_python/.
./Dockerfile
FROM python:buster
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
# Add .cargo/bin to PATH
ENV PATH="/root/.cargo/bin:${PATH}"
# Check cargo is visible
RUN cargo --help
WORKDIR /code
COPY requirements.txt requirements.txt
RUN pip3 install -r requirements.txt
EXPOSE 5000
COPY test_python .
CMD [ "python3", "base_routes.py" ]
test_python/Dockerfile
FROM python:buster
RUN pip3 install pytest requests
COPY . .
base_routes.py
from robyn import Robyn, static_file, jsonify
import asyncio
app = Robyn(__file__)
callCount = 0
#app.get("/")
async def h(request):
print(request)
global callCount
callCount += 1
message = "Called " + str(callCount) + " times"
return message
#app.get("/test")
async def test(request):
import os
path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "test_python/index.html"))
return static_file(path)
#app.get("/jsonify")
async def json_get(request):
return jsonify({"hello": "world"})
#app.post("/jsonify")
async def json(request):
print(request)
return jsonify({"hello": "world"})
#app.post("/post")
async def postreq(request):
return bytearray(request["body"]).decode("utf-8")
#app.put("/put")
async def putreq(request):
return bytearray(request["body"]).decode("utf-8")
#app.delete("/delete")
async def deletereq(request):
return bytearray(request["body"]).decode("utf-8")
#app.patch("/patch")
async def patchreq(request):
return bytearray(request["body"]).decode("utf-8")
#app.get("/sleep")
async def sleeper():
await asyncio.sleep(5)
return "sleep function"
#app.get("/blocker")
def blocker():
import time
time.sleep(10)
return "blocker function"
if __name__ == "__main__":
app.add_header("server", "robyn")
app.add_directory(route="/test_dir",directory_path="./test_dir/build", index_file="index.html")
app.start(port=5000)
These are the files that I have used in my project. When I try to open 127.0.0.1:5000 from my machine, it shows nothing. However, when I log in the web container and do curl http://localhost:5000/, it gives the right response.
I am unable to figure out how to access it on the host machine?
I had to make the python server listen at '0.0.0.0'.
I added the following line in my codebase
app.start(port=5000, url='0.0.0.0')
Firstly thank you for your time.
I can't get any date from an API in my docker container. Neverveless I can if I'm in the container with a curl command but not from postman or a curl command if I 'm not in the container I don't know why, somebody have a solution for me ? :(
I mapped the port, docker-compose ps :
python /bin/sh -c ./src/server.py - ... Up 0.0.0.0:9090->9090/tcp
Above my docker-compose file :
'''
# Docker Compose
version: "3"
# Docker Containers
services:
# python
python:
build: ./docker/python
working_dir: /root/app
command: ./src/server.py
environment:
- SERVER_PORT=9090
ports:
- 9090:9090
volumes:
- .:/root/app
- ./.bash-history/python:/root/.bash_history
'''
My python server :
'''
#!/usr/local/bin/python
import sys
sys.path.append('/root/app/python_modules')
from flask import Flask, json
companies = [{"id": 1, "name": "Company One"}, {"id": 2, "name": "Company Two"}]
api = Flask(__name__)
#api.route('/companies', methods=['GET'])
def get_companies():
return json.dumps(companies)
if __name__ == '__main__':
api.run(host='', port=9090, debug=True)
'''
I would like to test a simple UDP server in Docker before continue with the develop. I've never worked with Docker before so I assume it should be a beginner error
What I have currently?
The server
import threading
import socketserver
class ThreadedUDPRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
data = self.request[0].strip()
current_thread = threading.current_thread()
print("Thread: {} client: {}, wrote: {}".format(current_thread.name, self.client_address, data))
Split = threading.Thread(target=ParseIncomingData,args=(data,))
Split.start()
class ThreadedUDPServer(socketserver.ThreadingMixIn, socketserver.UDPServer):
pass
def publish_messages(data):
"""Publishes multiple messages to a Pub/Sub topic."""
print('Published {} .'.format(data))
def ParseIncomingData(message):
sender = threading.Thread(target=publish_messages, args=(message,))
sender.start()
if __name__ == "__main__":
HOST, PORT = "0.0.0.0", 6071
try:
serverUDP = ThreadedUDPServer((HOST, PORT), ThreadedUDPRequestHandler)
server_thread_UDP = threading.Thread(target=serverUDP.serve_forever)
server_thread_UDP.daemon = True
server_thread_UDP.start()
serverUDP.serve_forever()
except (KeyboardInterrupt, SystemExit):
serverUDP.shutdown()
serverUDP.server_close()
exit()
The dockerfile
# Use an official Python runtime as a base image
FROM python:3.7.2-slim
# Set the working directory to /app
WORKDIR /app
# Copy the current directory contents into the container at /app
ADD . /app
# Make port 6071 available outside this container
EXPOSE 6071/udp
# Run app.py when the container launches
CMD ["python", "app.py"]
How I execute the container?
docker run --rm -p 6071:6071/udp listener
I have tried with several combinations about the usage of the port but I don't see anything when I run it (Windows)
EXTRA:
To test the server I'm using hercules to send the USP data:
It works with this configuration:
Using unbuffered output in dockerfile with
CMD ["python","-u","main.py"]
I am having two dockerized application which needs to run in kubernetes.
Here is the scenario which needs to achieve.
Docker-1 which is flask application.
Docker-2 which is python script will take input from the Docker-1 and execute and need to write some file in a shared volume of the Docker-1 container.
Here is the flask web-app code.
from flask import Flask, request, Response, jsonify
app = Flask(__name__)
#app.route('/')
def root():
return "The API is working fine"
#app.route('/run-docker')
def run_docker_2():
args = "input_combo"
query = <sql query>
<initiate the docker run and pass params>
exit
#No return message need run as async
if __name__ == "__main__":
app.run(debug=True, host='0.0.0.0', port=8080, threaded=True)
Docker file
FROM ubuntu:latest
MAINTAINER Abhilash KK "abhilash.kk#searshc.com"
RUN apt-get update -y
RUN apt-get install -y python-pip python-dev build-essential python-tk
COPY . /app
WORKDIR /app
RUN pip install -r requirements.txt
ENTRYPOINT ["/usr/bin/python"]
CMD ["app.py"]
requirements.txt
flask
Python script for the second docker. start_docker.py
import sys
input_combo = sys.argv[1]
query = sys.argv[2]
def function_to_run(input_combination,query):
#starting the model final creating file
function_to_run(input_combo, query)
Docker file 2
FROM python
COPY . /script
WORKDIR /script
CMD ["python", "start_docker.py"]
Please help me to connect with the docker images. or let me know any other way to achieve this problem. The basic requirement is to add a message to some queue and that queue listens for in time interval and starts the process with FIFO manner.
Any other approach in GCP service to initiate an async job will take input from the client and create a file which is accessible from web-app python.
First, create a Pod running "Docker-1" application. Then Kubernetes python client to spawn a second pod with "Docker-2".
You can share a volume between your pods in order to return the data to Docker1. In my code sample I'm using a host_path volume but you need to ensure that both pods are on the same node. I did add that code for readability.
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: docker1
labels:
app: docker1
spec:
replicas: 1
selector:
matchLabels:
app: docker1
template:
metadata:
labels:
app: docker1
spec:
containers:
- name: docker1
image: abhilash/docker1
ports:
- containerPort: 8080
volumeMounts:
- mountPath: /shared
name: shared-volume
volumes:
- name: shared-volume
hostPath:
path: /shared
The code of run_docker_2 handler:
from kubernetes import client, config
...
args = "input_combo"
config.load_incluster_config()
pod = client.V1Pod()
pod.metadata = client.V1ObjectMeta(name="docker2")
container = client.V1Container(name="docker2")
container.image = "abhilash/docker2"
container.args = [args]
volumeMount = client.V1VolumeMount(name="shared", mount_path="/shared")
container.volume_mounts = [volumeMount]
hostpath = client.V1HostPathVolumeSource(path = "/shared")
volume = client.V1Volume(name="shared")
volume.host_path = hostpath
spec = client.V1PodSpec(containers = [container])
spec.volumes = [volume]
pod.spec = spec
v1.create_namespaced_pod(namespace="default", body=pod)
return "OK"
A handler to read the returned results:
#app.route('/read-results')
def run_read():
with open("/shared/results.data") as file:
return file.read()
Note that it could be useful to add a watcher to wait for the pod to finish the job and then do some cleanup (delete the pod for instance)
From what I can understand you'd want the so called "sidecar pattern", you can run multiple containers in one pod and share a volume, e.g.:
apiVersion: v1
kind: Pod
metadata:
name: www
spec:
containers:
- name: nginx
image: nginx
volumeMounts:
- mountPath: /srv/www
name: www-data
readOnly: true
- name: git-monitor
image: kubernetes/git-monitor
env:
- name: GIT_REPO
value: http://github.com/some/repo.git
volumeMounts:
- mountPath: /data
name: www-data
volumes:
- name: www-data
emptyDir: {}
You could also benefit from getting to know the basics of how Kubernetes work: Kubernetes Basics