I'm a running a proxy as suggested in Mitmproxy github examples:
from libmproxy import proxy, flow
class MitmProxy(flow.FlowMaster):
def run(self):
try:
flow.FlowMaster.run(self)
except KeyboardInterrupt:
self.shutdown()
def handle_request(self, r):
f = flow.FlowMaster.handle_request(self, r)
if f:
r.reply()
return f
def handle_response(self, r):
f = flow.FlowMaster.handle_response(self, r)
if f:
r.reply()
return f
config = proxy.ProxyConfig(
cacert = os.path.expanduser("~/.ssl/mitmproxy.pem")
)
state = flow.State()
server = proxy.ProxyServer(config, 8083)
m = MitmProxy(server, state)
try:
m.run()
except Exception, e:
print e.message
m.shutdown()
I want to handle each request/response without blocking the others,
for that i need to use the concurrent decorator and scripts
my question is: how do i load and unload scripts to the proxy running in this configuration?
You can use concurrent mode with script loading.
Here is an example for this kind of usage
I preferred to implement the mitmproxy logic in the flow level.
You can use this code
def handle_response(self, r):
reply = f.response.reply
f.response.reply = controller.DummyReply()
if hasattr(reply, "q"):
f.response.reply.q = reply.q
def run():
pass
threading.Thread(target=run)
You basically have to copy how handle_concurrent_reply works in libmproxy.script
f = flow.FlowMaster.handle_request(self,r)
if f:
def run():
request.reply() #if you forget this you'll end up in a loop and never reply
threading.Thread(target=run).start() #this will start run
Related
Relevant portion of my code looks something like this:
#directory_router.get("/youtube-dl/{relative_path:path}", tags=["directory"])
def youtube_dl(relative_path, url, name=""):
"""
Download
"""
relative_path, _ = set_path(relative_path)
logger.info(f"{DATA_PATH}{relative_path}")
if name:
name = f"{DATA_PATH}{relative_path}/{name}.%(ext)s"
else:
name = f"{DATA_PATH}{relative_path}/%(title)s.%(ext)s"
ydl_opts = {
"outtmpl": name,
# "quiet": True
"logger": logger,
"progress_hooks": [yt_dlp_hook],
# "force-overwrites": True
}
with yt.YoutubeDL(ydl_opts) as ydl:
try:
ydl.download([url])
except Exception as exp:
logger.info(exp)
return str(exp)
I am using this webhook/end point to allow an angular app to accept url/name input and download file to folder. I am able to logger.info .. etc. output the values of the yt_dlp_hook, something like this:
def yt_dlp_hook(download):
"""
download Hook
Args:
download (_type_): _description_
"""
global TMP_KEYS
if download.keys() != TMP_KEYS:
logger.info(f'Status: {download["status"]}')
logger.info(f'Dict Keys: {download.keys()}')
TMP_KEYS = download.keys()
logger.info(download)
Is there a way to stream a string of relevant variables like ETA, download speed etc. etc. to the front end? Is there a better way to do this?
You could use a Queue object to communicate between the threads. So when you call youtube_dl pass in a Queue that you can add messages inside yt_dlp_hook (you'll need to use partial functions to construct it). You'll be best off using asyncio to run the download at the same time as updating the user something like:
import asyncio
from functools import partial
import threading
from youtube_dl import YoutubeDL
from queue import LifoQueue, Empty
def main():
# Set the url to download
url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
# Get the current event loop
loop = asyncio.get_event_loop()
# Create a Last In First Out Queue to communicate between the threads
queue = LifoQueue()
# Create the future which will be marked as done once the file is downloaded
coros = [youtube_dl(url, queue)]
future = asyncio.gather(*coros)
# Start a new thread to run the loop_in_thread function (with the positional arguments passed to it)
t = threading.Thread(target=loop_in_thread, args=[loop, future])
t.start()
# While the future isn't finished yet continue
while not future.done():
try:
# Get the latest status update from the que and print it
data = queue.get_nowait()
print(data)
except Empty as e:
print("no status updates available")
finally:
# Sleep between checking for updates
asyncio.run(asyncio.sleep(0.1))
def loop_in_thread(loop, future):
loop.run_until_complete(future)
async def youtube_dl(url, queue, name="temp.mp4"):
"""
Download
"""
yt_dlp_hook_partial = partial(yt_dlp_hook, queue)
ydl_opts = {
"outtmpl": name,
"progress_hooks": [yt_dlp_hook_partial],
}
with YoutubeDL(ydl_opts) as ydl:
return ydl.download([url])
def yt_dlp_hook(queue: LifoQueue, download):
"""
download Hook
Args:
download (_type_): _description_
"""
# Instead of logging the data just add the latest data to the queue
queue.put(download)
if __name__ == "__main__":
main()
I'm trying to get the generated domain name or IP-address of flask_ngrok or py-ngrok after been deploy. I want to deploy flask_app to localhost and get the new IP-address or domain name on the main page.
I.E: If I access 127.0.0.1/ I want it to return something like
You can now log in through https://aaf8447ee878.ngrok.io/
I have tried checking through the directories and read some help but I can't still get it. Thanks in advance ❤
add
import atexit
import json
import os
import platform
import shutil
import subprocess
import tempfile
import time
import zipfile
from pathlib import Path
from threading import Timer
import requests
def _run_ngrok():
ngrok_path = str(Path(tempfile.gettempdir(), "ngrok"))
_download_ngrok(ngrok_path)
system = platform.system()
if system == "Darwin":
command = "ngrok"
elif system == "Windows":
command = "ngrok.exe"
elif system == "Linux":
command = "ngrok"
else:
raise Exception(f"{system} is not supported")
executable = str(Path(ngrok_path, command))
os.chmod(executable, 777)
ngrok = subprocess.Popen([executable, 'http', '5000'])
atexit.register(ngrok.terminate)
localhost_url = "http://localhost:4040/api/tunnels" # Url with tunnel details
time.sleep(1)
tunnel_url = requests.get(localhost_url).text # Get the tunnel information
j = json.loads(tunnel_url)
tunnel_url = j['tunnels'][0]['public_url'] # Do the parsing of the get
tunnel_url = tunnel_url.replace("https", "http")
return tunnel_url
def _download_ngrok(ngrok_path):
if Path(ngrok_path).exists():
return
system = platform.system()
if system == "Darwin":
url = "https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-darwin-amd64.zip"
elif system == "Windows":
url = "https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-windows-amd64.zip"
elif system == "Linux":
url = "https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip"
else:
raise Exception(f"{system} is not supported")
download_path = _download_file(url)
with zipfile.ZipFile(download_path, "r") as zip_ref:
zip_ref.extractall(ngrok_path)
def _download_file(url):
local_filename = url.split('/')[-1]
r = requests.get(url, stream=True)
download_path = str(Path(tempfile.gettempdir(), local_filename))
with open(download_path, 'wb') as f:
shutil.copyfileobj(r.raw, f)
return download_path
def start_ngrok():
global ngrok_address
ngrok_address = _run_ngrok()
print(f" * Running on {ngrok_address}")
print(f" * Traffic stats available on http://127.0.0.1:4040")
def run_with_ngrok(app):
"""
The provided Flask app will be securely exposed to the public internet via ngrok when run,
and the its ngrok address will be printed to stdout
:param app: a Flask application object
:return: None
"""
old_run = app.run
def new_run():
thread = Timer(1, start_ngrok)
thread.setDaemon(True)
thread.start()
old_run()
app.run = new_run
####################
dont import flask_ngrok
at the end at before name == 'main' add this function
def ngrok_url():
global tunnel_url
while True:
try:
print(ngrok_address)
except Exception as e:
print(e)
and after before app.run() put
thread = Timer(1, ngrok_url)
thread.setDaemon(True)
thread.start()
and run Warning: this will crash your code editor/ or terminal if u dont want that in the ngrok url function replace print with whatever you want to do with the url
and you dont need that
global tunnel_url
def ngrok_url():
while True:
try:
print(ngrok_address)
except Exception as e:
print(e)
you can delete the threading part before the name == 'main' too after the imports set
ngrok_address = ''
then you can accses the ngrok_address anywhere in your code
I found out the easiest way to do this is the just copy the url when the user is visiting the site. You can do this by...
#app.before_request
def before_request():
global url
url = request.url
# url = url.replace('http://', 'https://', 1)
url = url.split('.ngrok.io')[0]
url += '.ngrok.io'
I have a definition inside of a class which handles an exception in a manner I don't like.
The class itself is inside a module, which itself is called by a module that I import.
the error handling I don't like looks like this:
class BitSharesWebsocket(Events):
#[snip]
def run_forever(self):
""" This method is used to run the websocket app continuously.
It will execute callbacks as defined and try to stay
connected with the provided APIs
"""
cnt = 0
while not self.run_event.is_set():
cnt += 1
self.url = next(self.urls)
log.debug("Trying to connect to node %s" % self.url)
try:
# websocket.enableTrace(True)
self.ws = websocket.WebSocketApp(
self.url,
on_message=self.on_message,
on_error=self.on_error,
on_close=self.on_close,
on_open=self.on_open
)
self.ws.run_forever()
except websocket.WebSocketException as exc:
if (self.num_retries >= 0 and cnt > self.num_retries):
raise NumRetriesReached()
sleeptime = (cnt - 1) * 2 if cnt < 10 else 10
if sleeptime:
log.warning(
"Lost connection to node during wsconnect(): %s (%d/%d) "
% (self.url, cnt, self.num_retries) +
"Retrying in %d seconds" % sleeptime
)
time.sleep(sleeptime)
I wish to preempt the exception here:
except websocket.WebSocketException as exc:
and handle it in my own way, namely to try a new address rather than trying the same address again and again.
I am presented with this exception when calling:
from bitshares.blockchain import Blockchain
from bitshares import BitShares
try:
chain = Blockchain(bitshares_instance=BitShares(n))
except:
print ('hello world')
pass
when n is a bad/unresponsive websocket address
I never get the 'hello world' message because the module handles the exception before I do.
the module is hosted at github here:
https://github.com/xeroc/python-bitshares/blob/9250544ca8eadf66de31c7f38fc37294c11f9548/bitsharesapi/websocket.py
I can do:
from bitsharesapi import websocket as ws
but I am not sure what to do with the module ws now that it is imported to preempt its exception handling, or if this is even the correct way to approach it.
I resolved my issue here:
chain = Blockchain(bitshares_instance=BitShares(n))
I was able to do:
chain = Blockchain(bitshares_instance=BitShares(n,num_retries=0))
I had previously tried this and assumed it wouldn't work:
chain = Blockchain(bitshares_instance=BitShares(n),num_retries=0)
*note parenthesis placement
I have a piece of python code to practice python co-routines.
As explained by A. Jesse Jiryu Davis.
Firstly, I define a co-routine named 'get' to get the content of some
URL.
Then I define a Task class to iterate the co-routine to completion.
Then I create one Task object which open one URL.
If I put two successive socket.recv() method in the co-routine, I got the error message:
'A non-blocking socket operation could not be completed immediately' in the second chunk = s.recv(1000) line.
But if I change all the yield to time.sleep(1) and directly call
get() in the global context, the two successive s.recv(1000) will
cause no errors. Even more successive s.recv(1000) is OK.
After several days searching and reading python documents, I still have no idea why this is happening. I must missed some 'python Gotchas', do I?
I'm using python 3.6 to test. The code is as following and I have deleted all the irrelevant code to make the following code precise and relevant to the topic:
#! /usr/bin/python
import socket
import select
import time
selectors_read = []
selectors_write = []
class Task:
def __init__(self, gen):
self.gen = gen
self.step()
def step(self):
try:
next(self.gen)
except StopIteration:
return
def get():
s = socket.socket()
selectors_write.append(s.fileno())
s.setblocking(False)
try:
s.connect(('www.baidu.com', 80))
except:
pass
yield
selectors_write.remove(s.fileno())
print('[CO-ROUTINE] ', 'Send')
selectors_read.append(s.fileno())
s.send('GET /index.html HTTP/1.0\r\n\r\n'.encode())
yield
while True:
chunk = s.recv(1000)
chunk = s.recv(1000)
if chunk:
print('[CO-ROUTINE] received')
else:
selectors_read.remove(s.fileno())
break
# yield
task_temp = Task(get())
while True:
for filenums in select.select(selectors_read, selectors_write, []):
for fd in filenums:
task_temp.step()
I'm writing a program to get the domain in same server and it also can scan the web directory.
#!/usr/bin/env python
#encoding = utf-8
import threading
import urllib,urllib2,httplib
from urllib2 import Request, urlopen, URLError
import Queue,sys
import re
concurrent = 5
url = sys.argv[1]
class Scanner(threading.Thread):
def __init__(self, work_q):
threading.Thread.__init__(self)
self.work_q = work_q
def getdomains(self):
doreq = Request('http://www.logontube.com/website/'+ url)
response = urlopen(doreq)
html = response.read()
response.close()
domains = re.findall('<br><a href=\"(.*?)\" target=\"_blank\"',html)
return domains
def run(self):
alldomains = self.getdomains()
pathline = [line.rstrip() for line in open("path.txt")]
while True:
for aim in alldomains:
for path in pathline:
path = self.work_q.get()
req = Request(aim+path)
try:
response = urlopen(req)
except URLError, e:
if hasattr(e, 'reason'):
print aim+path,'Not Found'
elif hasattr(e,'code'):
print aim+path,'Not Found'
else:
try:
logs = open('log.txt',"a+")
except(IOError):
print "[x] Failed to create log file"
print aim+path,"Found"
logs.writelines(aim+path+"\n")
logs.close()
def main():
work_q = Queue.Queue()
paths = [line.rstrip() for line in open("path.txt")]
for i in range(concurrent):
t = Scanner(work_q)
t.setDaemon(True)
t.start()
for path in paths:
work_q.put(path)
work_q.join()
main()
The problem is this program only do the loop of the path, so i only can get the scan result of one website.
I've found the problem,
for path in paths:
work_q.put(path) # The program finishes when it puts all the path
If you want to help me to test this program, you may need some directory of website(save it as path.txt)
/default.asp
/index.asp
/index.htm
/index.html
/index.jsp
/index.php
/admin.asp
/admin.php
/admin.shtml
/admin.txt
/admin_admin.asp
/config.asp
/inc/
/login.asp
/login.jsp
/login.php
/login/
/phpinfo.php
/readme.txt
/robots.txt
/test.asp
/test.html
/test.txt
/test.php
/news/readme.txt
/addmember/
You need a:
while 1:
pass
or something that waits until your threads are completed then it exits.
What is happening is that you are starting the threads but you are terminating the main thread so you never get to see the results of your threads.