In the following locust file, we are sending two user details. This works fine when we run with 2 users. However, when we run the same with more than 2 users (for example 5 users), then it starts failing for that remaining new users ( i.e 3 users).
import json
from locust import User, HttpUser, task, between, SequentialTaskSet,
HttpLocust
USER_CREDENTIALS = [
("abc#xyz.com", "Yahoo#123"),
("xyz#xyz.com", "Yahh-69-Wrap")
]
class CLMLoginTaskSet(SequentialTaskSet):
hostname_app2 = "http://app2.stage.cloud.local"
hostname_app1 = "http://app1.stage.cloud.local"
port = "8090"
username = "NOT_FOUND"
password = "NOT_FOUND"
access_token_value1 = None
#task
def userlist(self):
if len(USER_CREDENTIALS) > 0:
self.username, self.password = USER_CREDENTIALS.pop()
#task
def generate_clm_token(self):
print("username", self.username)
print("password", self.password)
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
clm_response =
self.client.post(self.hostname_app2+":"+self.port+"/oauth2/access?
grant_type=password&username="+self.username+"&password="+self.password",
headers=headers)
print("CLM Response", clm_response.text)
json_clm_response = clm_response.json()
access_token_value = json_clm_response['access_token']
print("This is the access token value", access_token_value)
self.access_token_value1 = access_token_value
class CLMLogin(HttpUser):
tasks = [CLMLoginTaskSet]
host = ""
wait_time = between(1, 2)
It should repeat and use the same user list if we start the load test with more users. How to achieve this ?
you have 2 credentials and popping them on start. pop removes the item from the list so after your first 2 users the list is empty and has nothing to pop. Try increasing credentials if you still want to use pop method or you can just access one credential with random.choice method without popping so next user can still use that. Beware that some users will have same user so they can affect each other depending on how the system works.
Related
Is it possible to use multi processing in Django on a request.
#so if I send a request to http://127.0.0.1:8000/wallet_verify
def wallet_verify(request):
walelts = botactive.objects.all()
#here I check if the user want to be included in the process or not so if they set it to True then i'll include them else ignore.
for active in walelts:
check_active = active.active
if check_active == True:
user_is_active = active.user
#for the ones that want to be included I then go to get their key data.
I need to get both api and secret so then I loop through to get the data from active users.
database = Bybitapidatas.objects.filter(user=user_is_active)
for apikey in database:
apikey = apikey.apikey
for apisecret in database:
apisecret = apisecret.apisecret
#since I am making a request to an exchange endpoint I can only include one API and secret at a time . So for 1 person at a time this is why I want to run in parallel.
for a, b in zip(list(Bybitapidatas.objects.filter(user=user_is_active).values("apikey")), list(Bybitapidatas.objects.filter(user=user_is_active).values("apisecret"))):
session =spot.HTTP(endpoint='https://api-testnet.bybit.com/', api_key=a['apikey'], api_secret=b['apisecret'])
#here I check to see if they have balance to open trades if they have selected to be included.
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
for i in GET_USDT_BALANCE:
if 'USDT' in i.values():
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
idx_USDT = GET_USDT_BALANCE.index(i)
GET_USDTBALANCE = session.get_wallet_balance()['result']['balances'][idx_USDT]['free']
print(round(float(GET_USDTBALANCE),2))
#if they don't have enough balance I skip the user.
if round(float(GET_USDTBALANCE),2) < 11 :
pass
else:
session.place_active_order(
symbol="BTCUSDT",
side="Buy",
type="MARKET",
qty=10,
timeInForce="GTC"
)
How can I run this process in parallel while looping through the database to also get data for each individual user.
I am still new to coding so hope I explained that it makes sense.
I have tried multiprocessing and pools but then I get that the app has not started yet and I have to run it outside of wallet_verify is there a way to do it in wallet_verify
and when I send the Post Request.
Any help appreciated.
Filtering the Database to get Users who have set it to True
Listi - [1,3](these are user ID's Returned
processess = botactive.objects.filter(active=True).values_list('user')
listi = [row[0] for row in processess]
Get the Users from the listi and perform the action.
def wallet_verify(listi):
# print(listi)
database = Bybitapidatas.objects.filter(user = listi)
print("---------------------------------------------------- START")
for apikey in database:
apikey = apikey.apikey
print(apikey)
for apisecret in database:
apisecret = apisecret.apisecret
print(apisecret)
start_time = time.time()
session =spot.HTTP(endpoint='https://api-testnet.bybit.com/', api_key=apikey, api_secret=apisecret)
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
for i in GET_USDT_BALANCE:
if 'USDT' in i.values():
GET_USDT_BALANCE = session.get_wallet_balance()['result']['balances']
idx_USDT = GET_USDT_BALANCE.index(i)
GET_USDTBALANCE = session.get_wallet_balance()['result']['balances'][idx_USDT]['free']
print(round(float(GET_USDTBALANCE),2))
if round(float(GET_USDTBALANCE),2) < 11 :
pass
else:
session.place_active_order(
symbol="BTCUSDT",
side="Buy",
type="MARKET",
qty=10,
timeInForce="GTC"
)
print ("My program took", time.time() - start_time, "to run")
print("---------------------------------------------------- END")
return HttpResponse("Wallets verified")
Verifyt is what I use for the multiprocessing since I don't want it to run without being requested to run. also initialiser starts apps for each loop
def verifyt(request):
with ProcessPoolExecutor(max_workers=4, initializer=django.setup) as executor:
results = executor.map(wallet_verify, listi)
return HttpResponse("done")
```
Asking a new question as my original was closed by admin who said a similar question had the answer. I checked it and it didnt have the answer. That answer was related to user input in Python itself. I am using a front end in the form of an Api Uri which I have included in this question.
In my code below I use this to call an Api using 'Method' and 'Value' parameters. For example on the URL they use to call the Api they enter Method2=Min and Value2=2 This works fine and allows the user to input whatever they want into Value2 so they can change the settings of an AutoScaling Group.
If the user makes a mistake and inputs Value as 100 instead of 10 in the Uri, for example Value3=100, I would not want the Autoscaling Group Max size to change to 100. Therefore is there of editing my script so I can stop that Max size from changing to 100? Like some sort of max size value in the script itself?
To invoke the Api I use powershell command like this
invoke-webrequest -Uri 'https://name-of-api.amazonaws.com/?TagKey=tag-key-name&TagValue=tag-value-name&Method1=Capacity&Value1=1&Method2=Min&Value2=1&Method3=Max&Value3=4' -Headers #{"X-Api-Key"="name-of-api-key"}
As you can see the user has the power to change 'Method3=Max' to have a 'Value3' of 100 if they want
Thanks
import boto3
import botocore
import os
import json
import logging
# Set up logger.
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
# Set up clients.
client = boto3.client('autoscaling')
def handler(event, context):
# Step one: get all AutoScaling Groups.
response = client.describe_auto_scaling_groups(
MaxRecords=100
)
# Make an empty list for ASG info storage.
allAutoScaling = [] # empty list of no items.
# Get the initial results and posts them to CloudWatch logs
allAutoScaling.append(response['AutoScalingGroups'])
logger.info(allAutoScaling)
# If 'Marker' is present in the response, we have more to get.
while 'Marker' in response:
old_marker = response['Marker']
response = client.describe_auto_scaling_groups(
MaxRecords=100,
Marker = old_marker
)
allAutoScaling.append(response['AutoScalingGroups'])
# Cycles back up to repeat the pagination.
# Now to find the tags specified in the Api Uri
for autoscaling in allAutoScaling:
for key in autoscaling:
for tag in key['Tags']:
if tag['Key'] == event['TagKey'] and tag['Value'] == event['TagValue']:
if event['Method2'] == "Min":
if event['Value2']:
Value2 = int(event['Value2'])
response = client.update_auto_scaling_group(
AutoScalingGroupName=key['AutoScalingGroupName'],
MinSize=Value2
)
else:
print('Min already at required level')
if event['Method3'] == "Max":
if event['Value3']:
Value3 = int(event['Value3'])
response = client.update_auto_scaling_group(
AutoScalingGroupName=key['AutoScalingGroupName'],
MaxSize=Value3
)
else:
print('Max already at required level')
if event['Method1'] == "Capacity":
if event['Value1']:
Value1 = int(event['Value1'])
response = client.set_desired_capacity(
AutoScalingGroupName=key['AutoScalingGroupName'],
DesiredCapacity=Value1
)
else:
print('Capacity already at required level')
Something within these lines:
try:
value = int(event['Value3'])
except ValueError:
print(f"{event['Value3']} is not a valid integer")
else:
if 1 <= value <= 3:
# do something, process the request
else:
print(f'Value must be between 1 and 3, got {value} instead')
Can someone please show a script that is capable of doing the above? I have found a good amount of instruction on the web and tried a lot of different things but still can't get Zap to login to the page to perform a full scan.
The best I get is something like this:
'http://XXX',
'http://XXX/robots.txt',
'http://XXX/sitemap.xml',
'http://XXX/webui',
'http://XXX/webui/index.html',
'http://XXX/webui/index.html?Password=ZAP&Username=ZAP',
'http://XXX/webui/login',
'http://XXX/webui/login/assets',
'http://XXX/webui/login/assets/images',
'http://XXX/webui/login/assets/images/companylogo.png',
'http://XXX/webui/login/assets/styles',
'http://XXX/webui/login/assets/styles/login.css',
'http://XXX/webui/login/login.js',
'http://XXX/webui/login/redirect.js',
'http://XXX/webui?Password=ZAP&Username=ZAP'
Many thanks
from zapv2 import ZAPv2
from random import randint
import socket
zap_ip = 'zap' #name of a Docker container running Zap
target = 'http://example.com'
auth_url = target + "webui/index.html"
scanners = ['90020', '90029']
# authorized Web UI user
username = test
password = test
auth_data = 'password={%password%}&username={%username#%}'
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
zap = ZAPv2(proxies={'http': 'http://' + zap_ip + ':' + str(port),
'https': 'http://' + zap_ip + ':' + str(port)})
new_context = randint(1, 100000000000)
session = zap.core.session_location
session_name = 'session_1.session' if zap.core.session_location == \
'session_0.session' else 'session_0.session'
zap.core.new_session(name=session_name)
zap.core.load_session(session_name)
context_id = zap.context.new_context(new_context)
zap.context.include_in_context(new_context, '.*')
zap.ascan.disable_all_scanners()
for scanner in scanners:
zap.ascan.enable_scanners(scanner)
all_rules = [scanner for scanner in \
zap.ascan.scanners() if scanner['enabled'] == 'true']
start_url = auth_url if auth_url else target
zap.urlopen(start_url)
auth_method_name = 'formBasedAuthentication'
authmethod_configparams = 'loginUrl=%s&loginRequestData=%s' % (auth_url, auth_data)
authcred_configparams = 'username=%s&password=%s' % (username, password)
zap.authentication.set_authentication_method(contextid=context_id,
authmethodname=auth_method_name,
authmethodconfigparams=authmethod_configparams)
user_id = zap.users.new_user(contextid=context_id, name=username)
zap.users.set_authentication_credentials(contextid=context_id,
userid=user_id,
authcredentialsconfigparams=authcred_configparams)
zap.users.set_user_enabled(contextid=context_id, userid=user_id, enabled=True zap.forcedUser.set_forced_user(context_id, user_id)
zap.forcedUser.set_forced_user_mode_enabled('true')
spider = zap.spider.scan_as_user(url=target, contextid=context_id,
userid=user_id, recurse='false')
while (int(zap.spider.status()) < 100):
time.sleep(2)
zap.ascan.scan(target)
zap.ascan.remove_all_scans()
zap.core.delete_all_alerts()
zap.context.remove_context(new_context)
Authentication is, in general, a pain. There are so many different ways authentication can be implemented its really difficult to provide anything other than very generic advice.
However the fact that you've got a URL like 'http://XXX/webui?Password=ZAP&Username=ZAP' implies you have not configured something correctly as these are the default values supplied by the ZAP spider.
If you can supply more details about what your application appears to expect and what you are doing then we should be able to help some more.
I want to get all data in cassandra table "user"
i have 840000 users and i don't want to get all users in python list.
i want get users in packs of 100 users
in cassandra doc https://datastax.github.io/python-driver/query_paging.html
i see i can use fetch_size, but in my python code i have database object that contains all cql instruction
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement
class Database:
def __init__(self, name, salary):
self.cluster = Cluster(['192.168.1.1', '192.168.1.2'])
self.session = cluster.connect()
def get_users(self):
users_list = []
query = "SELECT * FROM users"
statement = SimpleStatement(query, fetch_size=10)
for user_row in session.execute(statement):
users_list.append(user_row.name)
return users_list
actually get_users return very big list of user name
but i want to transform return get_users to a "generator"
i don't want get all users name in 1 list and 1 call of function get_users, but i want to have lot of call get_users and return list with only 100 users max every call function
for example :
list1 = database.get_users()
list2 = database.get_users()
...
listn = database.get_users()
list1 contains 100 first user in query
list2 contains 100 "second" users in query
listn contains the latest elements in query (<=100)
is this possible ?
thanks for advance for your answer
According to Paging Large Queries:
Whenever there are no more rows in the current page, the next page
will be fetched transparently.
So, if you execute your code like this, you will still the whole result set, but this is paged in a transparent manner.
In order to achieve what you need to use callbacks. You can also find some code sample on the link above.
I added below the full code for reference.
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement
from threading import Event
class PagedResultHandler(object):
def __init__(self, future):
self.error = None
self.finished_event = Event()
self.future = future
self.future.add_callbacks(
callback=self.handle_page,
errback=self.handle_error)
def handle_page(self, rows):
for row in rows:
process_row(row)
if self.future.has_more_pages:
self.future.start_fetching_next_page()
else:
self.finished_event.set()
def handle_error(self, exc):
self.error = exc
self.finished_event.set()
def process_row(user_row):
print user_row.name, user_row.age, user_row.email
cluster = Cluster()
session = cluster.connect()
query = "SELECT * FROM myschema.users"
statement = SimpleStatement(query, fetch_size=5)
future = session.execute_async(statement)
handler = PagedResultHandler(future)
handler.finished_event.wait()
if handler.error:
raise handler.error
cluster.shutdown()
Moving to next page is done in handle_page when start_fetching_next_page is called.
If you replace the if statement with self.finished_event.set() you will see that the iteration stops after the first 5 rows as defined in fetch_size
I am currently having problems with Timeouts and performance on Django redirection. The issue was not visible until I was surfing to my locally hosted application with 2 devices and only one worker enabled on my localhost, timeout set to 30 seconds.
I have a views.py function that redirects a page, based on that is given the URL. I do a lookup for the pk in a table and return the url. I also have a counter that keeps track of the amount of forwards.
urls.py here:
url(r'^i/(?P<pk>[-\w]+)/$', frontendapp_views.item_view, name="item_view"),
The page redirects instantly to the "desired_url_forward", however, the connection stays open with the user, while in fact, the user has left my Django environment. This somehow leaves my worker waiting for 30 seconds while I was already forwarded to an external page, not allowing to process any other request with one worker.
I could increase the number of workers or shorten the timeout time, but that doesn't feel right as it is not fixing the core issue.
This is the only thing I found out on this topic but I am not skilled enough to understand this: https://github.com/requests/requests/issues/520
This is how the views.py looks like:
def item_view(request,pk):
pk_binairy = urlsafe_base64_decode(pk)
pk_int = int.from_bytes(pk_binairy, byteorder='little')
desired_url_forward_object = get_object_or_404(forwards,pk = pk_int)
channel_cleaned_utm = re.sub(' +',' ',"".join([request.GET.get('utm_source', ''),' ',request.GET.get('utm_medium', ''),' ',request.GET.get('utm_campaign', ''),' ',request.GET.get('utm_term', ''),' ',request.GET.get('utm_content', '')]))
channel_cleaned = request.META.get('HTTP_REFERER')
if channel_cleaned is None:
channel_cleaned = 'Direct Traffic'
visitor_ip_request = get_client_ip(request)
location_request = get_client_location(request, visitor_ip_request)
clickstat = clickstats(
urlid = pk_int,
user = desired_url_forward_object.user,
channel = channel_cleaned,
visitor_ip = visitor_ip_request,
city = location_request['city'],
region = location_request['region'],
country = location_request['country'],
device_type = request.user_agent.device.family,
browser = request.user_agent.browser.family,
browser_version = request.user_agent.browser.version_string,
operating_system = request.user_agent.os.family ,
operating_system_version = request.user_agent.os.version_string
)
clickstat.save()
if desired_url_forward_object.counterA <= desired_url_forward_object.counterB:
desired_url_forward = desired_url_forward_object.urlA
desired_url_forward_object.counterA = F('counterA') + 1
else:
desired_url_forward = desired_url_forward_object.urlB
desired_url_forward_object.counterB = F('counterB') + 1
desired_url_forward_object.save()
return redirect(desired_url_forward)
Anyone suggestions? Thanks for the help!