I want my program to go into a file and take all of the lines and put them into a json key.
The code:
def import_proxies():
global proxies_json
proxies_json = {'http': {}}
with open("proxies.txt", 'r', encoding='utf8') as proxy_file:
for line in proxy_file:
val = line.split()
proxies_json[['http'][val]] = val
print(proxies_json)
import_proxies()
My erorr: TypeError: list indices must be integers or slices, not list
What I want it to do is is import the file lines as values in the http key (proxies_json = {'http': {}}
proxies_json seems to be a dictionary. So something like this should work
def import_proxies():
global proxies_json
proxies_json = {'http': {}}
with open("proxies.txt", 'r', encoding='utf8') as proxy_file:
for line in proxy_file:
val = line.split()
data = proxies_json.get('http', {})
for item in val:
data[item] = item
proxies_json['http'] = data
print(proxies_json)
Related
I got a complex nested JSON file and I need to convert it completely, but my code can only convert part of the information,
How can I modify my code, or do you guys have a better code?
my json file
import csv
import json
import sys
import codecs
def trans(path):
jsonData = codecs.open('H://14.json', 'r', 'utf-8')
# csvfile = open(path+'.csv', 'w')
# csvfile = open(path+'.csv', 'wb')
csvfile = open('H://11.csv', 'w', encoding='utf-8', newline='')
writer = csv.writer(csvfile, delimiter=',')
flag = True
for line in jsonData:
dic = json.loads(line)
if flag:
keys = list(dic.keys())
print(keys)
writer.writerow(keys)
flag = False
writer.writerow(list(dic.values()))
jsonData.close()
csvfile.close()
if __name__ == '__main__':
path=str(sys.argv[0])
print(path)
trans(path)
my json file
{"id":"aa","sex":"male","name":[{"Fn":"jeri","Ln":"teri"}],"age":45,"info":[{"address":{"State":"NY","City":"new york"},"start_date":"2001-09","title":{"name":"Doctor","Exp":"head"},"year":"2001","month":"05"}],"other":null,"Hobby":[{"smoking":null,"gamble":null}],"connect":[{"phone":"123456789","email":"info#gmail.com"}],"Education":"MBA","School":{"State":"NY","City":"new york"}}
{"id":"aa","sex":"female","name":[{"Fn":"lo","Ln":"li"}],"age":34,"info":[{"address":{"State":"NY","City":"new york"},"start_date":"2008-11","title":{"name":"Doctor","Exp":"hand"},"year":"2008","month":"02"}],"other":null,"Hobby":[{"smoking":null,"gamble":null}],"connect":[{"phone":"123456789","email":"info#gmail.com"}],"Education":"MBA","School":{"State":"NY","City":"new york"}}
It only converts part of the information, 'name''info''Hobby''connect''School' these information are not converted,I need to convert all information completely,
You could use the below function to treat each dic. It will flatten the dict by recursive calls until there is no dict or list in the values. In order to avoid issues with 2 keys having the same name, I concatenate with the previous level.
WARNING: it is based on your format so if you have lists with more than one element in the middle, it will only take care of the first element.
def flatten_dict(input_dict, result = None):
result = result or {}
for key, value in input_dict.items():
if isinstance(value, list):
current_dict = {key+"_"+k: v for k, v in value[0].items()}
flatten_dict(current_dict, result)
elif isinstance(value, dict):
current_dict = {key+"_"+k: v for k, v in value.items()}
flatten_dict(current_dict, result)
else:
result[key] = value
return result
Then apply this function on each dic, transform to Dataframe and save as CSV.
res = []
for line in jsonData:
dic = json.loads(line)
res.append(flatten_dict(dic))
res_df = pd.DataFrame(res)
res_df.to_csv("result.csv")
Result:
I have to use a text file and extract the most frequent ip address and count how many times they come up
def anaylse_log(parameter):
myfile = open("sample_log_1 test.txt", "r")
iPdata = myfile.readlines()
mydict = {}
ipAddress = []
item_list = []
result_file = []
counter = ()
def extract_log(myfile):
#split the file line by line
for line in myfile:
splitData = line.split()
ipAddress = splitData[0]
numbers = splitData[1]
ipAddress.append(ipAddress)
numbers.append(numbers)
if numbers in mydict:
#if numbers is already a key in the dictionary
#increase the count
mydict[numbers] += 1
else:
# Otherwise if it's not yet in the dictionary
# Initialise it to 1
mydict[numbers] = 1
return numbers
myfile.close()
def find_most_frequent(maximum,iPdata):
with open("sample_log_1 text", "r") as myfile:
for text in myfile:
if str(maximum) in text:
return maximum
with open("resultss.csv", "w") as file:
file.write(maximum(maximum))
#This will put the dictionary into tuples and give each key a value
item_list = [(k, v) for k, v in mydict.items()]
#This will sort the list by v
item_list.sort(key=lambda x:x[1], reverse=True)
maximum = mydict()
def main(myfile,mydict,iPdata):
result_file = open("resultss.csv", "w")
main()
i had to fix the spacing for the code to be edited, i hope this is ok and you are able to run it, i have stuck on this for a while and i thought i was calling the functions too
Suppose your log file is like
15.25.7.3
25.25.2.5
25.25.2.5
115.25.7.3
215.25.7.3
25.25.2.5
Here is a simple way to count ips
ip_count_dict = {}
with open('ip.log', 'r') as f:
ip_file = f.read()
# if separated by coma
# ip_list = ip_file.split(',')
# if separated by \n new line
ip_list = ip_file.splitlines()
for ip in ip_list:
ip = ip.strip()
if ip in ip_count_dict:
ip_count_dict[ip] += 1
else:
ip_count_dict[ip] = 1
print(ip_count_dict)
Output: {'15.25.7.3': 1, '25.25.2.5': 3, '115.25.7.3': 1, '215.25.7.3': 1}
Instead of manually counting IPs as you loop through your log, try this:
from collections import Counter
log_entries = open("resultss.csv").read().split("\n")
ip_list = [log.split(",")[0] for log in log_entries]
counts = Counter(ip_list)
print(counts)
This works with a CSV file format like:
10.10.10.1,asdf,31
5.9.7.11,aajbczxz,54
5.9.7.11,zzzzz,2
I need to convert json to csv, I just want to extract some keys in the file, but some keys do not exist in the json file, I hope it can automatically fill in these non-existent keys
import csv
import json
import sys
import codecs
def trans(path):
jsonData = codecs.open('C:/Users/jeri/Desktop/1.json', 'r', 'utf-8')
# csvfile = open(path + '.csv', 'w')
# csvfile = open(path + '.csv', 'wb')
csvfile = open('C:/Users/jeri/Desktop/1.csv', 'w', newline='', encoding='utf-8')
writer = csv.writer(csvfile, delimiter=',')
keys = ['dob','firstname','lastname']
writer.writerow(keys)
for line in jsonData:
dic = json.loads(line)
writer.writerow([dic['dob'],dic['firstname'],dic['lastname'],])
jsonData.close()
csvfile.close()
if __name__ == '__main__':
path = str(sys.argv[0])
print(path)
trans(path)
Console prompt：：
Traceback (most recent call last):
File "C:\Users\jeri\PycharmProjects\pythonProject9\main.py", line 25, in <module>
trans(path)
File "C:\Users\jeri\PycharmProjects\pythonProject9\main.py", line 17, in trans
writer.writerow([dic['dob'],dic['firstname'],dic['lastname'],])
KeyError: 'dob'
If the key 'dob' might be missing, instead of dic['dob'], do dic.get('dob', None). That provides the default you want.
I think this would solve your problem.
(I defined a function to test the existence of each item in json, if exists it return the value and if it doesn't exists it returns 'N/A')
def getValue(dic, item):
try:
return dic[item]
except:
return 'N/A'
for line in jsonData:
dic = json.loads(line)
writer.writerow([getValue(dic, 'dob'), getValue(dic, 'firstname'), getValue(dic, 'lastname'),])
you can transform your for loop into something like this.
for line in jsonData:
dic = json.loads(line)
dob = dic['dob'] if "dob" in dic else None
firstname = dic['firstname'] if "firstname" in dic else None
lastname = dic['lastname'] if "lastname" in dic else None
writer.writerow([dob,firstname,lastname])
I am trying to set a cookie if incoming requests url contains a keyword. (This part is working)
And based on how the website responds in it's "Location" header take further action, removing cookie line from the file in this case.
I have tried craming everything together under "def request(flow)" function, but that didn't work.
As you might have already guessed from the code I'm not a programmer, but spent two last nights putting this code together from snippets,
So, how do I delete the same cookie that was used in request based on the response?
Thank you in advance!
P.s The error I'm currently getting is -
"lines.remove(v)
ValueError: list.remove(x): x not in list"
import random
from mitmproxy import ctx
v = random.choice(list(open('/etc/mitm/cookies.txt')))
cookie = v.strip()
def request(flow):
url = flow.request.pretty_url
matches = ["needcookie", "nocookie"]
if any(x in url for x in matches):
flow.request.headers[b'Cookie'] = cookie
def response(flow):
if flow.response.headers.get("Location", "").startswith("https://www.mywebsite.com/nocookie?"):
lines = []
with open('/etc/mitm/cookies.txt') as file:
lines = file.readlines()
lines.remove(v)
with open("/etc/mitm/deleted.txt", "w") as f:
f.write("%s\n" % v.strip())
with open("/etc/mitm/cookies.txt", "w") as f:
for line in lines:
f.write("%s\n" % line.strip())
Had to use global variables, working solution-
import random
from mitmproxy import ctx
v = random.choice(list(open('/etc/mitm/cookies.txt')))
cookie = v.strip()
def request(flow):
global v,cookie
url = flow.request.pretty_url
matches = ["needcookie", "cozzzzm"]
if any(x in url for x in matches):
ctx.log.warn("200")
flow.request.headers[b'Cookie'] = cookie
def response(flow):
global v,cookie
if flow.response.headers.get("Location", "").startswith("https://www.mywebsite.com/nocookie"):
lines = []
with open('/etc/mitm/cookies.txt') as file:
lines = file.readlines()
lines.remove(v)
with open("/etc/mitm/deleted.txt", "w") as f:
f.write("%s\n" % v.strip())
with open("/etc/mitm/cookies.txt", "w") as f:
for line in lines:
f.write("%s\n" % line.strip())
v = random.choice(list(open('/etc/mitm/cookies.txt')))
cookie = v.strip()
So far I have this code which is creating a dictionary from an input file:
def read_file(filename):
with open("menu1.csv") as file:
file.readline()
for line in file:
line_strip = [line.rstrip('\n')]
lines= [line.split(',')]
result = {key: (float(fl), int(intg),
text.strip()) for key,
fl, intg,text in lines}
print(result)
read_file("menu1.csv")
I have to keep that code in that def format. However, this outputs 27 different dictionaries. How do I make it so it is all in ONE dictionary?
ALso:
I want to alphabetize the keys and put them into a list. I tried something like this but it won't work:
def alphabetical_menu(dict):
names = []
for name in d:
names.append(name)
names.sort()
print(names)
What am I doing wrong? or do you have a way to do it?
Is this what you wanted?
def read_file(filename):
result = {}
with open(filename) as file:
file.readline()
for line in file:
line_strip = line.rstrip()
line_split= line.split(',')
key, fl, intg, text = tuple(line_split)
result[key] = (float(fl), int(intg), text.strip())
return result
def alphabetical_menu(d):
return sorted(d.keys())
menu_dict = read_file("menu1.csv")
menu_sorted_keys = alphabetical_menu(menu_dict)
# To check the result
print(menu_dict)
print(menu_sorted_keys)