New to Python...trying to perform an API call and output the data to CSV.
Works fine for a single variable being passed but when I parse through a list the last item in the list is the only output I receive.
Looking to find the best way to approach this. Not sure if it's just an issue of logic or I need to find a way to consistently append to the file and remove results when it's rerun.
list = open(infile, "r")
print("-------------Attempting to Query Data----------------")
for item in list:
try:
if inp == 1:
eval_string = "&value=" + item
elif inp == 2:
eval_string = "&value__regexp=.*." + item + "*"
else:
print("Invalid input!")
result = requests.get(url + "api_key=" + api + "&username=" + user + eval_string + "&limit=" + str(limit) + "&status=" + status)
data = result.json()
if result.status_code == 200:
with open(outfile, 'w', newline='') as data_file:
writer = csv.writer(data_file)
count = 0
for obj in data['objects']:
if count == 0:
header = 'value', 'confidence', 'type', 'source', 'date', 'status'
writer.writerow(header)
count += 1
writer.writerow([obj['value'], obj['confidence'], obj['type'], obj['source'], obj['date'], obj['status']])
else:
print("-------------Failed to connect - check API config info or that site is up----------------")
except OSError:
print("Failed to query.")
print("-------------Results returned in " + outfile + "----------------")
Related
I am pretty new to python and I have one csv file and based on one condition I need to create two different xml files.
The condition based on which I need to create two xml is as follows:
If Primary spindle == 1 then data related to it will go to xml1 else xml2.
This is how my xml looks like as shown in image:
The code I am writing is as follows:
import csv
file = open(r'C:\Users\hu170f\MultiSpindleArchitechture\New folder\MAAP-S12_LH_UP_PASS-LN1-V1.csv')
csvreader = csv.reader(file)
xmlFile = r'C:\Users\hu170f\Documents\TEST\myData4.xml'
xmlFile1 = r'C:\Users\hu170f\Documents\TEST\myData5.xml'
#header = next(csvreader)
xmlData = open(xmlFile, 'w')
xmlData.write('<?xml version="1.0" encoding = "UTF-8"?>' + "\n")
# there must be only one top-level tag
xmlData.write('<MCD>' + "\n")
xmlData1 = open(xmlFile1, 'w')
xmlData1.write('<?xml version="1.0" encoding = "UTF-8"?>' + "\n")
# there must be only one top-level tag
xmlData1.write('<MCD>' + "\n")
#print(header)
rows = []
for row in csvreader:
rows.append(row)
#print(len(rows))
#print(len(row))
Field = " "
Value = " "
counter = 0
for i in rows:
tag = i
#print(tag)
#print(len(rows))
for j in range(len(tag)):
tag[j] = tag[j].replace(' ', '_')
if j == 0:
#print("Field = ", tag[j])
Field = tag[j]
counter = counter +1
else:
#print("Value = ", tag[j])
Value = tag[j]
if(counter%2 == 0):
xmlData.write(' ' + '<' + Field + '>' \
+ Value + '</' + Field + '>' + "\n")
else :
xmlData1.write(' ' + '<' + Field + '>' \
+ Value + '</' + Field + '>' + "\n")
xmlData.write('</MCD>' + "\n")
xmlData.close()
xmlData1.write('</MCD>' + "\n")
xmlData1.close()
#print(rows[6])
file.close()
I want both xml to contain common data from Header to LN and then data from WorkStep UUID till secondary Spindle based on the condition in each xml file.
Truoble with a really annoying homework. I have a csv-file with lots of comma-delimitered fields per row. I need to take the last two fields from every row and write them into a new txt-file. The problem is that some of the latter fields have sentences, those with commas are in double quotes, those without them aren't. For example:
180,easy
240min,"Quite easy, but number 3, wtf?"
300,much easier than the last assignment
I did this and it worked just fine, but the double quotes disappear. The assignment is to copy the fields to the txt-file, use semicolon as delimiter and remove possible line breaks. The text must remain exactly the same. We have an automatic check system, so it's no use arguing if this makes any sense.
import csv
file = open('myfile.csv', 'r')
output= open('mytxt.txt', 'w')
csvr = csv.reader(file)
headline = next(csvr)
for line in csvr:
lgt = len(line)
time = line[lgt - 2].replace('\n', '')
feedb = line[lgt - 1].replace('\n', '')
if time != '' and feedb != '':
output.write(time + ';' + feedb + '\n')
output.close()
file.close()
Is there some easy solution for this? Can I use csv module at all? No one seems to have exactly the same problem.
Thank you all beforehand.
Try this,
import csv
file = open('myfile.csv', 'r')
output= open('mytxt.txt', 'w')
csvr = csv.reader(file)
headline = next(csvr)
for line in csvr:
lgt = len(line)
time = line[lgt - 2].replace('\n', '')
feedb = line[lgt - 1].replace('\n', '')
if time != '' and feedb != '':
if ',' in feedb:
output.write(time + ';"' + feedb + '"\n')
else:
output.write(time + ';' + feedb + '\n')
output.close()
file.close()
Had to do it the ugly way, the file was too irrational. Talked with some collaegues on the same course and apparently the idea was NOT to use csv module here, but to rehearse basic file handling in Python.
file = open('myfile.csv','r')
output = open('mytxt.txt', 'w')
headline = file.readline()
feedb_lst = []
count = 0
for line in file:
if line.startswith('1'): #found out all lines should start with an ID number,
data_lst = line.split(',', 16) #that always starts with '1'
lgt = len(data_lst)
time = data_lst[lgt - 2]
feedb = data_lst[lgt - 1].rstrip()
feedback = [time, feedb]
feedb_lst.append(feedback)
count += 1
else:
feedb_lst[count - 1][1] = feedb_lst[count - 1][1] + line.rstrip()
i = 1
for item in feedb_lst:
if item[0] != '' and item[1] != '':
if i == len(feedb_lst):
output.write(item[0] + ';' + item[1])
else:
output.write(item[0] + ';' + item[1] + '\n')
i += 1
output.close()
file.close()
Thank you for your help!
I have written code to extract tweets from a list of users [handles]. I am writing the information to a .txt file called "results".
with open("results", "w") as fp:
for handle in handles:
print("Analyzing tweets from " + handle + "...")
user = api.get_user(id=handle)
fp.write("Handle: " + handle + "\n")
fp.write("Name: " + user.name + "\n")
fp.write("Description: " + str(user.description.encode(sys.stdout.encoding, errors='replace')) + "\n")
fp.write("Followers: " + str(user.followers_count) + "\n")
fp.write("Following: " + str(user.friends_count) + "\n")
tweet_counter = 0
prosocial_tweets_count = 0
regular_tweets_count = 0
all_tweets = []
social_tweets_len = []
regular_tweets_len = []
social_tweets_valence = []
regular_tweets_valence = []
regular_attachments = 0
social_attachments = 0
for tweet in tweepy.Cursor(api.user_timeline, id=user.id).items():
#control for timeline
dt = tweet.created_at
if dt > date_until:
continue
if dt < date_from:
break # XXX: I hope it's OK to break here
if include_retweets == "no" and tweet.text.startswith("RT"):
continue
if include_replies == "no" and tweet.in_reply_to_user_id:
continue
tweet_counter += 1
for word in vocabulary:
if word in tweet.text.lower():
#increase count of pro social tweets
prosocial_tweets_count += 1
#clean the tweet for valence analysis
clean = TextBlob(tweet.text.lower())
#calculate valence
valence = clean.sentiment.polarity
#append the valence to a list
social_tweets_valence.append(valence)
#append the length of the tweet to a list
social_tweets_len.append(len(tweet.text))
#check if there is an attachment
counting = tweet.text.lower()
counting_attachments = counting.count(" https://t.co/")
social_attachments = social_attachments + counting_attachments
#write date
fp.write(" * " + str(dt) + "\n")
#write the tweet
fp.write(" " + str(tweet.text.encode(sys.stdout.encoding, errors='replace')) + "\n")
#write the length of the tweet
fp.write(" Length of tweet " + str(len(tweet.text)) + "\n")
#write the valence of the tweet
fp.write(" Tweet valance " + str(valence) + "\n")
#write the retweets of the tweet
fp.write(" Retweets count: " + str(tweet.retweet_count) + "\n")
#write the likes of the tweet
fp.write(" Likes count: " + str(tweet.favorite_count) + "\n")
# Report each tweet only once whenever it contains more than one prosocial words
break
else:
#this code runs if the tweet is not prosocial
regular_tweets_count += 1
clean = TextBlob(tweet.text.lower())
valence = clean.sentiment.polarity
counting = tweet.text.lower()
counting_attachments = counting.count(" https://t.co/")
regular_attachments = regular_attachments + counting_attachments
regular_tweets_valence.append(valence)
regular_tweets_len.append(len(tweet.text))
attachments = regular_attachments + social_attachments
I was wondering whether anyone knows of any nice way to check if the tweets contains images or videos. I would also like to create a list of average use of images and videos per user.
If you look at This thread, you will see that all media in a tweet are actually stored in tweet.entities['media'].
Therefore if you want to know if a given tweet (in the format tweepy.models.Status used by tweepy) contains a picture, you could try this:
try:
print(True in [medium['type'] == 'photo' for medium in tweet.entities['media']])
except:
print("No picture in this tweet")
I hope it helps.
Data is in JSON format when we fetch it from Twitter API. Though it contains all data about that id, and comment in form of value and fields. So if you just want to check whether image already exist or not you make a conditional statement stating
if(image == TRUE){
THEN 'yes'
}
ELSE
'no'
1.json file contain many sniffing WIFI packets, I want get the mac address of receiver and transmitter which can be found in the first "wlan" object called "wlan.ra" and "wlan.sa". data[0] is the first WIFI packet.
Q1:
But when I try to print the elements of wlan after json load, it only show the elements of the second "wlan" object so there is no "wlan.ra" and "wlan.sa" in the data.
with open('1.json','r') as json_data:
data = json.load(json_data)
a=data[0]
print a
Q2:
There are two 'wlan' objects in my json file. How can I merge the elements in these two 'wlan' objects into just one 'wlan' object?
The following is my code, but it doesn't work:
with open('1.json','r') as f:
data=json.load(f)
for i in data:
i['_source']['layers']['wlan'].update()
Screenshot of json file:
'''
Created on 2017/10/3
#author: DD
'''
import os
def modify_jsonfile(jsonfile):
'''
replace wlan to wlan1/wlan2
'''
FILESUFFIX = '_new' # filename suffix
LBRACKET = '{' # json object delimiter
RBRACKET = '}'
INTERSETED = '"wlan"' # string to be replaced
nBrackets = 0 # stack to record object status
nextIndex = 1 # next index of wlan
with open(jsonfile, 'r') as fromJsonFile:
fields = os.path.splitext(jsonfile) # generate new filename
with open(fields[0] + FILESUFFIX + fields[1], 'w') as toJsonFile:
for line in fromJsonFile.readlines():
for ch in line: # record bracket
if ch == LBRACKET:
nBrackets += 1
elif ch == RBRACKET:
nBrackets -= 1
if nBrackets == 0:
nextIndex = 1
if (nextIndex == 1 or nextIndex == 2) and line.strip().find(INTERSETED) == 0: # replace string
line = line.replace(INTERSETED, INTERSETED[:-1] + str(nextIndex) + INTERSETED[-1])
nextIndex += 1
toJsonFile.write(line);
print 'done.'
if __name__ == '__main__':
jsonfile = r'C:\Users\DD\Desktop\1.json';
modify_jsonfile(jsonfile)
I'm trying to write tuple ('hola', 'cheese', '1235265'), ('hey', 'weird', '30193') getting from mysql DB, putting values into binary file
I saw it got DB table as tuple. Tried to convert into binary, didn't work well.
So i tried another with tuple -> string -> binary, still has an error...
is there any good ways to write tuple to binary file in Python?
for i in text_query:
query = "select * from " + i
curs.execute(query)
# Data Fetch from Mysql
rows = curs.fetchall()
results = [".".join(map(str, r)) for r in rows]
make_file(name,i,results)
conn.close()
def make_file(name, filename, rows):
if filename == 'student':
with open(name + '_' + filename + '.dat', 'wb') as fp:
for i in rows:
fp.write(bytearray(rows + '\n'))
elif filename == 'course':
with open(name + '_' + filename + '.dat', 'wb') as fp:
for i in rows:
fp.write(bytearray(rows + '\n'))
elif filename == 'course_taken':
with open(name + '_' + filename + '.dat', 'wb') as fp:
for i in rows:
fp.write(bytearray(rows + '\n'))
else:
return 0;
You can write a binary representation of your data to file easily enough, not that it would be a good idea, from an encoding point of view. Nor will it be easy to read back in, but this will work:
def make_text_file(name, filename, rows):
if filename == 'student': #
with open(name + '_' + filename + '.txt', 'w') as fp:
row_index = 0
fp.write('<record start>')
for i in rows:
row_index += 1
fp.write('<row {0}>{1}</row>'.format(row_index, i))
fp.write('<record end>\n')
def make_binary_file(name, filename, rows):
if filename == 'student':
with open(name + '_' + filename + '.dat', 'wb') as fp:
row_index = 0
for i in rows:
row_index += 1
fp.write(bytes((i), 'utf8'))
def test_things():
""" Generate some data to write to file """
rows = ('hola', 'cheese', '1235265')
#make_file('student', 'test_student.txt', rows)
make_text_file('test', 'student', rows)
rows = ('hey', 'weird', '30193')
make_binary_file('test', 'student', rows)
if __name__ == '__main__':
test_things()