What I'm aiming to do is to get the top submission from Reddit, append it to an array and then get the second top submission, third, fourth, etc. I'm using place_holder to get the next submission, which works the first time but then just loops through getting the same, second, submission over and over again.
The current output is
Post 1
Post 2
Post 2
etc
When I want the output to be
Post 1
Post 2
Post 3
Post 4
etc
Here's my code:
import praw, time
r = praw.Reddit(user_agent='ADNPost')
already_done = []
while True:
for submission in r.get_top(limit=1):
id = submission.id
title = submission.title
url = submission.short_link
save_state = (id)
if id not in already_done:
already_done.append(submission.id)
post = title + " | " + url
print post
print save_state
if id in already_done:
for submission in r.get_front_page(limit=1, place_holder=submission.id):
id = submission.id
title = submission.title
url = submission.short_link
print title, url
save_state = (id)
already_done.append(submission.id)
time.sleep(2)
you forgot a time.sleep to comply with reddit policy
I replaced already_done with a set for efficiency
the basic idea is to getting new submission while the submission was already done
import praw, time
r = praw.Reddit(user_agent='ADNPost')
already_done = set()
while True:
l = r.get_top(limit=1)
submission = next(l,None)
if not submission:
continue
while submission.id in already_done:
submission=next(r.get_front_page(limit=1, params={'after':submission.fullname}),None)
if not submission:
break
if submission:
id = submission.id
title = submission.title
url = submission.short_link
print (title, url)
save_state = (id)
already_done.add(submission.id)
time.sleep(2)
EDIT: tested the place_holder is not what you want (the "t3_" is a constant which looks working
EDIT 2: replaced "t3_"+submission.id by submission.fullname according to #bboe proposition
Related
sorry i think this is pretty basic but i've spent a while trying to find an answer and i can't work it out.
i'm using spotify api / spotipy to make a playlist, it works and the playlist is created, but i want to retrieve the playlist_id so i can then add tracks to it, but i can't work out how to get any response information from the api.
def make_playlist(name='python_play'):
user = sp.me()['id']
sp.user_playlist_create(user=user,name=name)
response = make_playlist()
pprint (response)
returns "None"
i thought i would call response.text or repsonse.contents or something like that but response is none-type and i can't call anything from it?
i've been happily retrieving info from the api and then using that info to populate a db via sqlalchemy, but i don't understand how to actually get a response when i create the playlist..
example of successful api calls for info/db population:
def get_artists(num=1):
'''takes num, retrieves num artists, returns list of dicts'''
if num <=50:
limit = num
else:
limit = 50
artist_list = []
after = 0
for offset in range(0, num, 50):
response = sp.current_user_followed_artists(limit=limit, after=after)
for artist in response['artists']['items']:
artist_name = artist['name']
artist_id = artist['id']
artist_dict={"artist name": artist_name,
"artist id": artist_id}
artist_list.append(artist_dict)
after = artist_id
return artist_list
def new_artist(artist_list):
'''takes list of artists, writes to db'''
session = Session()
artist_list = artist_list
for artist in artist_list:
new_artist = Artist(artist_name=artist['artist name'], artist_id=artist['artist id'])
session.merge(new_artist)
session.commit()
session.close()
def populate_artists(num=1000):
'''takes num_artists, calls get_artists, calls new_artist'''
artist_list = get_artists(num)
new_artist(artist_list)
return artist_list
artist_list = populate_artists(10)
i found a workaround to search for the playlist id by playlist name, but i'm sure there must be a way to simply have spotify return the id when i make the playlist?
def GetPlaylistID(playlist_name):
playlist_id = ''
playlists = sp.current_user_playlists()
for playlist in playlists['items']: # iterate through playlists I follow
if playlist['name'] == playlist_name: # filter for newly created playlist
playlist_id = playlist['id']
return playlist_id
def make_playlist(playlist_name='python_play'):
user = sp.me()['id']
sp.user_playlist_create(user=user,name=playlist_name)
playlist_id = GetPlaylistID(playlist_name)
return playlist_id
What #StayPerfect means is, you forgot to return the response of sp.user_playlist_create() in row 3:
def make_playlist(name='python_play'):
user = sp.me()['id']
return sp.user_playlist_create(user=user,name=name)
response = make_playlist()
pprint (response)
You aren't actually return anything from your make_plalist() function in the first place.
I'm trying to retrieve data from a site by sending a POST request which appears to work, however, it returns the same result despite the request changing.
For example, if I change the dates, it will still return the same result and if I check this via the website manually, it should be different.
Any help would be greatly appreciated! I'm stuck with this.
Bdate = 20210220
Adate = 20220220
#Prepare the column headers
BdateVal = str(Bdate) + " Est. value ($m)"
AdateVal = str(Adate) + " Est. value ($m)"
#retrive payload and enter index + dates
d = {"__EVENTTARGET":"m$m$p1$p1$ITrakResults$DownloadButtoniTrak",
"__EVENTARGUMENT":"",
"__LASTFOCUS":"",
"__VIEWSTATE":VS,
"__VIEWSTATEGENERATOR":VSG ,
"__SCROLLPOSITIONX":0,
"__SCROLLPOSITIONY":0,
"m$m$p1$p1$ITrakFilter$ddlInvestmentIn":352,
"m$m$p1$p1$ITrakFilter$ddlComparisonPeriod":Bdate,
"m$m$p1$p1$ITrakFilter$HiddenFieldLatterDate":Adate,
"m$m$p1$p1$ITrakFilter$txtInvestorName":"",
"m$m$p1$p1$ITrakFilter$ddlInvestmentStyle":"",
"m$m$p1$p1$ITrakFilter$ddlInvestorList":"All Company Lists",
"m$m$p1$p1$ITrakFilter$ddlInvestmentTrends":0,
"m$m$p1$p1$ITrakFilter$slider_min":0,
"m$m$p1$p1$ITrakFilter$slider_max":71000,
"m$m$p1$p1$ITrakFilter$txtSliderLeftValue":0,
"m$m$p1$p1$ITrakFilter$txtSliderRightValue":71000,
"m$m$p1$p1$ITrakFilter$ucLocation$Location":"RadioButtonLocationIgnore",
"m$m$p1$p1$ITrakFilter$ucLocation$DropDownRegions":0,
"m$m$p1$p1$ITrakFilter$ucLocation$DropDownCountries":0}
params = {"InvIn":352,
"FormerDateID":Bdate,
"Holders":0,
"ShowDetail":"True",
"InvName":"",
"sort":"LatterEstMarketSegmentValue",
"sortAsc":"False"}
#Post request
GetCSV = s.post("https://secure.somewebsite.com/iTrak/Default.aspx?",params=params, headers=DownloadHeaders, data=d)
I'm trying to scrape username and media information from a list of Instagram user using unofficial instagram-Api with python.
The library is there
I understand how I can scrape information from the user that is logged but i can't understand how i can refer to another username.
This is the code for taking my Instagram information
from InstagramAPI import InstagramAPI
import time
username = 'myUser'
pwd = 'mypass'
API = InstagramAPI(username,pwd)
API.login()
time.sleep(2)
API.getProfileData()
pk = API.LastJson['user']['pk']
maxid = ''
while True:
API.getUserFeed(pk, maxid)
feed = API.LastJson
if 'fail' in feed['status']:
break
for i in range(0, len(feed['items']) - 1):
mediadata = feed['items'][i]
print("\033[0;34m"
"\n------------------------\n"
"Media number: "
"Like count: "
"Comment count: "
.format(i,
mediadata['like_count'],
mediadata['comment_count']))
if feed['items'][i]['caption'] is None:
print("Caption: ["
"\033[0;31m"
"No Caption available"
"\033[0;34m"
"]\n")
else:
caption = mediadata['caption']['text']
if len(caption) > 30:
caption = caption[:30] + ' (...)'
print("Caption:
[\033[0;32m{}\033[0;34m]\n".format(caption))
I solved it with:
API.searchUsername('IG_USERNAME') before this code: pk = API.LastJson['user']['pk'].
It Works for me.
Enjoy :D
Hi this is my piece of code to get the number of friends a user has by python . It returns nothing. Can anyone tell me which access privilege should i grant or anything wrong what i have done so far?
friend_count = 0
q = urllib.urlencode({'SELECT friend_count FROM user WHERE uid': 784877761})
url = 'https://api.facebook.com/method/fql.query?query=' + q
request = urllib2.Request(url)
data = urllib2.urlopen(request)
doc = parse(data)
friend_count_node = doc.getElementsByTagName("friend_count")
test = friend_count_node[0].firstChild.nodeValue
logging.info(test)
Try replacing your code with this:
q = urllib.urlencode({'SELECT friend_count FROM user WHERE uid = 784877761'})
url = 'https://graph.facebook.com/fql?q=' + q
You should not need an access token to get the friend_count.
I have a question similar to my previous question.
In the same app I have the VoteHandler that adds +1 when users click on the up arrow and then redirects to the same page.
Now that I added the url parameters to the pages and urls look like this
/dir?type=tshirt
I need to be able to redirect to the correct page.
And as mentioned here by Nick Johnson referrer is not a good way of redirecting.
I would appreciate help about how to fix VoteHandler so that it redirects to the correct page.
Thanks!
class VoteHandler(webapp.RequestHandler):
def get(self, id):
id = int(id)
item = Item.get_by_id(id)
user = users.get_current_user()
if user:
greeting = ("%s (<a href='%s'>sign out</a>)" %
(user.nickname(), users.create_logout_url(self.request.uri)))
#--check if user voted on this item before--#
query = SiteUser.all()
query.filter("liked_items =", (str(item.key().id())))
already_voted = query.get()
if already_voted:
self.redirect("/onevote")
else:
query = SiteUser.all()
query.filter("site_user =", users.get_current_user())
data = query.get()
data.total_votes += 1
data.liked_items = str(item.key().id())
data.site_user = users.get_current_user()
db.put(data)
item.points += 1
item.put()
if self.request.referrer == 'http://localhost:8083//newest':
self.redirect('/newest')
elif self.request.referrer == 'http://localhost:8083/hot':
self.redirect('/hot')
#How do I fix this line to get the correct url parameter?
elif self.request.referrer == 'http://localhost:8083/dir?type=tshirt':
self.redirect('/dir/tshirt')
else:
self.redirect("/")
else:
greeting = ("<a href='%s'>Sign in with your Google account or register</a>." %
cgi.escape(users.create_login_url(self.request.uri)))
UPDATE
The answer by systempuntoout works great but I had to change the vote link in Directory from
/vote/%s
to
/vote/%s?type=%s
so that I could get the merchandise_type in VoteHandler:
self.response.out.write("<ol>")
for item in items:
self.response.out.write("""<li>
^ %s <span id='Small'>(%s)</span><br />
<div id='Small'>
%s points %s by %s |
</div>
</li><br /> """ %
(str(item.key().id()), merchandise_type, item.url, item.title, urlparse(item.url).netloc,
item.points, item.date.strftime("%B %d, %Y %I:%M%p"), item.user_who_liked_this_item, str(item.key().id()), str(item.key().id())))
self.response.out.write("</ol>")
I would use the type parameter in this way:
class VoteHandler(webapp.RequestHandler):
def get(self, id):
id = int(id)
item = Item.get_by_id(id)
merchandise_type = self.request.get("type", "")
..
self.redirect('/%s' % merchandise_type)