I am trying to code rss news feeder bot for irc. So I search a bit on web a little and made out this code
#this code is for local testing
import feedparser
feed_list = {}
channel = '#hackingdefined'
class Feed:
def __init__(self, name, url):
self.name = name
self.url = url
self.feeds = {}
self.newest = ''
def update(self):
self.feeds = feedparser.parse(self.url)
if self.newest != self.feeds['items'][0].title:
self.newest = self.feeds['items'][0].title
say('{}: {} '.format(self.name,self.newest))
say('URL: {} '.format(self.feeds.entries[0].link))
def say(data=''):
print('PRIVMSG '+channel+' :'+ data+'\r\n')
def url_loader(txt):
f = open(txt, 'r')
for line in f:
name, url = line.split(':',1) # check how to spilt only once
print name+" "+url
feed_list[name] = Feed(name,url)
print feed_list
url_loader('feed_list.txt')
for feed in feed_list.values():
print feed
feed.update()
When I run the code I get this error
Traceback (most recent call last):
File "C:\Or\define\projects\rss feed\the progect\test.py", line 33, in <module>
feed.update()
File "C:\Or\define\projects\rss feed\the progect\test.py", line 14, in update
if self.newest != self.feeds['items'][0].title:
IndexError: list index out of range
Now the wierd thing is, if I create a new Feed class like test = Feed('example', 'http://rss.packetstormsecurity.com/')
and call test.update() Its all work fine, but the automation script raise an error.
So i checked my url_load and the test file,The test file is something like this:
packet storm:http://rss.packetstormsecurity.com/
sans:http://www.sans.org/rss.php/
...
And its all seems fine to me. Any one have a clue what this could be?
Thanks, Or
EDIT:
Its been solved, one of my url was wrong.
All seem clear after good night sleep :-)
Its been solved, one of my url that i post into the file was wrong.
The solution is use try on every url in the list.
Related
I'm new to web scraping with python and am having a problem with the weather web scraping script I wrote. Here is the whole code 'weather.py':
#! python3
import bs4, requests
weatherSite = requests.get('https://weather.com/en-CA/weather/today/l/eef019cb4dca2160f08eb9714e30f28e05e624bbae351ccb6a855dbc7f14f017')
weatherSoup = bs4.BeautifulSoup(weatherSite.text, 'html.parser')
weatherLoc = weatherSoup.select('.CurrentConditions--location--kyTeL')
weatherTime = weatherSoup.select('.CurrentConditions--timestamp--23dfw')
weatherTemp = weatherSoup.select('.CurrentConditions--tempValue--3a50n')
weatherCondition = weatherSoup.select('.CurrentConditions--phraseValue--2Z18W')
weatherDet = weatherSoup.select('.CurrentConditions--precipValue--3nxCj > span:nth-child(1)')
location = weatherLoc[0].text
time = weatherTime[0].text
temp = weatherTemp[0].text
condition = weatherCondition[0].text
det = weatherDet[0].text
print(location)
print(time)
print(temp + 'C')
print(condition)
print(det)
It basically parses the weather information from 'The Weather Channel' and prints it out. This code was working fine yesterday when I wrote it. But, I tried today and it is giving me the following error:
Traceback (most recent call last):
File "C:\Users\username\filesAndStuff\weather.py", line 16, in <module>
location = weatherLoc[0].text
IndexError: list index out of range
Replace:
weatherLoc = weatherSoup.select('.CurrentConditions--location--kyTeL')
# print(weatherLoc)
# []
By:
weatherLoc = weatherSoup.select('h1[class*="CurrentConditions--location--"]')
# print(weatherLoc)
# [<h1 class="CurrentConditions--location--2_osB">Hamilton, Ontario Weather</h1>]
As you can see, your suffix kYTeL is not the same for me 2_osB. You need a partial match on class attribute (class*=) (note the *)
This code is pre-made in a Zapier forum to pull failed responses from another piece of software called iAuditor. When I plug in the code and update the API token and webhook URL this error pops up:
Traceback (most recent call last):
SyntaxError: invalid syntax (usercode.py, line 42)
Here is the code:
[code]
import json
import requests
auth_header = {'Authorization': 'a4fca847d3f203bd7306ef5d1857ba67a2b3d66aa455e06fac0ad0be87b9d226'}
webhook_url = 'https://hooks.zapier.com/hooks/catch/3950922/efka9n/'
api_url = 'https://api.safetyculture.io/audits/'
audit_id = input['audit_id']
audit_doc = requests.get(api_url + audit_id, headers=auth_header).json()
failed_items = []
audit_author = audit_doc['audit_data']['authorship']['author']
conducted_on = audit_doc['audit_data']['date_completed']
conducted_on = conducted_on[:conducted_on.index('T')]
audit_title = audit_doc['template_data']['metadata']['name']
for item in audit_doc['items']:
if item.get('responses') and item['responses'].get('failed') == True:
label = item.get('label')
if label is None:
label = 'no_label'
responses = item['responses']
response_label = responses['selected'][0]['label']
notes = responses.get('text')
if notes is None:
notes = ''
failed_items.append({'label': label,
'response_label': response_label,
'conducted_on': conducted_on,
'notes': notes,
'author': audit_author
})
for item in failed_items:
r = requests.post(webhook_url, data = item)
return response.json()
[/code]
This looks like an error from the platform. It looks like Zapier uses a script called usercode.py to bootstrap launching your script and the error seems to be coming from that part.
I'm trying to get the names of my top 3 artists of last week with pylast (https://github.com/pylast/pylast) but I run into an error or get I get None as a result and I don't see what I'm doing wrong. pylast is a Python interface to Last.fm.
My code:
import pylast
API_KEY = ""
API_SECRET = ""
username = ""
password_hash = pylast.md5("")
network = pylast.LastFMNetwork(api_key=API_KEY, api_secret=API_SECRET, username=username, password_hash=password_hash)
user = network.get_authenticated_user();
weekly_artists = user.get_weekly_artist_charts();
# Keep the first three artists.
del weekly_artists[3:]
# Print the artist name and number of songs(weight).
for weekly_artist in weekly_artists:
artist,weight = weekly_artist
print (artist.get_name())
print (artist.get_correction())
artist.get_name() returns
None
artist.get_correction() returns
Traceback (most recent call last):
File "G:\projects\python\lastfm_weekly\lastfm-weekly.py", line 28, in <module>
print (artist.get_correction())
File "C:\Users\..\Python\Python36-32\lib\site-packages\pylast\__init__.py", line 1585, in get_correction
self._request(self.ws_prefix + ".getCorrection"), "name")
File "C:\Users\..\Python\Python36-32\lib\site-packages\pylast\__init__.py", line 1029, in _request
return _Request(self.network, method_name, params).execute(cacheable)
File "C:\Users\..\Python\Python36-32\lib\site-packages\pylast\__init__.py", line 744, in __init__
network._get_ws_auth()
AttributeError: 'str' object has no attribute '_get_ws_auth'
What am I doing wrong?
Here is a quick and dirty solution, i'm sure someone will provide something better but i just installed the package to test and it works.
network = pylast.LastFMNetwork(api_key=API_KEY, api_secret=API_SECRET)
artists = network.get_top_artists()
del artists[:3]
for i in artists:
artist, weight = i
print('Artist = {}. Weight = {}'.format(artist, weight))
I'm not really familiar with the package, I just installed it to help out with this but I do wonder what "get_name()" and "get_correction()" are as they're not in your provided code.
If they're not functions you created / are defined within your code then I'd look there for the problem.
Also, you're authenticating the user but the documentation explicitly states you don't need to unless you're writing data.
Below is the code I have been working on.
The very last line write_csv('twitter_gmail.csv', messages, append=True) throws a
[ec2-user#ip-172-31-46-164 ~]$ ./twitter_test16.sh
Traceback (most recent call last):
File "./twitter_test16.sh", line 53, in
write_csv('twitter_gmail.csv', messages, append=True)
NameError: name 'messages' is not defined
I have messages defined so I dont understand why it would do that.
import csv
import json
import oauth2 as oauth
import urllib
import sys
import requests
import time
CONSUMER_KEY = "
CONSUMER_SECRET = "
ACCESS_KEY = "
ACCESS_SECRET = "
class TwitterSearch:
def __init__(self, ckey=CONSUMER_KEY, csecret=CONSUMER_SECRET,
akey=ACCESS_KEY, asecret=ACCESS_SECRET,
query='https://api.twitter.com/1.1/search/tweets.{mode}?{query}'
):
consumer = oauth.Consumer(key=ckey, secret=csecret)
access_token = oauth.Token(key=akey, secret=asecret)
self.client = oauth.Client(consumer, access_token)
self.query = query
def search(self, q, mode='json', **queryargs):
queryargs['q'] = q
query = urllib.urlencode(queryargs)
return self.client.request(self.query.format(query=query, mode=mode))
def write_csv(fname, rows, header=None, append=False, **kwargs):
filemode = 'ab' if append else 'wb'
with open(fname, filemode) as outf:
out_csv = csv.writer(outf, **kwargs)
if header:
out_csv.writerow(header)
out_csv.writerows(rows)
def main():
ts = TwitterSearch()
response, data = ts.search('#gmail.com', result_type='recent')
js = json.loads(data)
messages = ([msg['created_at'], msg['txt'], msg['user']['id']] \
for msg in js.get('statuses', []))
write_csv('twitter_gmail.csv', messages, append=True)
The previous line is missing a parenthesis.
messages = ([msg['created_at'], msg['txt'], msg['user']['id']] for msg in js.get('statuses', [])
Should be:
messages = ([msg['created_at'], msg['txt'], msg['user']['id']] for msg in js.get('statuses', []))
I'm surprised that it works when you change to print? Are you also changing the comprehension when you do that?
You asked why the line number of the error was after the bad syntax?
Try putting this in line one of a file and running it, and note the line of the SyntaxError.
a = (]
Then try this and check out the line number:
a = (
b = "some stuff"
Finally, try this:
a = (
b = "some stuff"
Think about when you would know that the programmer had made a python-illegal typo if you were reading the code and carrying it out via pen and paper.
Basically, a SyntaxError is raised as soon as it can be unambiguously determined that invalid syntax was used, which is often immediately after a statement where a mistake was made, not immediately at.
You'll frequently get line numbers on SyntaxErrors that are a line (or several lines if there's empty lines or a corner case) below the actual typo.
I'm currently trying to use the pjsip api pjsua in python and therefor studying this Hello World example: http://trac.pjsip.org/repos/wiki/Python_SIP/Hello_World
I copied the code over, integrated account configuration according to http://trac.pjsip.org/repos/wiki/Python_SIP/Accounts etc. But when I run the sample, I get the following output:
Traceback (most recent call last):
File "/home/dmeli/workspace/eit.cubiephone.sip_test/eit/cubiephone/sip_test/hello.py", line 48, in <module>
acc = lib.create_account(acc_cfg)
File "/usr/local/lib/python2.7/dist-packages/pjsua.py", line 2300, in create_account
err, acc_id = _pjsua.acc_add(acc_config._cvt_to_pjsua(), set_default)
File "/usr/local/lib/python2.7/dist-packages/pjsua.py", line 900, in _cvt_to_pjsua
cfg.rtp_transport_cfg = self.rtp_transport_cfg._cvt_to_pjsua()
AttributeError: '_pjsua.Transport_Config' object has no attribute '_cvt_to_pjsua'
Because I'm not really a python expert and never worked with PJSIP before, I can't really figure out the error. Too me, it looks like it's actually an error in the pjsip python wrapper. But what do I know?
Code:
lib = pj.Lib()
lib.init(log_cfg = pj.LogConfig(level=3, callback=log_cb))
transport = lib.create_transport(pj.TransportType.UDP)
lib.start()
acc_cfg = pj.AccountConfig("XXXXX", "XXXXXX", "XXXXXX")
acc_cfg.id = "sip:XXXXXXX#XXXXXXXX"
acc_cfg.reg_uri = "sip:XXXXXXXXX"
acc_cfg.proxy = [ "sip:XXXXXXXXX;lr" ]
acc = lib.create_account(acc_cfg)
# Make call
call = acc.make_call("XXXXXXXXXXX", MyCallCallback())
Line where the error happens in pjsua.py:
cfg.rtp_transport_cfg = self.rtp_transport_cfg._cvt_to_pjsua()
(rtp_transport_cfg doesn't seem to have a member _cvt_to_pjsua()??)
For further work correctly, look at the PJSIP api (pjsua.py) that he is waiting for the order and structure!!!
## start lib.
def start(self):
try:
self._start_lib()
self._start_acc()
except pj.Error:
print "Error starting lib."
def _bind(self):
try:
t = pj.TransportConfig()
t.bound_addr = '0.0.0.0'
t.port = 5060
acc_transport = "udp" # depend if you need.
if acc_transport == "tcp":
self.transport = self.lib.create_transport(pj.TransportType.TCP, t)
# or this pj.TransportConfig(0) is creating random port ...
#self.transport = self.lib.create_transport(pj.TransportType.TCP, pj.TransportConfig(0))
else:
self.transport = self.lib.create_transport(pj.TransportType.UDP, t)
#self.transport = self.lib.create_transport(pj.TransportType.UDP, pj.TransportConfig(0))
except pj.Error:
print "Error creating transport."
#you need create callbacks for app, to work incoming calls, check on the server that returns the error code 200, and such a way your program will know that you are logged on correctly
#from callback.acc_cb import acc_cb
#self.acc_t = self.lib.create_account_for_transport(self.transport, cb=acc_cb())
def _start_lib(self):
self.lib.init(log_cfg = pj.LogConfig(level=3, callback=log_cb))
self.lib.start()
self._bind()
#codecs.load_codecs()
def _start_acc(self):
#from callback.acc_cb import acc_cb
try:
proxy = "sip server ip" # or proxy = socket.gethostbyname(unicode("sip.serverdnsname.com")) is needed to import socket
login = "Atrotygma" # real username
password = "Atrotygma_password" # real username
lib.create_account(acc_config=pj.AccountConfig(proxy, login, password))
except Exception, e:
print "Error creating account", e