Develop Reference

BeautifulSoup4 and Requests Module 'IndexError: list index out of range'

I'm new to web scraping with python and am having a problem with the weather web scraping script I wrote. Here is the whole code 'weather.py':
#! python3
import bs4, requests
weatherSite = requests.get('https://weather.com/en-CA/weather/today/l/eef019cb4dca2160f08eb9714e30f28e05e624bbae351ccb6a855dbc7f14f017')
weatherSoup = bs4.BeautifulSoup(weatherSite.text, 'html.parser')
weatherLoc = weatherSoup.select('.CurrentConditions--location--kyTeL')
weatherTime = weatherSoup.select('.CurrentConditions--timestamp--23dfw')
weatherTemp = weatherSoup.select('.CurrentConditions--tempValue--3a50n')
weatherCondition = weatherSoup.select('.CurrentConditions--phraseValue--2Z18W')
weatherDet = weatherSoup.select('.CurrentConditions--precipValue--3nxCj > span:nth-child(1)')
location = weatherLoc[0].text
time = weatherTime[0].text
temp = weatherTemp[0].text
condition = weatherCondition[0].text
det = weatherDet[0].text
print(location)
print(time)
print(temp + 'C')
print(condition)
print(det)
It basically parses the weather information from 'The Weather Channel' and prints it out. This code was working fine yesterday when I wrote it. But, I tried today and it is giving me the following error:
Traceback (most recent call last):
File "C:\Users\username\filesAndStuff\weather.py", line 16, in <module>
location = weatherLoc[0].text
IndexError: list index out of range

Replace:
weatherLoc = weatherSoup.select('.CurrentConditions--location--kyTeL')
# print(weatherLoc)
# []
By:
weatherLoc = weatherSoup.select('h1[class*="CurrentConditions--location--"]')
# print(weatherLoc)
# [<h1 class="CurrentConditions--location--2_osB">Hamilton, Ontario Weather</h1>]
As you can see, your suffix kYTeL is not the same for me 2_osB. You need a partial match on class attribute (class*=) (note the *)

KeyError in Weather API Response

I was making a python program which tells about weather but facing the following error
Please tell how to correct it?
CODE
import requests
import json
api = "<MY_API_KEY>"
BASE_URL = "https://api.openweathermap.org/data/2.5/weather?"
print("for which city?")
CITY = input("city?: ")
URL = BASE_URL + "q=" + CITY + "&appid=" + api
response = requests.get(URL)
if __name__ == "__main__":
data = response.json()
main = data['main']
temperature = main['temp']
humidity = main['humidity']
pressure = main['pressure']
report = data['weather']
print(f"{CITY:-^30}")
print(f"Temperature: {temperature}")
print(f"Humidity: {humidity}")
print(f"Pressure: {pressure}")
print(f"Weather Report: {report[0]['description']}")
TERMINAL
PS C:\Users\mamta\Documents\PythonPanti\JARVISprj> c:; cd 'c:\Users\mamta\Documents\PythonPanti\JARVISprj'; & 'C:\Users\mamta\AppData\Local\Programs\Python\Python39\python.exe' 'c:\Users\mamta\.vscode\extensions\ms-python.python-2021.5.842923320\pythonFiles\lib\python\debugpy\launcher'
'53873' '--' 'c:\Users\mamta\Documents\PythonPanti\JARVISprj\exp.py'
for which city?
city?: Delhi
Traceback (most recent call last):
File "c:\Users\mamta\Documents\PythonPanti\JARVISprj\exp.py", line 13, in <module>
main = data['main']
KeyError: 'main'

Your code is perfectly fine only problem here I see is when you enter any garbage value. You need to do error handling.

python crawler ieee paper keywords

i trying to use crawler to get ieee paper keywords but now i get a error
how can to fix my crawler?
my code is here
import requests
import json
from bs4 import BeautifulSoup
ieee_content = requests.get("http://ieeexplore.ieee.org/document/8465981", timeout=180)
soup = BeautifulSoup(ieee_content.text, 'xml')
tag = soup.find_all('script')
for i in tag[9]:
s = json.loads(re.findall('global.document.metadata=(.*;)', i)[0].replace("'", '"').replace(";", ''))
and error is here
Traceback (most recent call last):
File "G:/github/爬蟲/redigg-leancloud/crawlers/sup_ieee_keywords.py", line 90, in <module>
a.get_es_data(offset=0, size=1)
File "G:/github/爬蟲/redigg-leancloud/crawlers/sup_ieee_keywords.py", line 53, in get_es_data
self.get_data(link=ieee_link, esid=es_id)
File "G:/github/爬蟲/redigg-leancloud/crawlers/sup_ieee_keywords.py", line 65, in get_data
s = json.loads(re.findall('global.document.metadata=(.*;)', i)[0].replace(";", '').replace("'", '"'))
IndexError: list index out of range

Here's another answer. I don't know what you are doing with 's' in your code after the load (replace) in my code.
The code below doesn't thrown an error, but again how are you using 's'
import requests
import json
from bs4 import BeautifulSoup
ieee_content = requests.get("http://ieeexplore.ieee.org/document/8465981", timeout=180)
soup = BeautifulSoup(ieee_content.text, 'xml')
tag = soup.find_all('script')
# i is a list
for i in tag[9]:
metadata_format = re.compile(r'global.document.metadata=.*', re.MULTILINE)
metadata = re.findall(metadata_format, i)
if len(metadata) != 0:
# convert the list
convert_to_json = json.dumps(metadata)
x = json.loads(convert_to_json)
s = x[0].replace("'", '"').replace(";", '')
###########################################
# I don't know what you plan to do with 's'
###########################################
print (s)

Apparently in line 65 some of the data provided in i did not suite the regex pattern you're trying to use. Therefor your [0] will not work as the data returned is not an array of suitable length.
Solution:
x = json.loads(re.findall('global.document.metadata=(.*;)', i)
if x:
s = x[0].replace("'", '"').replace(";", ''))

Python - an identical line of code using datetime fails in one file but not another

I have the following lines of code in project A
#filename : mod_dates
#Handles date calculations etc
import datetime
class datecalcs:
def __init__(self):
self.__menuChoice = 0
self.__datemonth = "not set"
self.__effectivedate = ""
self.__year = 0
self.__month = 0
return None
#
def interestcouponpaydates(self,effectivedate,couponday):
self.__effectivedate = effectivedate
year, month, day = map(int,self.__effectivedate.split('-'))
print(year)
print(month)
return self.__effectivedate
When I call them from another file with
import mod_dates
import datetime
import modcalinputs
datesclass = mod_dates.datecalcs()
calcInputs = modcalinputs.calcinputs()
#Get the coupon date
interestdateeffective = calcInputs.interestdateffective()
interestdatecoupon = calcInputs.interestdatecoupon()
x = datesclass.interestcouponpaydates(interestdateeffective,interestdatecoupon)
print(x)
However this returns an error on the x = datesclass... line of
year, month, day = map(int,self.__effectivedate.split('-'))
raises:
> AttributeError: 'datetime.date' object has no attribute 'split'
When I run from a similar project to the same line with the same syntax it works fine. Any ideas on what I am doing wrong?

Looks like something is assigning a datetime.date object to __effectivedate. You can't call split() on that:
>>> import date
>>> d = d = datetime.date(2012,3,12)
>>> d.split('-')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
AttributeError: 'datetime.date' object has no attribute 'split'
You can convert it to a string and split:
>>>str(d).split('-')
['2012', '03', '12']

How to resolve TypeError?

from __future__ import division
import urllib,urllib2
import urllib
import json
from math import log
def hits(word1,word2=""):
query = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=%s"
if word2 == "":
results = urllib.urlopen(query % word1)
else:
results = urllib.urlopen(query % word1+" "+"AROUND(10)"+" "+word2)
json_res = json.loads(results.read())
google_hits=int(json_res['responseData']['cursor']['estimatedResultCount'])
return google_hits
def so(phrase):
num = hits(phrase,"excellent") * hits("poor")
den = hits(phrase,"poor") * hits("excellent")
ratio = num / den
sop = log(ratio,2)
return sop
print so("beautiful product")
I require the above code to calculate the semantic orientation of a given phrase(or string).
When I execute the code, I get the following error:
Traceback (most recent call last):
File "C:\Python27\nltkexp\ddddd.py", line 32, in <module>
print so("beautiful product")
File "C:\Python27\nltkexp\ddddd.py", line 24, in so
den = hits(phrase,"poor") * hits("excellent")
File "C:\Python27\nltkexp\ddddd.py", line 16, in hits
google_hits=int(json_res['responseData']['cursor']['estimatedResultCount'])
TypeError: 'NoneType' object has no attribute '__getitem__'
How to resolve this error? Can someone point out where I am going wrong with the code?

You are accessing json_res['responseData']['cursor']['estimatedResultCount'], the problem is
json_res or json_res['responseData'] or json_res['responseData']['cursor'] might be None and its can't access key of the NoneType object.
Please print json_resp and check which keys are available in that.