I'm very new to python and having trouble adjusting the results of an api request to handle UK daylight savings (Greenwich Mean Time / British Summer Time). When reading the Dark Sky documents they state :
The timezone is only used for determining the time of the request; the response will always be relative to the local time zone
I have built the below code to return historic weather information based upon hourly min/max/avg temperatures for each day for specific weather stations. I've been able to turn the returned UNIX time into a time stamp, but I need a way to deal with the data when the clocks change.
This is my whole code, if anyone can offer any advice I would be very grateful
import requests
import json
import pytemperature
import pandas as pd
pd.options.mode.chained_assignment = None # default='warn'
from pandas.io.json import json_normalize
import datetime
#Import CSV with station data
data_csv = pd.read_csv("Degree Days Stations.csv")
#Basic Information for the API request
URL = "https://api.darksky.net/forecast/"
AUTH = "REDACTED/"
EXCLUDES = "?exclude=currently,daily,minutely,alerts,flags"
#Use ZIP function to loop through the station_df dataframe
for STATION, NAME, CODE, LAT, LON in zip(data_csv['Station'], data_csv['Name'], data_csv['Code'], data_csv['Lat'], data_csv['Lon']):
#Result is based upon the time zone when running the script
date1 = '2019-10-26T00:00:00'
date2 = '2019-10-29T00:00:00'
start = datetime.datetime.strptime(date1, '%Y-%m-%dT%H:%M:%S')
end = datetime.datetime.strptime(date2, '%Y-%m-%dT%H:%M:%S')
step = datetime.timedelta(days=1)
while start <= end:
#Compile the Daily Values
DATE = (datetime.date.strftime(start.date(), "%Y-%m-%dT%H:%M:%S"))
#build the api url request
response = requests.get(URL+AUTH+str(LAT)+","+str(LON)+","+str(DATE)+EXCLUDES+"?units=si")
json_data = response.json()
#Flatten the data
json_df = json_normalize(json_data['hourly'],record_path=['data'],sep="_")
#Extract to new df
output_df = json_df[['time','temperature']]
#insert station name to dataframe for my debugging
output_df.insert(0, 'Name', NAME)
#Convert UNIX date to datetime
output_df['time'] = pd.to_datetime(output_df['time'], unit = 's')
##################
# Deal with timezone of output_df['time'] here
##################
#Convert temperatures from oF to oC
output_df['temperature'] = pytemperature.f2c(output_df['temperature'])
#Find the MIN/MAX/AVG from the hourly data for this day
MIN = output_df['temperature'].min()
MAX = output_df['temperature'].max()
AVG = output_df['temperature'].mean()
#Build the POST query
knackURL = "https://api.knack.com/v1/objects/object_34/records"
payload = '{ \
"field_647": "' + STATION + '", \
"field_649": "' + NAME + '", \
"field_650": "' + str(CODE) + '", \
"field_651": "' + str(DATE) + '", \
"field_652": "' + str(MIN) + '", \
"field_653": "' + str(MAX) + '", \
"field_655": "' + str(AVG) + '" \
}'
knackHEADERS = {
'X-Knack-Application-Id': "REDCATED",
'X-Knack-REST-API-Key': "REDACTED",
'Content-Type': "application/json"
}
#response = requests.request("POST", knackURL, data=payload, headers=knackHEADERS)
start += step
My results for October 27th (BST) and 28th (GMT) are shown below and are relevant to the current timezone (GMT). How can I ensure that that I get the same size of dataset starting from 00:00:00 ?
I've looked at arrow and pytz but can't seem to get it to work in the context of the dataframe. I've been working on the assumption that I need to test and deal withe the data when converting it from Unix... but I just can't get it right. Even trying to pass the data through arrow.get eg
d1 = arrow.get(output_df['time'])
print(d1)
Leaves me with the error : "Can't parse single argument type of '{}'".format(type(arg))
TypeError: Can't parse single argument type of ''
So I'm guessing that it doesn't want to work as part of a dataframe ?
Thank you in advance
Related
After trying to scrape data from twitter using Snscrape, I am unable to get the data of tweets posted within the past hour only.
import pandas as pd
import snscrape.modules.twitter as sntwitter
from datetime import datetime, time
from datetime import timedelta
now = datetime.utcnow()
since = now - timedelta(hours=1)
since_str = since.strftime('%Y-%m-%d %H:%M:%S.%f%z')
until_str = now.strftime('%Y-%m-%d %H:%M:%S.%f%z')
# Query tweets with hashtag #SOSREX in the last one hour
query = '#SOSREX Since:' + since_str + ' until:' + until_str
SOSREX_data = []
for tweet in sntwitter.TwitterSearchScraper(query).get_items():
if len(SOSREX_data)>100:
break
else:
SOSREX_data.append([tweet.date,tweet.user.username,tweet.user.displayname,
tweet.content,tweet.likeCount,tweet.retweetCount,
tweet.sourceLabel,tweet.user.followersCount,tweet.user.location
])
Tweets_data = pd.DataFrame(SOSREX_data,
columns=["Date_tweeted","username","display_name",
"Tweets","Number_of_Likes","Number_retweets",
"Source_of_Tweet",
"number_of_followers","location"
])
In order to create get-requests I create a Python script. In order to create the URL's for this request I have made the following code:
today = str(datetime.date.today())
start = str(datetime.date.today()- datetime.timedelta (days=30))
report = ["Shifts",
"ShiftStops",
"ShiftStopDetailsByProcessDate",
"TimeRegistrations",
"ShiftsByProcessDate",
"ShiftStopsByProcessDate",
]
for x in report:
url_data = "https://URL"+ report + "?from=" + start + "&until=" + today
data = requests.get(url_data, headers = {'Host': 'services.URL.com', 'Authorization': 'Bearer ' + acces_token})
But the error I get is:
TypeError: can only concatenate str (not "list") to str
What can I do to solve this and create 6 unique url's?
p.s. I have added the word URL to the URL's in order to anonymize my post.
Where you're going wrong is in the following line:
url_data = "https://URL"+ report + "?from=" + start + "&until=" + today
Specifically, you use report which is the entire list. What you'll want to do is use x instead, i.e. the string in the list.
Also you'll want to indent the next line, so altogether it should read:
for x in report:
url_data = "https://URL"+ x + "?from=" + start + "&until=" + today
data = requests.get(url_data, headers = {'Host': 'services.URL.com', 'Authorization': 'Bearer ' + acces_token})
I have already found the answer. The list of url's is created by replacing report by x while create the url_data.
url_data = "https://URL"+ x + "?from=" + start + "&until=" + today
I am trying to fetch an updated dataset from an SQL database every day. However, whenever I call the function inside the schedule I get a job instance back instead of the data. Why is the schedule not returning the data frame?
def request_data_insitereports(creds:dict, report_id:int, from_date:str, to_date:str) -> pd.DataFrame:
serverpath = r'https://portal.insitereports.nl/data.php?'
r = requests.post(serverpath + 'action=login', data=creds)
result = r.json()
if(result['success']==True):
token = result['token']
report = requests.get(serverpath + 'action=getdata&token=' + str(token) + '&report='+str(report_id) + '&from=' + str(from_date) + '&to=' + str(to_date) + '&output=csv')
textfile = report.text
textfile = io.StringIO(textfile)
data = pd.read_csv(textfile, sep=';', parse_dates=True, index_col='Tijdpunten [jaar-maand-dag uur:min:sec]')
return data
else:
print(result['error'])
data = schedule.every().day.at('13:25').do(creds=creds, report_id=29025, from_date='2022-03-10', to_date='2022-07-01')
while True:
schedule.run_pending()
time.sleep(
I am making a weather app and instead of having someone enter their coordinates, I have it where when they enter their city it will pull the coordinates of the city from an API. I then have the lat and lon values that were just pulled stored in two different variables. So what I want to do is be able to insert those lat and lon variables into another API link then make the request for the weather data.
import requests
import math
from datetime import datetime
import tzlocal
api_address1 = 'http://api.openweathermap.org/data/2.5/weather?appid=fAPI KEY='
city = input("City Name :")
url1 = api_address1 + city
json_data1 = requests.get(url1).json()
lat = json_data1['coord']['lat']
lon = json_data1['coord']['lon']
print (lat, lon)
api_address2 = 'https://api.openweathermap.org/data/2.5/onecall?{}=&{}=&appid=API KEY'
sun_rise = json_data1['sys']['sunrise']
local_timezone = tzlocal.get_localzone()
local_time = datetime.fromtimestamp(sun_rise, local_timezone)
print(local_time.strftime("%B %d %Y %H:%M" + "AM"))de here
In such cases you should use formatted string literals
In your example it would work like this:
api_address2 = f'https://api.openweathermap.org/data/2.5/onecall?lat={lat}&lon={lon}&appid=API KEY'
Using Python, how do you read Outlook's Shared Calendar events, and hopefully, also using a time filter?
Here is a relevant post, but to answer this fully:
import win32com.client # for outlook
import datetime
"""This code reads shared calendars."""
# set variables
days = 3
begin = datetime.date.today()
end = begin + datetime.timedelta(days=days)
events = [] # to write results from calendar loop
# begin importing calendar
Outlook = win32com.client.Dispatch("Outlook.Application")
ns = Outlook.GetNamespace("MAPI")
# turn this into a list to read more calendars
recipient = ns.CreateRecipient("username") # cmd whoami to find this
resolved = recipient.Resolve() # checks for username in address book
# olFolderCalendar = 9
# appointments = ns.GetDefaultFolder(9).Items # for personal calendar
appointments = ns.GetSharedDefaultFolder(recipient, 9).Items
# filtering criteria
# https://learn.microsoft.com/en-us/office/vba/api/outlook.items.includerecurrences
appointments.Sort("[Start]") # suspect problem
appointments.IncludeRecurrences = "True"
restriction = "[Start] >= '" + begin.strftime("%m/%d/%Y") \
+ "' AND [End] <= '" + end.strftime("%m/%d/%Y") + "'"
# list of appointments
restrictedItems = appointments.Restrict(restriction)
# loop through all calendar events, and add elements to list
count = 0
for app in restrictedItems:
count += 1 # no len(range(restrictedItems)) allowed
# display values
print()
print("item: " + str(count))
print("start: \t\t" + str(app.Start))
print("subject: \t" + app.Subject)
print("end: \t\t" + str(app.End))
print("recurring: \t" + str(app.IsRecurring))
print("status: \t" + str(app.MeetingStatus))
# collect values
app_instance = [app.Subject,
app.Start,
app.End,
app.BusyStatus]
events.append(app_instance)