Develop Reference

Updates to text file are not being parsed using python

I'm parsing data from a text file ('placlog.txt') that is continuously being updated. As I run the code everything prints as expected, but if there are any updates to the placlog file while the code is running it is not printed.
The placlog file is being updated by a third-party program and I am using the above code to read the file and print any updates.
Once formatted, the text should be sent via a Telegram API. This part is also working initially.
import urllib.parse
import time
import requests
import os
def post_to_telegram(msg):
#print(msg)
base_url = 'https://api.telegram.org/bot&text="{}'.format(msg)
requests.get(base_url)
def check_url_inMsgList(stringToMatch, msgList):
for i in msgList:
if (stringToMatch in i):
return False
return True
try:
f = open("oldFile.txt", "r")
msgList = f.read().split("\n")
f.close()
except:
f = open("oldFile.txt", "w")
msgList = []
f.close()
selections = []
urr = ""
name = ""
pie = ""
ourLines = 2400
url_found = 0
name_found = 0
pie_found = 0
while (True):
file1 = open('placlog.txt', 'r')
Lines = file1.readlines()
file1.close()
while (True):
# print("-------------------------------")
if (ourLines == len(Lines)):
break
elif (ourLines > len(Lines)):
ourLines = 0
else:
txt = Lines[ourLines].strip()
tlist = txt.split("&")
ourLines = ourLines + 1
for subtxt in tlist:
if "eventurl=" in subtxt:
a = subtxt[9:len(subtxt) - 3]
url = "www.awebsite.com/%23" + a.replace("%23", "/")
#url = url.replace("%23", "#")
for i in range(10):
if "F" + str(i) + "/" in url:
url = url.split("F" + str(i) + "/")[0] + "F" + str(i) + "/"
urr = url
url_found = 1
elif "bit=" in subtxt:
name = urllib.parse.unquote(subtxt[4:len(subtxt)])
name_found = 1
elif "pie\":" in subtxt:
a = subtxt.split("price")[1]
pie = a.split("\"")[2]
pie = float(pie)
pie = round(pie, 1)
pie = str(pie)
pie_found = 1
selections.append(url + name + pie)
msg = (url + " " + name + " " + pie)
stringToFind = url + " " + name
if (check_url_inMsgList(stringToFind, msgList)):
post_to_telegram(msg)
msgList.append(msg)
print(msg)
f = open("oldFile.txt", "a+")
f.write(msg + "\n")
f.close()
time.sleep(0.5)
elif "minodds=" in subtxt:
a = subtxt.split("minodds=")[1]
pie = a.split("&")[0]
pie = float(pie)
rie = round(pie, 1)
pie = str(pie)
pie_found = 1
selections.append(url + name + pie)
msg = (url + " " + name + " " + pie)
stringToFind = url + " " + name
if (check_url_inMsgList(stringToFind, msgList)):
post_to_telegram(msg)
msgList.append(msg)
print(msg)
f = open("oldFile.txt", "a+")
f.write(msg + "\n")
f.close()
time.sleep(0.5)
time.sleep(1)

I would recommend using watchdog, and seeing if that helps your situation. It can monitor for file system changes, so you could define a function which is executed when the placlog.txt file is changed/updated.
A good guide can be found here: http://thepythoncorner.com/dev/how-to-create-a-watchdog-in-python-to-look-for-filesystem-changes/
From that guide, you can simply change the functions defined to suit your needs i.e.
def on_modified(event):
if event.src_path == "path/to/placlog.txt":
with open('placlog.txt', 'r') as placlog:
lines = file1.readlines()
Could you try this out and see if it helps? I still recommend the with statement for file i/o since you always want your file to close no matter what.
This link might also be useful since they are also monitoring a single .txt file: Python Watchdog - src_path inconsistent
watchdog documentation: https://pythonhosted.org/watchdog/
Note: Deleted the old answer since you clarified the question.

full URL isn't being parsed correctly to a telegram bot

I have written the following to automate the parsing of text to a telegram bot from a .txt file that is continuously being updated.
import urllib.parse
import time
import requests
def post_to_telegram(msg):
print(msg)
base_url = 'https://api.telegram.org/bot&text="{}"'.format(msg)
requests.get(base_url)
urr = ""
name = ""
price = ""
ourLines=0
while(True):
file1 = open('example.txt', 'r')
Lines = file1.readlines()
time.sleep(1)
while(True):
if(ourLines==len(Lines)):
break
else:
txt = Lines[ourLines].strip()
tlist = txt.split("&")
ourLines=ourLines+1
for subtxt in tlist:
if "eventurl=" in subtxt:
a = subtxt[9:len(subtxt) - 3]
url = 'www.bbey43.com/#'+a.replace("%23", "/")
#print(url)
urr = url
elif "bet=" in subtxt:
name = urllib.parse.unquote(subtxt[4:len(subtxt)])
#print(name)
elif "price\":" in subtxt:
a = subtxt.split("price")[1]
price = a.split("\"")[2]
#print(price)
post_to_telegram(urr + " "+ name + " " + price)
the 'name' & 'price' is successfully posted to the bot, but the 'url' doesn't post correctly. The only thing that gets through is "bbey43.com/#/

The solution to this was rather simple in the end. As the "#" was apart of a URL it required special formatting when being parsed.
Simply adding %23 instead of # solved it.

Python Unittest testing program with input and output file

I made a program which has one function. This function has a file as an input and function writes result to output. I need to test if the my result is the same as expected. Below you can find a code of a program:
import os
def username(input):
with open(input, 'r') as file:
if os.stat(input).st_size == 0:
print('File is empty')
else:
print('File is not empty')
for line in file:
count = 1
id, first, middle, surname, department = line.split(":")
first1 = first.lower()
middle1 = middle.lower()
surname1 = surname.lower()
username = first1[:1] + middle1[:1] + surname1
username1 = username[:8]
if username1 not in usernames:
usernames.append(username1)
data = id + ":" + username1 + ":" + first + ":" + middle + ":" + surname + ":" + department
else:
username2 = username1 + str(count)
usernames.append(username2)
data = id + ":" + username2 + ":" + first + ":" + middle + ":" + surname + ":" + department
count += 1
with open("output.txt", "a+") as username_file:
username_file.write(data)
usernames = []
if __name__ == '__main__':
username("input_file1.txt")
username("input_file2.txt")
username("input_file3.txt")
with open("output.txt", "a+") as username_file:
username_file.write("\n")
How do I write an unittest on this type of program? I tried this but it gave me this error "TypeError: expected str, bytes or os.PathLike object, not _io.TextIOWrapper" . Code of my test is below:
import unittest
import program.py
class TestProgram(unittest.TestCase):
def test_username(self):
i_f = open("input_file1.txt", 'r')
result = program.username(i_f)
o_f = open("expected_output.txt", 'r')
self.assertEqual(result, o_f)
if __name__ == '__main__':
unittest.main()
I would be really happy if you could help me!!!

You didn't read the file, just pass the IO object.
Edit the TestProgram and add .read()
class TestProgram(unittest.TestCase):
def test_username(self):
i_f = open("input_file1.txt", 'r').read() # read the file
result = program.username(i_f)
o_f = open("expected_output.txt", 'r').read()
self.assertEqual(result, o_f)
you can also use with-as for automatic closing the file.

writing data in file using python

I am new in programming. Just bought a book for beginners in Python. In it I got this code:
name = input("name")
email = input("whats ure email:)
favoriteband = input("ure fav band")
outputString = name + "|" email + "|" + favoriteband
fileName = name + ".txt"
file = open(fileName, "wb")
file.write (outputString)
print (outputString , " saved in ", fileName)
file.close ()
According to book its fine but I got this error:
TypeError: a bytes-like object is required, not 'str'
I got no clue how to fix it and book isn't explaining this as well.

Let's go through this:
name = input("Your name: ")
email = input("Your email: ")
The close quotes are needed as has been pointed out.
outputString = name + "|" + email + "|" + favoriteband
outputString was missing a + before email
Finally, we need to rewrite you file management:
with open(fileName, "a") as file:
file.write (outputString)
print (outputString , " saved in ", fileName)
Writing this as a with statement guarantees it will close. Using open(..., "a") opens the file in "append" mode and lets you write multiple strings to a file of the same name.
Finally, if I can editorialize, I am not a fan of this book so far.
Edit: here is the whole code with fixes, in hopes of getting you there.
name = input("name")
email = input("whats ure email:")
favoriteband = input("ure fav band")
outputString = name + "|" + email + "|" + favoriteband
fileName = name + ".txt"
with open(fileName, "a") as file:
file.write (outputString)
print (outputString , " saved in ", fileName)
You can verify it works with:
with open(fileName, "r") as file:
print(file.read())

I did some editing (closing quotes and a missing +):
name = input("name:")
email = input("whats ure email:")
favoriteband = input("ure fav band:")
outputString = name + " | " + email + " | " + favoriteband
fileName = name + ".txt"
file = open(fileName, "w") #opened in write mode but not in binary
file.write (outputString)
print (outputString , " saved in ", fileName)
file.close()
You're getting that error because you're writing in binary mode, hence the b in wb for
file = open(fileName, "wb")
Try this instead :
file = open(fileName, "w")

How avoid URL error in my code python and open the file only one time?

I would like to know how avoid the URL error in my script.and would like to know the way to open only one time the file(not two like in my code)without create an error when run it.
(actual code work)
the help could be here :
http://docs.python.org/2/library/urllib2.html#urllib2.URLError
with "try" and "exept" ?
can you advice on this than this could work on my code ?
for the error url would like to post a message for user when it's wrong :
for exemple i change the adress :
and when run : error appear now
so i want to put like "bad URL please check" etc
+
with the change i would like it print new data on a new line without deleting the old data,like this :
here my code :
#Source : http://www.wunderground.com/weather/api/d/docs?d=resources/code-samples
import urllib2
import json
import time
import csv
from datetime import datetime#set the time
try:
wunder_url_obj = urllib2.urlopen('http://api.wunderground.com/api/8d3b5d3fa03ddb6f/conditions/weather/q/China/Beijing.json')
except:
print 'Could not open URL'
exit()
else:
now = datetime.now()
current_year = now.year
current_day = now.day
current_month = now.month
current_hour = now.hour
current_minute = now.minute
current_second = now.second
json_string = wunder_url_obj.read()
parsed_json = json.loads(json_string)
temp_f = parsed_json['current_observation']['temp_f']
weather = parsed_json['current_observation']['weather']
#--- Open the file + write on it ---
prev_data = open('out.csv', 'r').read()
# Add a header only if the file is empty
if prev_data == '':
with open('out.csv','a') as f:
header = "Datetime,current condition,Temperature,\n"
prev_data.write(header)
date = str(now.month) + "/" + str(now.day) + "/" + str(now.year) + " " + str(now.hour) + ":" + str(now.minute) + ":" + str(now.second)
prev_data.write(','.join([date,weather,str(temp_f)]))
prev_data.write('\n')
in my editor i tried this too ;

First off, I'd try to keep your variable names different (you have two different f). Now for the try: except
try:
wunder_url_obj = urllib2.urlopen(api_wunder_url_opath)
except:
print 'Could not open URL'
sys.exit()
I'm assuming that you'd want to exit on a bad URL open - but that's up to you, do what you want after the except.
You are attempting to open an already opened file (out.csv). Try tidying up your file read / write - maybe try using the with statement to keep things clean and simple:
#--- Open the file + write on it ---
prev_data = open('out.csv', 'r').read()
# Add a header only if the file is empty
if prev_data == '':
with open('out.csv','a') as f:
header = "Datetime,current condition,Temperature,\n"
f.write(header)
date = str(now.month) + "/" + str(now.day) + "/" + str(now.year) + " " + str(now.hour) + ":" + str(now.minute) + ":" + str(now.second)
f.write(','.join([date,weather,str(temp_f)]))
f.write('\n')
# --- And Close the file ---
Edit
Rather than re-write your code for you, I encourage you to rethink your problem. You want to
Read information from an URL
Write that information to a file
Repeat (goto 1)
The important thing here is to note that we're accessing the file write in a regular order - once to write the header, and then sequentially to write the information.
Maybe a better approach would be to do something like
def get_information(url):
try:
wunder_url_obj = urllib2.urlopen('http://api.wunderground.com/api/8d3b5d3fa03ddb6f/conditions/weather/q/China/Beijing.json')
except:
print 'Could not open URL'
return None
now = datetime.now()
current_year = now.year
current_day = now.day
current_month = now.month
current_hour = now.hour
current_minute = now.minute
current_second = now.second
json_string = wunder_url_obj.read()
parsed_json = json.loads(json_string)
temp_f = parsed_json['current_observation']['temp_f']
weather = parsed_json['current_observation']['weather']
date = str(now.month) + "/" + str(now.day) + "/" + str(now.year) + " " + str(now.hour) + ":" + str(now.minute) + ":" + str(now.second)
return ','.join([date, weather, str(temp_f)]) + '\n'
so this function can be called over and over again and will return a line for your csv file each time. Then just call it and write to file;
with open(my_csv_file, 'w') as fo:
fo.write(header)
for i in range(10): # or whatever time scale you have
fo.write(get_information(my_url))