Develop Reference

Updates to text file are not being parsed using python

I'm parsing data from a text file ('placlog.txt') that is continuously being updated. As I run the code everything prints as expected, but if there are any updates to the placlog file while the code is running it is not printed.
The placlog file is being updated by a third-party program and I am using the above code to read the file and print any updates.
Once formatted, the text should be sent via a Telegram API. This part is also working initially.
import urllib.parse
import time
import requests
import os
def post_to_telegram(msg):
#print(msg)
base_url = 'https://api.telegram.org/bot&text="{}'.format(msg)
requests.get(base_url)
def check_url_inMsgList(stringToMatch, msgList):
for i in msgList:
if (stringToMatch in i):
return False
return True
try:
f = open("oldFile.txt", "r")
msgList = f.read().split("\n")
f.close()
except:
f = open("oldFile.txt", "w")
msgList = []
f.close()
selections = []
urr = ""
name = ""
pie = ""
ourLines = 2400
url_found = 0
name_found = 0
pie_found = 0
while (True):
file1 = open('placlog.txt', 'r')
Lines = file1.readlines()
file1.close()
while (True):
# print("-------------------------------")
if (ourLines == len(Lines)):
break
elif (ourLines > len(Lines)):
ourLines = 0
else:
txt = Lines[ourLines].strip()
tlist = txt.split("&")
ourLines = ourLines + 1
for subtxt in tlist:
if "eventurl=" in subtxt:
a = subtxt[9:len(subtxt) - 3]
url = "www.awebsite.com/%23" + a.replace("%23", "/")
#url = url.replace("%23", "#")
for i in range(10):
if "F" + str(i) + "/" in url:
url = url.split("F" + str(i) + "/")[0] + "F" + str(i) + "/"
urr = url
url_found = 1
elif "bit=" in subtxt:
name = urllib.parse.unquote(subtxt[4:len(subtxt)])
name_found = 1
elif "pie\":" in subtxt:
a = subtxt.split("price")[1]
pie = a.split("\"")[2]
pie = float(pie)
pie = round(pie, 1)
pie = str(pie)
pie_found = 1
selections.append(url + name + pie)
msg = (url + " " + name + " " + pie)
stringToFind = url + " " + name
if (check_url_inMsgList(stringToFind, msgList)):
post_to_telegram(msg)
msgList.append(msg)
print(msg)
f = open("oldFile.txt", "a+")
f.write(msg + "\n")
f.close()
time.sleep(0.5)
elif "minodds=" in subtxt:
a = subtxt.split("minodds=")[1]
pie = a.split("&")[0]
pie = float(pie)
rie = round(pie, 1)
pie = str(pie)
pie_found = 1
selections.append(url + name + pie)
msg = (url + " " + name + " " + pie)
stringToFind = url + " " + name
if (check_url_inMsgList(stringToFind, msgList)):
post_to_telegram(msg)
msgList.append(msg)
print(msg)
f = open("oldFile.txt", "a+")
f.write(msg + "\n")
f.close()
time.sleep(0.5)
time.sleep(1)

I would recommend using watchdog, and seeing if that helps your situation. It can monitor for file system changes, so you could define a function which is executed when the placlog.txt file is changed/updated.
A good guide can be found here: http://thepythoncorner.com/dev/how-to-create-a-watchdog-in-python-to-look-for-filesystem-changes/
From that guide, you can simply change the functions defined to suit your needs i.e.
def on_modified(event):
if event.src_path == "path/to/placlog.txt":
with open('placlog.txt', 'r') as placlog:
lines = file1.readlines()
Could you try this out and see if it helps? I still recommend the with statement for file i/o since you always want your file to close no matter what.
This link might also be useful since they are also monitoring a single .txt file: Python Watchdog - src_path inconsistent
watchdog documentation: https://pythonhosted.org/watchdog/
Note: Deleted the old answer since you clarified the question.

Is there any way to get source code inside context manager as string?

Source code of function can be received with inspect.getsourcelines(func) function. Is there any way to do same for context manager?
with test():
print('123')
# How to get "print('123')" as line here?

What do you think about this solution?
import traceback
class ContextManagerContent(object):
def __enter__(self):
return
def __exit__(self, _type, value, _traceback):
stack = traceback.extract_stack()
f, last_line = self._get_origin_info(stack)
with open(f) as fin:
lines = list(fin)
search = 'with {cls_name}'.format(cls_name=self.__class__.__name__)
for i, x in enumerate(lines[:last_line + 1][::-1]):
if search in x:
first_line = len(lines) - i
break
selected_lines = lines[first_line:last_line + 1]
print ''.join(selected_lines)
def _get_origin_info(self, stack):
origin = None
for i, x in enumerate(stack[::-1]):
if x[2] == '__exit__':
origin = stack[::-1][i + 1]
break
return origin[0], origin[1] - 1
with ContextManagerContent():
print '123'
print '456'
print '789'
If you save this in a .py file and run it you can see being printed the numbers 123, 456 and 789, after that you can see the block of the context manager.
Notice that I did not handle possibile exceptions or formatting of the output, and some parts can be improved, but I think that this is a good starting point.

Here is another solution inspired from #se7entyse7en's answer. I think it is a bit cleaner and more efficient this way
from inspect import currentframe, getframeinfo
from contextlib import contextmanager
#contextmanager
def debug(string):
# before block
cf = currentframe()
first_line = cf.f_back.f_back.f_lineno
filename = getframeinfo(cf.f_back.f_back).filename
yield
# after block
cf = currentframe()
last_line = cf.f_back.f_back.f_lineno
with open(filename) as f:
lines = f.readlines()[first_line:last_line]
print(string + '\n' + ''.join(lines).rstrip())
if __name__ == '__main__':
with debug("show this code in stdout:"):
a = 1
b = 2
a, b = b, a

a bytes-like object is required, not 'str' JSON File opened as STR

I've only learnt the basics of Python please forgive me but I was not able to determine the fix from the other posts. I open my JSON files with 'r' and I think I'm writing to them in r but it doesn't like that. Changing it to 'r' doesn't help :(
For the following section:
if isinstance(to_write, list):
self.log_file.write(''.join(to_write) + "<r/>")
else:
self.log_file.write(str(to_write) + "<r/>")
self.log_file.flush()
The error I get is: a bytes-like object is required, not 'str'
import math
import time
from random import randint
import json
from instagram.client import InstagramAPI
class Bot:
def __init__(self, config_file, tags_file):
# Loading the configuration file, it has the access_token, user_id and others configs
self.config = json.load(config_file)
# Loading the tags file, it will be keep up to date while the script is running
self.tags = json.load(tags_file)
# Log file to output to html the debugging info about the script
self.filename = self.config["path"] + self.config["prefix_name"] + time.strftime("%d%m%Y") + ".html"
self.log_file = open(self.filename, "wb")
# Initializing the Instagram API with our access token
self.api = InstagramAPI(access_token=self.config["access_token"], client_secret=self.config['client_secret'])
# Likes per tag rate
self.likes_per_tag = math.trunc(min(self.config["follows_per_hour"],
self.config["likes_per_hour"]) / len(self.tags["tags"]))
def save_tags(self):
j = json.dumps(self.tags, indent=4)
f = open('tags.json', 'w')
print >> f, j
f.close()
def insta_write(self, to_write):
if self.filename != self.config["path"] + self.config["prefix_name"] + time.strftime("%d%m%Y") + ".html":
self.log_file.close()
self.filename = self.config["path"] + self.config["prefix_name"] + time.strftime("%d%m%Y") + ".html"
self.log_file = open(self.filename, "wb")
if isinstance(to_write, list):
self.log_file.write(''.join(to_write) + "<r/>")
else:
self.log_file.write(str(to_write) + "<r/>")
self.log_file.flush()
def going_sleep(self, timer):
sleep = randint(timer, 2 * timer)
self.insta_write("SLEEP " + str(sleep))
time.sleep(sleep)
def like_and_follow(self, media, likes_for_this_tag):
try:
var = self.api.user_relationship(user_id=media.user.id)
if self.config["my_user_id"] != media.user.id:
self.insta_write("--------------")
self.insta_write(var)
if var.outgoing_status == 'none':
self.insta_write("LIKE RESULT:")
self.insta_write(self.api.like_media(media_id=media.id))
self.insta_write("FOLLOW RESULT:")
self.insta_write(self.api.follow_user(user_id=media.user.id))
likes_for_this_tag -= 1
self.going_sleep(self.config["sleep_timer"])
else:
self.going_sleep(self.config["sleep_timer"] / 2)
except Exception as e:
self.insta_write(str(e))
self.insta_write("GOING SLEEP 30 min")
time.sleep(1800)
self.like_and_follow(media, likes_for_this_tag)
return likes_for_this_tag
def run(self):
while True:
for tag in self.tags["tags"].keys():
tag = str(tag)
self.insta_write("--------------------")
self.insta_write("TAG: " + tag)
self.insta_write("--------------------")
self.insta_write("--------------------")
self.insta_write("DICTIONARY STATUS:")
for keys, values in self.tags["tags"].items():
self.insta_write(keys)
if values is not None:
self.insta_write(values)
likes_for_this_tag = self.likes_per_tag
while likes_for_this_tag > 0 and self.tags["tags"][tag] != 0:
if self.tags["tags"][tag] is None:
media_tag, self.tags["tags"][tag] = self.api.tag_recent_media(tag_name=tag,
count=likes_for_this_tag)
else:
media_tag, self.tags["tags"][tag] = self.api.tag_recent_media(tag_name=tag,
count=likes_for_this_tag,
max_tag_id=self.tags["tags"][tag])
self.insta_write("API CALL DONE")
if len(media_tag) == 0 or self.tags["tags"][tag] is None:
self.tags["tags"][tag] = 0
likes_for_this_tag = 0
else:
self.insta_write(self.tags["tags"][tag])
self.tags["tags"][tag] = self.tags["tags"][tag].split("&")[-1:][0].split("=")[1]
self.save_tags()
for m in media_tag:
likes_for_this_tag = self.like_and_follow(m, likes_for_this_tag)
if reduce(lambda r, h: r and h[1] == 0, self.tags["tags"].items(), True):
self.insta_write("END")
exit(1)
if __name__ == '__main__':
bot = Bot(open("config_bot.json", "r"), open("tags.json", "r"))
bot.run()

You opened the file as binary:
self.log_file = open(self.filename, "wb")
but are writing str Unicode strings to it. Either open the file in text mode (with an encoding set) or encode each string, separately.
Opening the file in text mode is easiest:
self.log_file = open(self.filename, "w", encoding="utf8")

In my case, the reason for the error was the conflict between json.load function and another function from another module w/ the same name load. Specifying explicitly which load function to use i.e. json.load, solved the problem.

Python code is not returning the full value of an element in an array

The program is intended to display a map with pins showing the locations of institutions that make use of one of our facilities. The program takes a csv file, reads the postcodes, geocodes them and places the pins on the map. The size of the pins is relative to the number of times they have used the facility.
However, when the csv file is uploaded the program generates a map with all the pins over Nigeria. Looking through the output from the program, it seems to be geocoding correctly so I am not sure what is going on. The program geocodes using an offline database as python's urllib is not compatible with the proxy setup at my office.
The program is split into two separate modules, the map generation module and the geocoding module.
Here is the map generation part:
import folium
from bottle import route, run, template, static_file, request
import urllib.request
import urllib.parse
import json
import os
os.system('start geocoder.bat')
institutionList = []
map_osm = folium.Map(location=[55, -2], zoom_start=5)
#route('/spreadsheet.html')
def send_static():
return static_file('spreadsheet.html',root='')
#route('/upload', method='POST')
def do_upload():
category = request.forms.get('category')
upload = request.files.get('upload')
name, ext = os.path.splitext(upload.filename)
if ext not in ('.csv'):
return 'File extension not allowed.'
upload.save('uploads/' + upload.filename)
fileList = []
with open('spreadsheetList','r') as f:
while True:
line = f.readline()
if not line: break
print(line.strip())
print("line should have just printed")
fileList.append(line.strip())
f.close()
lengthFileList = len(fileList)
x = 0
while x < lengthFileList:
with open(('uploads/' + fileList[x]),'r') as spread:
while True:
line = spread.readline()
if not line: break
institutionDetails = line.split(',')
institutionList.append(institutionDetails)
spread.close()
x = x + 1
spreadsheetName = upload.filename
f = open('spreadsheetList','a')
f.write(spreadsheetName + '\n')
f.close()
with open('uploads/' + spreadsheetName, 'r') as f:
while True:
line = f.readline()
if not line: break
institutionDetails = line.split(',')
institutionList.append(institutionDetails)
print(institutionList)
f.close()
lengthOfList = len(institutionList)
x = 0
coords = []
while x < lengthOfList:
address = urllib.parse.quote_plus(institutionList[x][1])
response = urllib.request.urlopen('http://localhost:80/geoCodeRequest/' + address).read().decode('utf-8')
cleanResponse = str(response).replace('"','')
coords = cleanResponse
print(cleanResponse)
institutionList[x].append(coords)
x = x + 1
print("http sources successfully accessed")
print(institutionList)
x = 0
while x < lengthOfList:
try:
map_osm.circle_marker(location=institutionList[x][3], radius=(int(institutionList[x][2]) * 10),popup=institutionList[x][0], line_color='#3186cc',fill_color='#3186cc', fill_opacity=0.2)
print("marker added")
except:
print("marker could not be added")
x = x + 1
map_osm.create_map(path='osm.html')
return '<meta http-equiv="refresh" content="0; url=osm.html">'
#route('/osm.html')
def send_static():
return static_file('osm.html',root='')
run(host='localhost', port=8080)
A batch file is used to start the second module:
#echo off
python geocodeProxyBypass.py
Here is the second module of code, the geocoding module:
from bottle import route, run, template
import string
location = []
x = 0
#route('/geoCodeRequest/<name>')
def redir(name):
x = 0
print(name)
print(name[:4])
print(name[:3])
with open('ukPostcode.csv','r') as f:
while True:
line = f.readline()
print(line)
if not line: break
locationDetails = line.split(',')
location.append(locationDetails)
print(location[x][0])
if location[x][0] == ('"' + name[:4] + '"'):
coords = location[x][3] + ", " + location[x][4]
return coords
elif location[x][0] == ('"' + name[:3] + '"'):
coords = location[x][3] + ", " + location[x][4]
return ((coords.replace('"','')))
else:
print("no match found for " + name)
x = x + 1
f.close()
run(host='localhost', port=80)
Here is what an example pin generated by the program should look like:
var circle_1 = L.circle([51.74, -1.25
], 7460, {
color: '#3186cc',
fillColor: '#3186cc',
fillOpacity: 0.2
});
circle_1.bindPopup("University of Oxford");
circle_1._popup.options.maxWidth = 300;
map.addLayer(circle_1)
Here is what is actually being output:
var circle_1 = L.circle([5, 1
], 7460, {
color: '#3186cc',
fillColor: '#3186cc',
fillOpacity: 0.2
});
circle_1.bindPopup("University of Oxford");
circle_1._popup.options.maxWidth = 300;
map.addLayer(circle_1)
Apologies for the very long question, please help!

python CDATA in subelement

I'm trying to add CDATA to an subElement of XML. But the XML keeps converting the < character to <.
I've read some posts here that do something with CDATA, but I can't believe it's this hard to make it work. Besides that, I can't get those examples working.
Below a part of my code simplified, using python 3.4.
import xml.etree.cElementTree as ET
from xml.dom import minidom
class MyClass():
def __init__(self):
self.xml = None
def generateXML(self):
self.xml = ET.Element("AVXML")
row = ET.SubElement(self.xml, "ROW")
ET.SubElement(row, "DATA").text = "<![CDATA[ <ART_HDR.COMMENT1>.PDF ]]>"
def saveXML(self):
rough_string = ET.tostring(self.xml, 'Windows-1252')
reparsed = minidom.parseString(rough_string)
prettyxml = reparsed.toprettyxml(indent=" ", encoding="Windows-1252")
print(prettyxml)
f = open("filetosave.xml", "wb")
f.write(prettyxml)
f.close()
m = MyClass()
m.generateXML()
m.saveXML()
This however generates:
<DATA><![CDATA[ <ART_HDR.COMMENT1>.PDF ]]></DATA>
Instead of
<DATA><![CDATA[ <ART_HDR.COMMENT1>.PDF ]]></DATA>

Oke, I used this comment and got it working.
import xml.etree.cElementTree as ET
from xml.dom import minidom
class MyClass():
def __init__(self):
self.xml = None
def generateXML(self):
self.xml = ET.Element("AVXML")
row = ET.SubElement(self.xml, "ROW")
data = " <ART_HDR.COMMENT1>.PDF "
cdata = ET.SubElement(row, "DATA")
cdata.append(ET.Comment(' --><![CDATA[' + data.replace(']]>', ']]]]><![CDATA[>') + ']]><!-- '))
def saveXML(self):
rough_string = ET.tostring(self.xml, 'Windows-1252')
reparsed = minidom.parseString(rough_string)
prettyxml = reparsed.toprettyxml(indent=" ", encoding="Windows-1252")
f = open("filetosave.xml", "wb")
f.write(prettyxml)
f.close()
m = MyClass()
m.generateXML()
m.saveXML()