I've a problem when I have to display the results as a histogram with the plt.show() function, I need the function to be non-blocking because I need the return value of the principal funciont to make new calculations and show a new histogram.
(P.S. i'm using PyCharm ide and Python 3.6.7rc2 matplotlib 3.0.0)
This is the function in the code:
def plotHistStd(data_file, attribute, number, method):
original = copy.deepcopy(data_file)
modified = copy.deepcopy(data_file)
tmp_arr = []
tmp_arr2 = []
if method == 1:
for i in range(0, original.__len__()):
if original[i] > 0:
if number > 0:
tmp = log1p(original[i]+number)
tmp_arr.append(copy.deepcopy(original[i]))
tmp_arr2.append(copy.deepcopy(tmp.real))
else:
tmp = np.log1p(original[i]+number)
tmp_arr.append(copy.deepcopy(original[i]))
tmp_arr2.append(copy.deepcopy(tmp.real))
else:
tmp = np.log1p(original[i]+number)
tmp_arr.append(copy.deepcopy(original[i]))
tmp_arr2.append(copy.deepcopy(tmp.real))
modified[i] = tmp.real
elif method == 2:
for i in range(0, original.__len__()):
if original[i] > 0:
tmp2 = pow((original[i]), (1.0/number))
tmp_arr.append(copy.deepcopy(original[i]))
tmp_arr2.append(copy.deepcopy(tmp2.real))
else:
tmp2 = cm.exp(number*cm.log(original[i]))
tmp_arr.append(copy.deepcopy(original[i]))
tmp_arr2.append(copy.deepcopy(tmp2.real))
modified[i] = tmp2.real
v_max = max(tmp_arr)
v_min = min(tmp_arr)
v_max2 = max(tmp_arr2)
v_min2 = min(tmp_arr2)
v_max = max(v_max, v_max2)
v_min = min(v_min, v_min2)
fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax2 = fig.add_subplot(2, 1, 2)
n, bins, patches = ax1.hist(original, color="blue", bins=15, range=(v_min, v_max))
ax1.set_xlabel(attribute)
ax1.set_ylabel('Frequency')
n, bins, patches = ax2.hist(modified, color="red", bins=15, range=(v_min, v_max))
ax2.set_xlabel(attribute)
ax2.set_ylabel('Frequency')
plt.subplots_adjust(hspace=0.3)
plt.xticks(rotation=90)
plt.show() #need these non-blocking version
return modified
The function is called from the main window, developed in another Python class file.
I tried to insert plt.ion() after plt.show() but it does not work.
P.s. add the function in the main window that calls that indicates in precedence:
def apply_stdandardization(self):
if self.file[self.file.__len__() - 4:self.file.__len__()] != ".csv":
QtWidgets.QMessageBox.warning(QtWidgets.QWidget(),"Alert Message", "No CSV file was selected!")
else:
number = self.lineEdit_2.text()
index = self.comboBox_2.currentText()
std_method = 0
#data_file = pd.read_csv(self.file)
number = float(number)
try:
number = float(number)
except Exception:
QtWidgets.QMessageBox.warning(QtWidgets.QWidget(), "Alert Message",
"Insert only numeric value")
if self.rad_log.isChecked():
std_method = 1
elif self.rad_exp.isChecked():
std_method = 2
self.new_df = std_function(str(index), number, std_method)
EDIT:
I tried to insert plt.ion() before plt.show(), actually the program is not blocked, but the histogram window does not open correctly and if I click on it, the program crashes with the following errors:
Fatal Python error: PyEval_RestoreThread: NULL tstate
Thread 0x000016c0 (most recent call first):
File "C:\Python36\lib\threading.py", line 299 in wait
File "C:\Python36\lib\threading.py", line 551 in wait
File "C:\PyCharm\helpers\pydev\pydevd.py", line 128 in _on_run
File "C:\PyCharm\helpers\pydev\_pydevd_bundle\pydevd_comm.py", line 320 in run
File "C:\Python36\lib\threading.py", line 916 in _bootstrap_inner
File "C:\Python36\lib\threading.py", line 884 in _bootstrap
Thread 0x00000fb8 (most recent call first):
File "C:\PyCharm\helpers\pydev\_pydevd_bundle\pydevd_comm.py", line 382 in _on_run
File "C:\PyCharm\helpers\pydev\_pydevd_bundle\pydevd_comm.py", line 320 in run
File "C:\Python36\lib\threading.py", line 916 in _bootstrap_inner
File "C:\Python36\lib\threading.py", line 884 in _bootstrap
Thread 0x00000b28 (most recent call first):
File "C:\Python36\lib\threading.py", line 299 in wait
File "C:\Python36\lib\queue.py", line 173 in get
File "C:\PyCharm\helpers\pydev\_pydevd_bundle\pydevd_comm.py", line 459 in _on_run
File "C:\PyCharm\helpers\pydev\_pydevd_bundle\pydevd_comm.py", line 320 in run
File "C:\Python36\lib\threading.py", line 916 in _bootstrap_inner
File "C:\Python36\lib\threading.py", line 884 in _bootstrap
Current thread 0x00000b08 (most recent call first):
File "C:/Users/Enrico/PycharmProjects/PythonDataset/PDA.py", line 681 in <module>
File "C:\PyCharm\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18 in execfile
File "C:\PyCharm\helpers\pydev\pydevd.py", line 1135 in run
File "C:\PyCharm\helpers\pydev\pydevd.py", line 1735 in main
File "C:\PyCharm\helpers\pydev\pydevd.py", line 1741 in <module>
Process finished with exit code 255
Related
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexes\range.py", line 351, in get_loc
return self._range.index(new_key)
ValueError: 0 is not in range
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\tkinter\__init__.py", line 1892, in __call__
return self.func(*args)
File "<ipython-input-18-518ef7ebbd84>", line 4, in myClick2
a.plot(p1.total_payoff()[0],p1.total_payoff()[1])
File "<ipython-input-10-57724152e4a6>", line 59, in total_payoff
prices = self.option_payoff()[0]
File "<ipython-input-10-57724152e4a6>", line 47, in option_payoff
temppayoff += callpayoff(i,j.get_strike(),j.find_bidask()[1])*j.get_quantity()
File "<ipython-input-6-59b6ad5c0680>", line 28, in find_bidask
bid = data['bid'][0]
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\series.py", line 853, in __getitem__
return self._get_value(key)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\series.py", line 961, in _get_value
loc = self.index.get_loc(label)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexes\range.py", line 353, in get_loc
raise KeyError(key) from err
KeyError: 0
This is the error message I am getting when I try to execute a function by clicking a TKinter button. The function is below, basically it takes some data (x, y), and plots it with matplotlib.
def myClick2():
f = Figure(figsize=(4,4), dpi=100)
a = f.add_subplot(111)
a.plot(p1.total_payoff()[0],p1.total_payoff()[1])
a.grid(True, which='both')
a.axhline(y=0, color='k')
a.axvline(x=0, color='k')
canvas = FigureCanvasTkAgg(f, master=root)
canvas.draw()
canvas.get_tk_widget().grid(row = 8, column = 1)
The error is saying that "total_payoff", which calls "option_payoff", which calls "find_bidask" is leading to the error. Specifically, the part which I assign bid = data['bid'][0].
def find_bidask(self):
if str.upper(self.cp) == 'C':
data = self.data['calls']
else:
data = self.data['puts']
data = data[data['contractSymbol']==self.symbol].reset_index(drop=True)
bid = data['bid'][0]
ask = data['ask'][0]
However, when I run this separately outside of TKinter, it produces no error, and ['bid'][0] is available as a value. I don't understand what is wrong with my code - is it something in the tkinter myclick2 function that is wrong?
I have a long running python script for a weather station. The script captures data from the weather sensors and uploads them to a mySql database as well as weather underground.
The issue I am having is that the script will run for multiple days and then crashes without a traceback. I have been troubleshooting this for weeks to no avail. I was initially running the script in IDLE. I have also ran it in the terminal. The latest update that I "think" is the real issue is that I ran the script in Visual Studio Code and when it crashed it posted: munmap_chunk() invalid pointer in the terminal.
My understanding is that munmap_chunk is a C issue so it is most likely being generated by a module that I import. I also understand that is is thrown when a pointer is passed to free() that hasn't been obtained through malloc().
With that said, I don't know how to narrow down where the issue is actually being generated from so I can potentially correct it.
Prior to seeing the munmap_chunk error I had thought maybe it was a memory leak somewhere. I manually called the garbage collector which in almost all cases you shouldn't do. That allowed for the script to run longer before the it finally crashed again.
The main thing I'm trying to determine is how to zero in on where the issue is coming from since the script just crashes and I don't get any traceback information or exception being thrown.
#!/usr/bin/python3
from gpiozero import Button
import time
import math
import Wind_Direction
import statistics
import Database
import requests
import sys
import logging
import board
from adafruit_bme280 import basic as adafruit_bme280
logging.basicConfig(format='%(asctime)s **%(levelname)4s** - %(module)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', filename='/home/testa/testb/testc.txt', filemode = 'w', level=logging.DEBUG)
logging.debug("The program has started")
print ("\nWeather_Station_BYO: The program has started")
i2c = board.I2C() # uses board.SCL and board.SDA
# ------- WU URL and credentials ---------
WUurl = "https://weatherstation.wunderground.com/weatherstation/updateweatherstation.php?"
WUid = "TESTEST" # add your WS ID
WUpwd = "TESTEST" # add your password
WUcreds = "ID=" + WUid + "&PASSWORD="+ WUpwd
wind_count = 0 # Count how many half-rotations of the anemometer
radius_in = 3.54 # Radius of your anemometer in inches (9 cm)
wind_interval = 5 # How often (secs) to report speed
interval = 300 # 5*60 seconds = 5 minutes = 300 seconds
inches_in_mile = 63360
SECS_IN_AN_HOUR = 3600
ADJUSTMENT = 1.18 # Anemometer adjustment
BUCKET_SIZE = 0.011 # Rain gauge tips once when (.2794 mm = .011 in) of rain falls
rain_count = 0
store_speeds = []
store_directions = []
def spin():
"""Counts the number of every half-rotation of the anenmometer and returns that count"""
global wind_count
wind_count = wind_count + 1 # Every half-rotation, add 1 to count
# print("spin" + str(wind_count))
def calculate_speed(time_sec):
"""Uses the number of half-rotations stored in the global variable wind_count and the time used to count the rotations and generates a final wind speed"""
global wind_count
circumference_in = (2 * math.pi) * radius_in
rotations = wind_count / 2.0
dist_in = (circumference_in * rotations) / inches_in_mile # Calculate distance travelled by a cup in inches
mile_per_sec = dist_in / time_sec
mph = mile_per_sec * SECS_IN_AN_HOUR
final_speed = mph * ADJUSTMENT
# print("final speed" + str(final_speed))
return final_speed
def bucket_tipped():
"""Counts the number of times the rain gauge bucket filled and tipped"""
global rain_count
rain_count = rain_count + 1
# print(rain_count * BUCKET_SIZE)
def reset_rainfall():
"""Resets the global rain_count variable set in the bucket_tipped function"""
global rain_count
rain_count = 0
def reset_wind():
"""Resets the global wind_count variable set in the spin function"""
global wind_count
wind_count = 0
def read_all():
"""Reads bme280 (temp, humididty, pressure) values converts them to proper units and calculates the dewpoint"""
bme280 = adafruit_bme280.Adafruit_BME280_I2C(i2c)
bme280.sea_level_pressure = 1013.25 # change this to match the location's pressure (hPa) at sea level
humidity = bme280.humidity # % Relative humidity
pressure = bme280.pressure * 100 / 3386 # Pascals converted to inHg
ambient_temperature = bme280.temperature * 9/5 + 32 # Convert to fahrenheit rounded
b = 17.62
c = 243.12
gamma = (b * bme280.temperature /(c + bme280.temperature)) + math.log(bme280.humidity / 100.0)
dewpoint = (c * gamma) / (b - gamma)
return humidity, pressure, ambient_temperature, dewpoint
wind_speed_sensor = Button(5)
wind_speed_sensor.when_pressed = spin # Whenever the wind sensor magnet closes the reed switch then add 1 to the number of rotations
# temp_probe = ds18b20_therm.DS18B20()
rain_sensor = Button(6)
rain_sensor.when_pressed = bucket_tipped
db = Database.weather_database()
i = 1
while db.wxlogin() != True:
time.sleep(30)
i = i + 1
if (i>5):
logging.debug("Could not establish a connection to the mysql database and program will be terminated.")
time.sleep(5)
sys.exit("Could not establish connection") # Exit the script after i amount of connection attempts
logging.debug("Initial connection to database was successful!")
print("\nWeather_Station_BYO: Initial connection to database was successful!")
# Loop to measure wind speed at 300-second intervals
while True:
try:
start_time = time.time()
while time.time() - start_time <= interval:
# print("In loop 1")
wind_start_time = time.time()
reset_wind()
while time.time() - wind_start_time <= wind_interval:
# print("In loop 2")
store_directions.append(Wind_Direction.get_value(wind_interval)) # wind_interval is how long to take measuremennts for wind direction
final_speed = calculate_speed(wind_interval) # Add this speed to the list & wind_interval is how long to take measurements for wind speed
store_speeds.append(final_speed)
logger = logging.getLogger(__name__)
logger.debug("Wind speed and direction obtained")
# print("Out of both loops")
# print("stored directions " + str(store_directions))
# print("stored speeds " + str(store_speeds))
wind_dir = Wind_Direction.get_average(store_directions)
wind_gust = max(store_speeds)
wind_speed = statistics.mean(store_speeds)
rainfall = rain_count * BUCKET_SIZE
reset_rainfall()
store_speeds = []
store_directions = []
humidity, pressure, ambient_temp, dewpoint = read_all()
logger.debug("Humidity, pressure, ambient_temp and dewpoint obtained")
# print(wind_dir)
if 348.75 <= wind_dir or wind_dir <= 11.25:
wind_direction = 'N'
elif 11.26 <= wind_dir <= 33.75:
wind_direction = 'NNE'
elif 33.76 <= wind_dir <= 56.25:
wind_direction = 'NE'
elif 56.26 <= wind_dir <= 78.75:
wind_direction = 'ENE'
elif 78.76 <= wind_dir <= 101.25:
wind_direction = 'E'
elif 101.25 <= wind_dir <= 123.75:
wind_direction = 'ESE'
elif 123.76 <= wind_dir <= 146.25:
wind_direction = 'SE'
elif 146.26 <= wind_dir <= 168.75:
wind_direction = 'SSE'
elif 168.76 <= wind_dir <= 191.25:
wind_direction = 'S'
elif 191.26 <= wind_dir <= 213.75:
wind_direction = 'SSW'
elif 213.76 <= wind_dir <= 236.25:
wind_direction = 'SW'
elif 226.26 <= wind_dir <= 258.75:
wind_direction = 'WSW'
elif 258.76 <= wind_dir <= 281.25:
wind_direction = 'W'
elif 281.26 <= wind_dir <= 303.75:
wind_direction = 'WNW'
elif 303.76 <= wind_dir <= 326.25:
wind_direction = 'NW'
elif 326.25 <= wind_dir <= 348.75:
wind_direction = 'NNW'
else: wind_direction = ''
#print('',wind_direction, 'Wind Direction \n',round(wind_speed,1), 'Wind Speed (mph) \n', round(wind_gust,1),'Wind Gust (mph) \n', round(rainfall,1), 'Rainfall (in) \n',
# round(humidity,1), '% Relative Humidity \n', round(dewpoint,1), 'Dew Point F \n', round(pressure,2), 'Barometric Pressure (inHg) \n', round(ambient_temp,1), 'Temperature F \n')
i = 1
while db.check_connection() != True:
time.sleep(30)
i = i + 1
if (i>5):
logger.debug("Connection to the mysql database was disconnected and connection could not be restablished. Program will be terminated.")
time.sleep(5)
sys.exit("Could not re-establish connection") # Exit the script after i amount of connection attempts
db.wxlogin()
db.insert(1,round(ambient_temp,1), 0, 0, round(pressure,2), round(humidity,1), round(dewpoint,1), wind_direction, round(wind_speed,1), round(wind_gust,1), round(rainfall,1)) # 10 items
# Weatherunderground
# print("Uploading to Weather Underground")
f_date = "&dateutc=now"
f_humid = "&humidity=" + str(humidity) # % Relative humidity
f_wspeed = "&windspeedmph=" + str(wind_speed) # mph
f_gust = "&windgustmph=" + str(wind_gust) # mph
f_airtemp = "&tempf=" + str(ambient_temp) # degrees F
f_rain = "&rainin=" + str(rainfall) # inches
f_press = "&baromin=" + str(pressure) # inches
# f_groundtemp = "&soiltempf=" + str(ground_temp_f) # degrees F
f_winddir = "&winddir=" + str(wind_dir) # degrees
f_action = "&action=updateraw"
try:
r = requests.get(WUurl+WUcreds+f_date+f_humid+f_wspeed+f_winddir+f_gust+f_airtemp+f_rain+f_press+f_action, timeout = 30)
print("\nReceived " + str(r.status_code) + " " + str(r.text) + " from WU")
except requests.ConnectionError as e:
logger.debug("HTTP Error - Connection Error. Make sure you are connected to the internet. Technical details given below.\n")
logger.debug(str(e))
print("\nHTTP Error - Connection Error. Make sure you are connected to internet. Technical details given below.\n")
print(str(e))
continue
except requests.Timeout as e:
logger.debug("HTTP Error - Timeout Error")
logger.debug(str(e))
print("\nHTTP Error - Timeout Error")
print(str(e))
continue
except requests.RequestException as e:
logger.debug("HTTP Error - General Error")
logger.debug(str(e))
print("\nHTTP Error - General Error")
print(str(e))
continue
except KeyboardInterrupt:
print("\nSomeone closed the program")
if (str(r.status_code) != '200'):
logger.debug("Upload to Weather Underground was not sucessful. HTTP Response Code: " + str(r.status_code) + " " + str(r.text))
except Exception:
logger.exception("Unhandled exception in the script!", exc_info = True)
Traceback from faulthandler in VSCode:
munmap_chunk(): invalid pointer
Fatal Python error: Aborted
Thread 0xb0fa5440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/usr/lib/python3/dist-packages/gpiozero/mixins.py", line 539 in held
File "/usr/lib/python3.7/threading.py", line 865 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py", line 1054 in __call__
Thread 0xb27ff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/usr/lib/python3/dist-packages/gpiozero/mixins.py", line 543 in held
File "/usr/lib/python3.7/threading.py", line 865 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py", line 1054 in __call__
Thread 0xb31ff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/pydevd.py", line 246 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb3bff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/pydevd.py", line 200 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb45ff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 296 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_timeout.py", line 43 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb4fff440 (most recent call first):
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py", line 219 in _read_line
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py", line 237 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb599b440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/queue.py", line 179 in get
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py", line 382 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Current thread 0xb6f34b40 (most recent call first):
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 85 in _words_to_int
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 220 in _read
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 157 in value
File "/home/pi/Liberty Ridge WX Reduced/Wind_Direction.py", line 97 in get_value
File "/home/pi/Liberty Ridge WX Reduced/Liberty_Ridge_WX.py", line 199 in <module>
File "/usr/lib/python3.7/runpy.py", line 85 in _run_code
File "/usr/lib/python3.7/runpy.py", line 96 in _run_module_code
File "/usr/lib/python3.7/runpy.py", line 263 in run_path
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/../debugpy/server/cli.py", line 285 in run_file
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/../debugpy/server/cli.py", line 444 in main
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/__main__.py", line 45 in <module>
File "/usr/lib/python3.7/runpy.py", line 85 in _run_code
File "/usr/lib/python3.7/runpy.py", line 193 in _run_module_as_main
Traceback from faulthandler in terminal:
munmap_chunk(): invalid pointer
Fatal Python error: Aborted
Thread 0xb44ff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/usr/lib/python3/dist-packages/gpiozero/mixins.py", line 539 in held
File "/usr/lib/python3.7/threading.py", line 865 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb4eff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/usr/lib/python3/dist-packages/gpiozero/mixins.py", line 539 in held
File "/usr/lib/python3.7/threading.py", line 865 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Current thread 0xb6f53b40 (most recent call first):
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 85 in _words_to_int
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 220 in _read
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 157 in value
File "/home/pi/Liberty Ridge WX Reduced/Wind_Direction.py", line 97 in get_value
File "/home/pi/Liberty Ridge WX Reduced/Liberty_Ridge_WX.py", line 189 in <module>
Aborted
I'm running very similar code on my Raspberry Pi weather station, and also get the munmap_chunk() error after a couple of days, running either from a command line or from Thonny.
Like you, I get no information, apart from the munmap_chunk() line. That makes me think the error is not in Python, as that would surely provide trace information. So, maybe it's from the libraries Python uses, or from the part of the operating system that does memory management.
Here's what I believe to be the source code for where the error happens, and the comments are interesting...
static void
munmap_chunk (mchunkptr p)
{
INTERNAL_SIZE_T size = chunksize (p);
assert (chunk_is_mmapped (p));
/* Do nothing if the chunk is a faked mmapped chunk in the dumped
main arena. We never free this memory. */
if (DUMPED_MAIN_ARENA_CHUNK (p))
return;
uintptr_t block = (uintptr_t) p - prev_size (p);
size_t total_size = prev_size (p) + size;
/* Unfortunately we have to do the compilers job by hand here. Normally
we would test BLOCK and TOTAL-SIZE separately for compliance with the
page size. But gcc does not recognize the optimization possibility
(in the moment at least) so we combine the two values into one before
the bit test. */
if (__builtin_expect (((block | total_size) & (GLRO (dl_pagesize) - 1)) != 0, 0))
malloc_printerr ("munmap_chunk(): invalid pointer");
atomic_decrement (&mp_.n_mmaps);
atomic_add (&mp_.mmapped_mem, -total_size);
/* If munmap failed the process virtual memory address space is in a
bad shape. Just leave the block hanging around, the process will
terminate shortly anyway since not much can be done. */
__munmap ((char *) block, total_size);
}
In the end, I decided a better thing to do was restart the program once a day, but I didn't want to use crontab, rc.local or similar. There's actually a way for a Python program to restart itself! I found it here https://blog.petrzemek.net/2014/03/23/restarting-a-python-script-within-itself/
Here's my test program...
#!/usr/bin/env/python3
import sys
import os
import time
import datetime as dt
began = dt.datetime.now()
print("Began at ", str(began))
ending = began + dt.timedelta(minutes = 2)
print("Ending at ", str(ending))
while True:
time.sleep(10)
present = dt.datetime.now()
print(" It's ", str(present))
if present > ending:
print("Time's up!")
os.execv(sys.executable, ['python3'] + sys.argv)
quit()
I think amending your weather station code to restart itself daily, before the munmap_chunk() error happens is likely to be easier than finding out why it happens.
Update:
That's what I did, adding this code before the main loop...
ending = dt.datetime.now() + dt.timedelta(hours = 24)
...and this at the end of the main loop
present = dt.datetime.now()
if present > ending:
print("Time for my 24 hour restart!")
os.execv(sys.executable, ['python3'] + sys.argv)
quit()
The program restarts every day (approximately), and has so far run for five days without the munmap_chunk() error.
I'm reading TESS data and as you might expect it can be really large, My Buffer is too small. Is there a way I can avoid this error? Like maybe skip the file that has a large file? Or is there a more permanent solution(not involving more memory)? My Code is below, along with the Full Error
from lightkurve import TessTargetPixelFile
import lightkurve as lk
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
def func(tpf):
tpf.plot(aperture_mask=tpf.pipeline_mask);
lc = tpf.to_lightcurve()
mask = (lc.time.value < 1464)
masked_lc = lc[mask]
clipped_lc = masked_lc.remove_outliers(sigma=5);
flat_lc = clipped_lc.flatten()
binned_lc = flat_lc.bin(binsize=5)
periodogram = binned_lc.to_periodogram(method="bls", period=np.arange(1, 20, 0.001))
print(periodogram.plot())
planet_b_period = periodogram.period_at_max_power
planet_b_t0 = periodogram.transit_time_at_max_power
planet_b_dur = periodogram.duration_at_max_power
ax = binned_lc.fold(period=planet_b_period, epoch_time=planet_b_t0).scatter()
ax.set_xlim(-5, 5);
best_fit_period = periodogram.period_at_max_power
print('Best fit period: {:.3f}'.format(best_fit_period))
bfc=best_fit_period
print(bfc)
folded_lc = binned_lc.fold(period=bfc)
folded_lc.scatter(s=7);
plt.show()
planet_b_period
planet_b_model = periodogram.get_transit_model(period=planet_b_period,
transit_time=planet_b_t0,
duration=planet_b_dur)
ax = binned_lc.fold(planet_b_period, planet_b_t0).scatter(s=7)
planet_b_model.fold(planet_b_period, planet_b_t0).plot(ax=ax, c='r', lw=2)
ax.set_xlim(-5, 5);
dataset=pd.read_csv("all_targets_S001_v1.csv")
d=dataset["TICID"]
print(d)
IDs=[]
for i in range(len(d)):
IDs.append("TIC"+str(d[i]))
for i in range(len(IDs)):
tpf_file = lk.search_targetpixelfile(IDs[i], mission="TESS", sector=5).download(quality_bitmask='default')
try:
func(tpf_file)
continue
except:
continue
Thanks
The Full Error
WARNING: File may have been truncated: actual file length (262144) is smaller than the expected size (46402560) [astropy.io.fits.file]
Traceback (most recent call last):
File "lc.py", line 42, in <module>
tpf_file = lk.search_targetpixelfile(IDs[i], mission="TESS", sector=5).download(quality_bitmask='default')
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\utils.py", line 555, in wrapper
return f(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\search.py", line 355, in download
return self._download_one(
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\search.py", line 290, in _download_one
return read(path, quality_bitmask=quality_bitmask, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\io\read.py", line 112, in read
return getattr(__import__("lightkurve"), filetype)(path_or_url, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\targetpixelfile.py", line 2727, in __init__
quality_array=self.hdu[1].data["QUALITY"], bitmask=quality_bitmask
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\utils\decorators.py", line 758, in __get__
val = self.fget(obj)
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\io\fits\hdu\table.py", line 399, in data
data = self._get_tbdata()
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\io\fits\hdu\table.py", line 171, in _get_tbdata
raw_data = self._get_raw_data(self._nrows, columns.dtype,
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\io\fits\hdu\base.py", line 520, in _get_raw_data
return self._file.readarray(offset=offset, dtype=code, shape=shape)
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\io\fits\file.py", line 330, in readarray
return np.ndarray(shape=shape, dtype=dtype, offset=offset,
TypeError: buffer is too small for requested array
I am trying to use multiprocessing library to speed up CSV reading from files. I've done so using Pool and now I'm trying to do it with Process(). However when concatenating the list to create a dataframe, it's giving me the following error:
ValueError: No objects to concatenate
To me it looks like the processes are overwriting the uber_data list. What am I missing here?
import glob
import pandas as pd
from multiprocessing import Process
import matplotlib.pyplot as plt
import os
location = "/home/data/csv/"
uber_data = []
def read_csv(filename):
return uber_data.append(pd.read_csv(filename))
def data_wrangling(uber_data):
uber_data['Date/Time'] = pd.to_datetime(uber_data['Date/Time'], format="%m/%d/%Y %H:%M:%S")
uber_data['Dia Setmana'] = uber_data['Date/Time'].dt.weekday_name
uber_data['Num dia'] = uber_data['Date/Time'].dt.dayofweek
return uber_data
def plotting(uber_data):
weekdays = uber_data.pivot_table(index=['Num dia','Dia Setmana'], values='Base', aggfunc='count')
weekdays.plot(kind='bar', figsize=(8,6))
plt.ylabel('Total Journeys')
plt.title('Journey on Week Day')
def main():
processes = []
files = list(glob.glob(os.path.join(location,'*.csv*')))
for file in files:
print(file)
p = Process(target=read_csv, args=[file])
processes.append(p)
p.start()
for i, process in enumerate(processes):
process.join()
print(uber_data)
combined_df = pd.concat(uber_data, ignore_index=True)
dades_mod = data_wrangling(combined_df)
Plotting(dades_mod)
main()
Traceback is:
Process Process-223:
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "<timed exec>", line 17, in read_csv
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 255, in concat
sort=sort,
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 301, in __init__
objs = list(objs)
TypeError: 'NoneType' object is not iterable
Process Process-224:
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "<timed exec>", line 17, in read_csv
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 255, in concat
sort=sort,
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 301, in __init__
objs = list(objs)
TypeError: 'NoneType' object is not iterable
Process Process-221:
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "<timed exec>", line 17, in read_csv
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 255, in concat
sort=sort,
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 301, in __init__
objs = list(objs)
TypeError: 'NoneType' object is not iterable
Process Process-222:
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
Process Process-225:
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "<timed exec>", line 17, in read_csv
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 255, in concat
sort=sort,
File "<timed exec>", line 17, in read_csv
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 301, in __init__
objs = list(objs)
TypeError: 'NoneType' object is not iterable
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 255, in concat
sort=sort,
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 301, in __init__
objs = list(objs)
TypeError: 'NoneType' object is not iterable
Process Process-220:
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "<timed exec>", line 17, in read_csv
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 255, in concat
sort=sort,
File "/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py", line 301, in __init__
objs = list(objs)
TypeError: 'NoneType' object is not iterable
[]
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<timed eval> in <module>
<timed exec> in main()
/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py in concat(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, sort, copy)
253 verify_integrity=verify_integrity,
254 copy=copy,
--> 255 sort=sort,
256 )
257
/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py in __init__(self, objs, axis, join, join_axes, keys, levels, names, ignore_index, verify_integrity, copy, sort)
302
303 if len(objs) == 0:
--> 304 raise ValueError("No objects to concatenate")
305
306 if keys is None:
ValueError: No objects to concatenate
Thank you
The uber_data's in each process are not the same object as the uber_data in the main process. You can't really share data between processes.
from multiprocessing import Process
def read_csv(filename=r'c:\pyProjects\data.csv'):
print(id(uber_data))
uber_data.append(pd.read_csv(filename))
def main():
processes = []
for file in range(4):
p = Process(target=read_csv)
processes.append(p)
p.start()
for i, process in enumerate(processes):
process.join()
return processes
if __name__ == '__main__':
uber_data = []
print(id(uber_data))
ps = main()
Prints
PS C:\pyProjects> py -m tmp
2632505050432
1932359777344
2230288136512
2039196563648
2479121315968
You could use a Queue to send the data back to the main process.
from multiprocessing import Process, Queue
def read_csv(filename=r'c:\pyProjects\data.csv', q=None):
q.put(pd.read_csv(filename))
def main(q):
processes = []
for file in range(4):
p = Process(target=read_csv, kwargs={'q':q})
processes.append(p)
p.start()
for i, process in enumerate(processes):
process.join()
while not q.empty():
print('.')
uber_data.append(q.get(block=True))
return processes
if __name__ == '__main__':
uber_data = []
q = Queue()
ps = main(q)
for thing in uber_data:
print(thing.head().to_string())
print('**')
Or you could use threads.
from threading import Thread
def g(filename):
uber_data.append(pd.read_csv(filename))
if __name__ == '__main__':
uber_data = []
threads = []
for _ in range(4):
threads.append(Thread(target=g, args=(r'c:\pyProjects\data.csv',)))
for t in threads:
t.start()
while any(t.is_alive() for t in threads):
pass
for thing in uber_data:
print(thing.head().to_string())
print('**')
My problem is following:
I'm taking a data from files and want to make an animation of four plots at the same time: two colourbars and two lines.
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as anim
import sys
begin = float(sys.argv[1])
end = float(sys.argv[2])
dataCl = np.loadtxt("file1.txt")
dataSS = np.loadtxt("file2.txt")
datajSR = np.loadtxt("file3.txt")
ibegin = 0
iend = 0
for i in range(len(dataCl[:,0])):
if np.abs(dataCl[i,0] - begin) < 1e-9:
ibegin = i
iend = i
while abs(dataCl[i,0] - end) >= 1e-9:
iend = iend + 1
i = i + 1
break
fig = plt.figure()
f, axarr = plt.subplots(2, 2)
temp = np.zeros((10,10))
Qs = axarr[0,0].imshow(temp,cmap = plt.cm.OrRd)
El = axarr[0,1].imshow(temp,cmap = plt.cm.OrRd)
SS, = axarr[1,0].plot([],[])
jSR, = axarr[1,1].plot([],[])
def init():
Qs.set_array(temp)
El.set_array(temp)
SS.set_data([],[])
jSR.set_data([],[])
return Qs,El,SS,jSR,
def animate(i):
a = 0
b = 0
dataQ = np.zeros((10,10))
dataE = np.zeros((10,10))
for j in range(100):
if b >= 10:
a = a + 1
b = 0
dataQ[a][b] = dataCl[i,2*j + 1]
dataE[a][b] = dataCl[i,2*(j+1)]
b = b + 1
Qs.set_array(dataQ)
El.set_array(dataE)
SS.set_data(dataSS[ibegin:ibegin+i,0],dataSS[ibegin:ibegin+i,1])
jSR.set_data(datajSR[ibegin:ibegin+i,0],datajSR[ibegin:ibegin+i,1])
return Qs,El,SS,jSR,
ani = anim.FuncAnimation(fig, animate, init_func = init, frames = iend-ibegin,interval=25, blit=True)
plt.show()
After running it shows these messages:
Exception in Tkinter callback
Traceback (most recent call last):
File "/usr/lib/python2.7/lib-tk/Tkinter.py", line 1413, in __call__
return self.func(*args)
File "/usr/lib/pymodules/python2.7/matplotlib/backends/backend_tkagg.py", line 236, in resize
self.show()
File "/usr/lib/pymodules/python2.7/matplotlib/backends/backend_tkagg.py", line 239, in draw
FigureCanvasAgg.draw(self)
File "/usr/lib/pymodules/python2.7/matplotlib/backends/backend_agg.py", line 421, in draw
self.figure.draw(self.renderer)
File "/usr/lib/pymodules/python2.7/matplotlib/artist.py", line 55, in draw_wrapper
draw(artist, renderer, *args, **kwargs)
File "/usr/lib/pymodules/python2.7/matplotlib/figure.py", line 904, in draw
self.canvas.draw_event(renderer)
File "/usr/lib/pymodules/python2.7/matplotlib/backend_bases.py", line 1544, in draw_event
self.callbacks.process(s, event)
File "/usr/lib/pymodules/python2.7/matplotlib/cbook.py", line 262, in process
proxy(*args, **kwargs)
File "/usr/lib/pymodules/python2.7/matplotlib/cbook.py", line 192, in __call__
return mtd(*args, **kwargs)
File "/usr/lib/pymodules/python2.7/matplotlib/animation.py", line 273, in _end_redraw
self._post_draw(None, self._blit)
File "/usr/lib/pymodules/python2.7/matplotlib/animation.py", line 220, in _post_draw
self._blit_draw(self._drawn_artists, self._blit_cache)
File "/usr/lib/pymodules/python2.7/matplotlib/animation.py", line 235, in _blit_draw
a.axes.draw_artist(a)
File "/usr/lib/pymodules/python2.7/matplotlib/axes.py", line 2008, in draw_artist
assert self._cachedRenderer is not None
AssertionError
I cannot find a mistake in my code ;(
The error message might be backend and platform specific. As the error message seems to point to the blitting mechanism, you might want to try setting blit=False in FuncAnimation. Also, you might try some other backend to see if the problem persists. (Knowing your platform and matplotlib version might also help.)
Update: If setting blit=False, trying another backend, and updating matplotlib does not help, then a few suggestions:
Try to see manually if you code works with the initial data (init(); animate(0); fig.savefig("/tmp/test.png")) - if it throws an error, there is a static plotting problem to fix.
Now you initialize the plot twice (first in the code, then in init), you can take one away (e.g. do not define init_func)
Initializing the plots with [],[] leaves the scale uninitialized. You should probably use set_ylim, set_xlim with the plots and vmin, vmax keywords with the imshow images when you initialize them. (This could possibly have something to do with the exception you get!)