I'm reading TESS data and as you might expect it can be really large, My Buffer is too small. Is there a way I can avoid this error? Like maybe skip the file that has a large file? Or is there a more permanent solution(not involving more memory)? My Code is below, along with the Full Error
from lightkurve import TessTargetPixelFile
import lightkurve as lk
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
def func(tpf):
tpf.plot(aperture_mask=tpf.pipeline_mask);
lc = tpf.to_lightcurve()
mask = (lc.time.value < 1464)
masked_lc = lc[mask]
clipped_lc = masked_lc.remove_outliers(sigma=5);
flat_lc = clipped_lc.flatten()
binned_lc = flat_lc.bin(binsize=5)
periodogram = binned_lc.to_periodogram(method="bls", period=np.arange(1, 20, 0.001))
print(periodogram.plot())
planet_b_period = periodogram.period_at_max_power
planet_b_t0 = periodogram.transit_time_at_max_power
planet_b_dur = periodogram.duration_at_max_power
ax = binned_lc.fold(period=planet_b_period, epoch_time=planet_b_t0).scatter()
ax.set_xlim(-5, 5);
best_fit_period = periodogram.period_at_max_power
print('Best fit period: {:.3f}'.format(best_fit_period))
bfc=best_fit_period
print(bfc)
folded_lc = binned_lc.fold(period=bfc)
folded_lc.scatter(s=7);
plt.show()
planet_b_period
planet_b_model = periodogram.get_transit_model(period=planet_b_period,
transit_time=planet_b_t0,
duration=planet_b_dur)
ax = binned_lc.fold(planet_b_period, planet_b_t0).scatter(s=7)
planet_b_model.fold(planet_b_period, planet_b_t0).plot(ax=ax, c='r', lw=2)
ax.set_xlim(-5, 5);
dataset=pd.read_csv("all_targets_S001_v1.csv")
d=dataset["TICID"]
print(d)
IDs=[]
for i in range(len(d)):
IDs.append("TIC"+str(d[i]))
for i in range(len(IDs)):
tpf_file = lk.search_targetpixelfile(IDs[i], mission="TESS", sector=5).download(quality_bitmask='default')
try:
func(tpf_file)
continue
except:
continue
Thanks
The Full Error
WARNING: File may have been truncated: actual file length (262144) is smaller than the expected size (46402560) [astropy.io.fits.file]
Traceback (most recent call last):
File "lc.py", line 42, in <module>
tpf_file = lk.search_targetpixelfile(IDs[i], mission="TESS", sector=5).download(quality_bitmask='default')
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\utils.py", line 555, in wrapper
return f(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\search.py", line 355, in download
return self._download_one(
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\search.py", line 290, in _download_one
return read(path, quality_bitmask=quality_bitmask, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\io\read.py", line 112, in read
return getattr(__import__("lightkurve"), filetype)(path_or_url, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\lightkurve\targetpixelfile.py", line 2727, in __init__
quality_array=self.hdu[1].data["QUALITY"], bitmask=quality_bitmask
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\utils\decorators.py", line 758, in __get__
val = self.fget(obj)
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\io\fits\hdu\table.py", line 399, in data
data = self._get_tbdata()
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\io\fits\hdu\table.py", line 171, in _get_tbdata
raw_data = self._get_raw_data(self._nrows, columns.dtype,
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\io\fits\hdu\base.py", line 520, in _get_raw_data
return self._file.readarray(offset=offset, dtype=code, shape=shape)
File "C:\ProgramData\Anaconda3\lib\site-packages\astropy\io\fits\file.py", line 330, in readarray
return np.ndarray(shape=shape, dtype=dtype, offset=offset,
TypeError: buffer is too small for requested array
Related
I faced with the problem during telegram bot writing. I would be very happy if somebody help me with this.
My code
import telebot
import matplotlib.pyplot as plt
import numpy as np
...
def plot_func(message):
x = np.linspace(-5,5,100)
y = message.text # <-- here is something wrong I supppose
plt.plot(x, y, 'r')
plt.savefig('plot_name.png', dpi = 300)
bot.send_photo(message.chat.id, photo=open('plot_name.png', 'rb'))
#plt.show()
Main idea
User send function, bot plot it and send image back:
ERROR
/home/anmnv/Desktop/news_scrapper_bot/bot.py:148: UserWarning: Starting a Matplotlib GUI outside of the main thread will likely fail.
plt.plot(x, y, 'r')
Traceback (most recent call last):
File "/home/anmnv/Desktop/news_scrapper_bot/bot.py", line 424, in <module>
bot.polling(none_stop=True)
File "/home/anmnv/.local/lib/python3.10/site-packages/telebot/__init__.py", line 1047, in polling
self.__threaded_polling(non_stop=non_stop, interval=interval, timeout=timeout, long_polling_timeout=long_polling_timeout,
File "/home/anmnv/.local/lib/python3.10/site-packages/telebot/__init__.py", line 1122, in __threaded_polling
raise e
File "/home/anmnv/.local/lib/python3.10/site-packages/telebot/__init__.py", line 1078, in __threaded_polling
self.worker_pool.raise_exceptions()
File "/home/anmnv/.local/lib/python3.10/site-packages/telebot/util.py", line 154, in raise_exceptions
raise self.exception_info
File "/home/anmnv/.local/lib/python3.10/site-packages/telebot/util.py", line 98, in run
task(*args, **kwargs)
File "/home/anmnv/Desktop/news_scrapper_bot/bot.py", line 148, in plot_func
plt.plot(x, y, 'r')
File "/home/anmnv/.local/lib/python3.10/site-packages/matplotlib/pyplot.py", line 2730, in plot
return gca().plot(
File "/home/anmnv/.local/lib/python3.10/site-packages/matplotlib/axes/_axes.py", line 1662, in plot
lines = [*self._get_lines(*args, data=data, **kwargs)]
File "/home/anmnv/.local/lib/python3.10/site-packages/matplotlib/axes/_base.py", line 311, in __call__
yield from self._plot_args(
File "/home/anmnv/.local/lib/python3.10/site-packages/matplotlib/axes/_base.py", line 504, in _plot_args
raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (100,) and (1,)
Thank you in advance
Assuming message.text contains the string 'x**2', you can use numexpr.evaluate to convert to numpy array:
import numexpr
import matplotlib.pyplot as plt
x = np.linspace(-5, 5, 100)
y = numexpr.evaluate(message.text) # message.text = 'x**2'
plt.plot(x, y, 'r')
Output:
I use the script
import matplotlib.pyplot as plt
import numpy as np
import yoda
def readScatter2D(histname, filename):
histos = yoda.core.read(filename+".yoda")
x = []
y = []
h = histos[histname]
for b in h:
x.append(b.x())
y.append(b.y())
plt.plot([x],[y], 'bo')
plt.savefig(str(histname)+".pdf")
readScatter2D("totalCh", "properties_file")
Compiling this with python3 gives me the error:
Traceback (most recent call last):
File "enhancement.py", line 18, in <module>
readScatter2D("totalCh", "properties_file")
File "enhancement.py", line 16, in readScatter2D
plt.savefig(str(histname)+".pdf")
File "/usr/lib64/python2.7/site-packages/matplotlib/pyplot.py", line 697, in savefig
res = fig.savefig(*args, **kwargs)
File "/usr/lib64/python2.7/site-packages/matplotlib/figure.py", line 1573, in savefig
self.canvas.print_figure(*args, **kwargs)
File "/usr/lib64/python2.7/site-packages/matplotlib/backend_bases.py", line 2252, in print_figure
**kwargs)
File "/usr/lib64/python2.7/site-packages/matplotlib/backends/backend_pdf.py", line 2519, in print_pdf
file = PdfFile(filename)
File "/usr/lib64/python2.7/site-packages/matplotlib/backends/backend_pdf.py", line 422, in __init__
fh = open(filename, 'wb')
IOError: [Errno 2] No such file or directory: 'totalCh.pdf'
I do not understand why matplotlib cannot name the file while saving, by following the histogram names. What is the solution to save the figure following the original histogram? I am using matplotlib version 2.0.2
I'm trying to plot time series data using matplotlib using the following code
import matplotlib.pyplot as plt
x, y = data.shape
y_metrics = alarms_metrics + udp_alarms_metrics_names
print x, y
for j in range(1, y):
time_stamp = []
metric = []
plt.figure()
for i in range(x):
time_stamp.append(data_time[i][0])
metric.append(data[i][j])
plt.xlabel("Time")
plt.ylabel(str(y_metrics[j-1]))
plt.plot(time_stamp, metric)
plt.savefig(str(j))
The format of time_stamp is
['2017:02:01:14:44:00', '2017:02:01:14:44:01',...etc]
time_stamp is a list of type 'numpy.string_'
I'm getting the following error
Traceback (most recent call last):
File "/home/joseph/PycharmProjects/demo/src/alarm_log/alarm_db_plot.py", line 23, in <module>
plt.plot(time_stamp, metric)
File "/usr/lib/python2.7/dist-packages/matplotlib/pyplot.py", line 3154, in plot
ret = ax.plot(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/matplotlib/__init__.py", line 1814, in inner
return func(ax, *args, **kwargs)
File "/usr/lib/python2.7/dist-packages/matplotlib/axes/_axes.py", line 1425, in plot
self.add_line(line)
File "/usr/lib/python2.7/dist-packages/matplotlib/axes/_base.py", line 1708, in add_line
self._update_line_limits(line)
File "/usr/lib/python2.7/dist-packages/matplotlib/axes/_base.py", line 1730, in _update_line_limits
path = line.get_path()
File "/usr/lib/python2.7/dist-packages/matplotlib/lines.py", line 925, in get_path
self.recache()
File "/usr/lib/python2.7/dist-packages/matplotlib/lines.py", line 612, in recache
x = np.asarray(xconv, np.float_)
File "/usr/local/lib/python2.7/dist-packages/numpy/core/numeric.py", line 482, in asarray
return array(a, dtype, copy=False, order=order)
ValueError: invalid literal for float(): 2017:02:01:14:44:00
I think I might need to used matplotlib dateformatter to parse the time_stamp. I tried so but I couldn't figure out how. Any help please?
Not exactly sure what you're trying to plot without seeing the data, but you could try to convert the string to a datetime object. See here for a list with the format codes.
from datetime import datetime
time_stamp = ['2017:02:01:14:44:00', '2017:02:01:14:44:01']
time_stamp = [datetime.strptime(i, '%Y:%m:%d:%H:%M:%S') for i in time_stamp]
print(time_stamp)
My problem is following:
I'm taking a data from files and want to make an animation of four plots at the same time: two colourbars and two lines.
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as anim
import sys
begin = float(sys.argv[1])
end = float(sys.argv[2])
dataCl = np.loadtxt("file1.txt")
dataSS = np.loadtxt("file2.txt")
datajSR = np.loadtxt("file3.txt")
ibegin = 0
iend = 0
for i in range(len(dataCl[:,0])):
if np.abs(dataCl[i,0] - begin) < 1e-9:
ibegin = i
iend = i
while abs(dataCl[i,0] - end) >= 1e-9:
iend = iend + 1
i = i + 1
break
fig = plt.figure()
f, axarr = plt.subplots(2, 2)
temp = np.zeros((10,10))
Qs = axarr[0,0].imshow(temp,cmap = plt.cm.OrRd)
El = axarr[0,1].imshow(temp,cmap = plt.cm.OrRd)
SS, = axarr[1,0].plot([],[])
jSR, = axarr[1,1].plot([],[])
def init():
Qs.set_array(temp)
El.set_array(temp)
SS.set_data([],[])
jSR.set_data([],[])
return Qs,El,SS,jSR,
def animate(i):
a = 0
b = 0
dataQ = np.zeros((10,10))
dataE = np.zeros((10,10))
for j in range(100):
if b >= 10:
a = a + 1
b = 0
dataQ[a][b] = dataCl[i,2*j + 1]
dataE[a][b] = dataCl[i,2*(j+1)]
b = b + 1
Qs.set_array(dataQ)
El.set_array(dataE)
SS.set_data(dataSS[ibegin:ibegin+i,0],dataSS[ibegin:ibegin+i,1])
jSR.set_data(datajSR[ibegin:ibegin+i,0],datajSR[ibegin:ibegin+i,1])
return Qs,El,SS,jSR,
ani = anim.FuncAnimation(fig, animate, init_func = init, frames = iend-ibegin,interval=25, blit=True)
plt.show()
After running it shows these messages:
Exception in Tkinter callback
Traceback (most recent call last):
File "/usr/lib/python2.7/lib-tk/Tkinter.py", line 1413, in __call__
return self.func(*args)
File "/usr/lib/pymodules/python2.7/matplotlib/backends/backend_tkagg.py", line 236, in resize
self.show()
File "/usr/lib/pymodules/python2.7/matplotlib/backends/backend_tkagg.py", line 239, in draw
FigureCanvasAgg.draw(self)
File "/usr/lib/pymodules/python2.7/matplotlib/backends/backend_agg.py", line 421, in draw
self.figure.draw(self.renderer)
File "/usr/lib/pymodules/python2.7/matplotlib/artist.py", line 55, in draw_wrapper
draw(artist, renderer, *args, **kwargs)
File "/usr/lib/pymodules/python2.7/matplotlib/figure.py", line 904, in draw
self.canvas.draw_event(renderer)
File "/usr/lib/pymodules/python2.7/matplotlib/backend_bases.py", line 1544, in draw_event
self.callbacks.process(s, event)
File "/usr/lib/pymodules/python2.7/matplotlib/cbook.py", line 262, in process
proxy(*args, **kwargs)
File "/usr/lib/pymodules/python2.7/matplotlib/cbook.py", line 192, in __call__
return mtd(*args, **kwargs)
File "/usr/lib/pymodules/python2.7/matplotlib/animation.py", line 273, in _end_redraw
self._post_draw(None, self._blit)
File "/usr/lib/pymodules/python2.7/matplotlib/animation.py", line 220, in _post_draw
self._blit_draw(self._drawn_artists, self._blit_cache)
File "/usr/lib/pymodules/python2.7/matplotlib/animation.py", line 235, in _blit_draw
a.axes.draw_artist(a)
File "/usr/lib/pymodules/python2.7/matplotlib/axes.py", line 2008, in draw_artist
assert self._cachedRenderer is not None
AssertionError
I cannot find a mistake in my code ;(
The error message might be backend and platform specific. As the error message seems to point to the blitting mechanism, you might want to try setting blit=False in FuncAnimation. Also, you might try some other backend to see if the problem persists. (Knowing your platform and matplotlib version might also help.)
Update: If setting blit=False, trying another backend, and updating matplotlib does not help, then a few suggestions:
Try to see manually if you code works with the initial data (init(); animate(0); fig.savefig("/tmp/test.png")) - if it throws an error, there is a static plotting problem to fix.
Now you initialize the plot twice (first in the code, then in init), you can take one away (e.g. do not define init_func)
Initializing the plots with [],[] leaves the scale uninitialized. You should probably use set_ylim, set_xlim with the plots and vmin, vmax keywords with the imshow images when you initialize them. (This could possibly have something to do with the exception you get!)
I wrote a Python script (below) which load data from a text file (using pandas) and checks the values in the columns.
import sys
import pandas as pd
import numpy as np
from numpy import ndarray
import math
import matplotlib.pyplot as plt
from matplotlib.pyplot import *
from skimage import data
from skimage.feature import match_template
if __name__ == '__main__':
data = pd.read_csv('Fe_PSI_spt_refined.txt', sep=" ", header = None)
data.columns = ["Angle_number", "Omega", "Intensity", "X", "Y", "Address", "ID"]#, "flag"]
Number_of_projections = 181
Number_of_lines_in_txt = 3493
numrows = len(data)
counter_array = []
correlation_threshold_value = 0.7
a = np.zeros(Number_of_lines_in_txt)
output_file = ("output.txt")
for i in range(2, (Number_of_projections + 1)):
filename_cutouts_combined = ("cutouts_combined_%03i.txt" % (i))
filename_cutouts_combined_tag = ("cutouts_combined_tag_%03i.txt" % (i))
image = np.loadtxt(filename_cutouts_combined)
image_tagged = np.loadtxt(filename_cutouts_combined_tag)
for j in range(0, Number_of_lines_in_txt - 1):
print data.Angle_number[j], i
After one iteration of j I get the error below. Do you spot any error I should fix? Thanks
`Traceback (most recent call last):
File "Hyperbola_search.py", line 46, in <module>
print data.Angle_number[j], i
File "/Users/Alberto/anaconda/lib/python2.7/site-packages/pandas/core/series.py", line 491, in __getitem__
result = self.index.get_value(self, key)
File "/Users/Alberto/anaconda/lib/python2.7/site-packages/pandas/core/index.py", line 1032, in get_value
return self._engine.get_value(s, k)
File "index.pyx", line 97, in pandas.index.IndexEngine.get_value (pandas/index.c:2661)
File "index.pyx", line 105, in pandas.index.IndexEngine.get_value (pandas/index.c:2476)
File "index.pyx", line 149, in pandas.index.IndexEngine.get_loc (pandas/index.c:3215)
File "hashtable.pyx", line 382, in pandas.hashtable.Int64HashTable.get_item (pandas/hashtable.c:6450)
File "hashtable.pyx", line 388, in pandas.hashtable.Int64HashTable.get_item (pandas/hashtable.c:6394)
KeyError: 3491`
You load files into image and image_tagged, while a remains unused.
I don't know what data.Angle_number and numrows are, but they appear to be from libraries, not related to your files.