Related
I'm just beginning to start with machine learning and want to predict values/sales in a timeseries. I found this two blog posts, which basically match what I'm looking for.
Basics of Time Series Prediction - Setup of the Timeseries and Datasets found in here
Techniques for Time Series Prediction - NN Setup in here
Instead of predicting the value for the next timestep I would like to predict the value 4 timesteps ahead. Originally I have weekly data, so I want to predict the value 4 weeks / 1 month ahead.
As I understand this, I therefor need to change the "label" the model is trained with, which will be done within the function windowed_dataset() (Source 2).
def windowed_dataset(series, window_size, batch_size, shuffle_buffer):
dataset = tf.data.Dataset.from_tensor_slices(series)
dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)
dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))
dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1])) # <-- change will be in here
dataset = dataset.batch(batch_size).prefetch(1)
return dataset
If I change dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1])) to dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-4], window[-1])) the labels in my opinion are correctly adjusted to my goal.
But running the next step
dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)
print(dataset)
l0 = tf.keras.layers.Dense(1, input_shape=[window_size])
model = tf.keras.models.Sequential([l0])
model.compile(loss="mse", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-6, momentum=0.9))
model.fit(dataset,epochs=100,verbose=0)
throws an error:
runcell('Build model', 'C:/Users/USER/Desktop/Local/Prediction/untitled0.py')
<PrefetchDataset element_spec=(TensorSpec(shape=(None, None), dtype=tf.float32, name=None), TensorSpec(shape=(None,), dtype=tf.float32, name=None))>
Traceback (most recent call last):
File "C:\Users\USER\Desktop\Local\Prediction\untitled0.py", line 102, in <module>
model.fit(dataset,epochs=100,verbose=0)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\Users\USER\Anaconda3\lib\site-packages\tensorflow\python\eager\execute.py", line 54, in quick_execute
tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
InvalidArgumentError: Graph execution error:
Detected at node 'sequential_9/dense_10/BiasAdd' defined at (most recent call last):
File "C:\Users\USER\Anaconda3\lib\runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\USER\Anaconda3\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "C:\Users\USER\Anaconda3\lib\site-packages\spyder_kernels\console\__main__.py", line 23, in <module>
start.main()
File "C:\Users\USER\Anaconda3\lib\site-packages\spyder_kernels\console\start.py", line 328, in main
kernel.start()
File "C:\Users\USER\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 677, in start
self.io_loop.start()
File "C:\Users\USER\Anaconda3\lib\site-packages\tornado\platform\asyncio.py", line 199, in start
self.asyncio_loop.run_forever()
File "C:\Users\USER\Anaconda3\lib\asyncio\base_events.py", line 596, in run_forever
self._run_once()
File "C:\Users\USER\Anaconda3\lib\asyncio\base_events.py", line 1890, in _run_once
handle._run()
File "C:\Users\USER\Anaconda3\lib\asyncio\events.py", line 80, in _run
self._context.run(self._callback, *self._args)
File "C:\Users\USER\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 457, in dispatch_queue
await self.process_one()
File "C:\Users\USER\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 446, in process_one
await dispatch(*args)
File "C:\Users\USER\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 353, in dispatch_shell
await result
File "C:\Users\USER\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 648, in execute_request
reply_content = await reply_content
File "C:\Users\USER\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 353, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Users\USER\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2901, in run_cell
result = self._run_cell(
File "C:\Users\USER\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2947, in _run_cell
return runner(coro)
File "C:\Users\USER\Anaconda3\lib\site-packages\IPython\core\async_helpers.py", line 68, in _pseudo_sync_runner
coro.send(None)
File "C:\Users\USER\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3172, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "C:\Users\USER\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3364, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "C:\Users\USER\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3444, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "C:\Users\USER\AppData\Local\Temp/ipykernel_15784/4252985979.py", line 1, in <module>
runcell('Build model', 'C:/Users/USER/Desktop/Local/Prediction/untitled0.py')
File "C:\Users\USER\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 673, in runcell
exec_code(cell_code, filename, ns_globals, ns_locals,
File "C:\Users\USER\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 465, in exec_code
exec(compiled, ns_globals, ns_locals)
File "C:\Users\USER\Desktop\Local\Prediction\untitled0.py", line 102, in <module>
model.fit(dataset,epochs=100,verbose=0)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\training.py", line 1409, in fit
tmp_logs = self.train_function(iterator)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\training.py", line 1051, in train_function
return step_function(self, iterator)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\training.py", line 1040, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\training.py", line 1030, in run_step
outputs = model.train_step(data)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\training.py", line 889, in train_step
y_pred = self(x, training=True)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\training.py", line 490, in __call__
return super().__call__(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\base_layer.py", line 1014, in __call__
outputs = call_fn(inputs, *args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 92, in error_handler
return fn(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\sequential.py", line 374, in call
return super(Sequential, self).call(inputs, training=training, mask=mask)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\functional.py", line 458, in call
return self._run_internal_graph(
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\functional.py", line 596, in _run_internal_graph
outputs = node.layer(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\engine\base_layer.py", line 1014, in __call__
outputs = call_fn(inputs, *args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 92, in error_handler
return fn(*args, **kwargs)
File "C:\Users\USER\Anaconda3\lib\site-packages\keras\layers\core\dense.py", line 232, in call
outputs = tf.nn.bias_add(outputs, self.bias)
Node: 'sequential_9/dense_10/BiasAdd'
Matrix size-incompatible: In[0]: [24,9], In[1]: [12,1]
[[{{node sequential_9/dense_10/BiasAdd}}]] [Op:__inference_train_function_231055]
What am I missing here? Is there another, better approach to model the timeseries?
Note: Somewhere in the future I also would like to add more parameters/indicators to the model to test if this increases the accuracy.
Edit:
Creation of Data and Series:
#%% Setup
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow import keras
#%% Creating Timeseries
def plot_series(time, series, format="-", start=0, end=None):
plt.plot(time[start:end], series[start:end], format)
plt.xlabel("Time")
plt.ylabel("Value")
plt.grid(True)
def trend(time, slope=0):
return slope * time
def seasonal_pattern(season_time):
"""Just an arbitrary pattern, you can change it if you wish"""
return np.where(season_time < 0.4,
np.cos(season_time * 2 * np.pi),
1 / np.exp(3 * season_time))
def seasonality(time, period, amplitude=1, phase=0):
"""Repeats the same pattern at each period"""
season_time = ((time + phase) % period) / period
return amplitude * seasonal_pattern(season_time)
def noise(time, noise_level=1, seed=None):
rnd = np.random.RandomState(seed)
return rnd.randn(len(time)) * noise_level
time = np.arange(4 * 365 + 1, dtype="float32")
baseline = 10
series = trend(time, 0.1)
baseline = 10
amplitude = 40
slope = 0.05
noise_level = 5
# Create the series
series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)
# Update with noise
series += noise(time, noise_level, seed=42)
plt.figure(figsize=(10, 6))
plot_series(time, series)
plt.show()
#%% Create Data Sets
split_time = 1000
time_train = time[:split_time]
x_train = series[:split_time]
time_valid = time[split_time:]
x_valid = series[split_time:]
plt.figure(figsize=(10, 6))
plot_series(time_train, x_train)
plt.show()
plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plt.show()
Parameters:
#%% Set Parameters
window_size = 4
batch_size = 4
shuffle_buffer_size = 10
Changing dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1])) to dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-4], window[-1])) will actually influence the window, as one can find out looking at the outputs of these following steps:
# Previous steps within function
dataset = tf.data.Dataset.from_tensor_slices(series)
dataset = dataset.window(12 + 1, shift=1, drop_remainder=True)
dataset = dataset.flat_map(lambda window: window.batch(12 + 1))
# Regular approach predicting the next value
dataset_1 = dataset.shuffle(3).map(lambda window: (window[:-1], window[-1]))
counter = 0
for n in dataset_1:
counter +=1
if counter == 1:
print(n)
Output:
> (<tf.Tensor: shape=(12,), dtype=float32, numpy=
array([52.48357 , 49.35275 , 53.314735, 57.711823, 48.934444, 48.931244,
57.982895, 53.897125, 47.67393 , 52.68371 , 47.591717, 47.506374],
dtype=float32)>, <tf.Tensor: shape=(), dtype=float32, numpy=50.959415>)
# Approach predicting a value 4 timesteps ahead
dataset_4 = dataset.shuffle(3).map(lambda window: (window[:-4], window[-1]))
counter = 0
for n in dataset_4:
counter +=1
if counter == 1:
print(n)
Output:
> (<tf.Tensor: shape=(9,), dtype=float32, numpy=
array([52.48357 , 49.35275 , 53.314735, 57.711823, 48.934444, 48.931244,
57.982895, 53.897125, 47.67393 ], dtype=float32)>, <tf.Tensor: shape=(), dtype=float32, numpy=50.959415>)
In the first (regular) case the tensor has shape (12,) while in the second case the tensor has shape (9,).
It is therefor necessary to adjust the window_size in the following steps to reflect the actual shape and its influences e.g. on the length of the result array.
window_size_adjusted = window_size - 3
dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)
#%% Build model
l0 = tf.keras.layers.Dense(1, input_shape=[window_size_adjusted])
model = tf.keras.models.Sequential([l0])
model.compile(loss="mse", optimizer=tf.keras.optimizers.SGD(learning_rate=1e-6, momentum=0.9))
model.fit(dataset,epochs=100,verbose=0)
#%% Forecast
forecast = []
for time in range(len(series) - window_size_adjusted):
forecast.append(model.predict(series[time:time + window_size_adjusted][np.newaxis]))
forecast = forecast[split_time-window_size_adjusted:]
results = np.array(forecast)[:, 0, 0]
I would like to know how I can plot on a map the data from the variable "r" (relative humidity) of a NetCDF file with cartopy?
enter code here
from netCDF4 import Dataset
import cartopy.crs as ccrs
import numpy as np
import xarray as xr
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
ds = Dataset('relative_humidityEne_Dic2003_2020CMAS.nc', 'r')
lon = ds.variables["longitude"]
lat = ds.variables["latitude"]
level = ds.variables["level"]
time = ds.variables["time"]
r = ds.variables["r"]
ax = plt.axes(projection=ccrs.PlateCarree())
r_ds = ds.variables['r'][0, :, :, :]
plt.contourf(lon, lat, r_ds, transform=ccrs.PlateCarree())
ax.coastlines()
plt.show()
root group (NETCDF3_64BIT_OFFSET data model, file format NETCDF3):
Conventions: CF-1.6
history: 2021-02-19 10:03:12 GMT by grib_to_netcdf-2.16.0: /opt/ecmwf/eccodes/bin/grib_to_netcdf -S param -o /cache/data7/adaptor.marsdev.internal-1613728991.1847723-6796-17-440f19fd-bb8f-478f-bc36-959e3e9d9c42.nc /cache/tmp/440f19fd-bb8f-478f-bc36-959e3e9d9c42-adaptor.marsdev.internal-1613728966.210842-6796-20-tmp.grib
dimensions(sizes): longitude(480), latitude(241), level(8), time(53)
variables(dimensions): float32 longitude(longitude), float32 latitude(latitude), int32 level(level), int32 time(time), int16 r(time,level,latitude,longitude)
groups:
dict_keys(['longitude', 'latitude', 'level', 'time', 'r'])
las dimenciones variable r es: ('time', 'level', 'latitude', 'longitude')
(53, 8, 241, 480)
----------------------------------------------------------------------
<class 'netCDF4._netCDF4.Variable'>
int16 r(time, level, latitude, longitude)
scale_factor: 0.001964639476266317
add_offset: 64.43876426873042
_FillValue: -32767
missing_value: -32767
units: %
long_name: Relative humidity
standard_name: relative_humidity
unlimited dimensions:
current shape = (53, 8, 241, 480)
filling on
las dimenciones variable r es: ('time', 'level', 'latitude', 'longitude')
(53, 8, 241, 480)
#########################################################
menssage error
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py", line 3331, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 1, in
runfile('/home/leo/Documentos/Universidad/Trabajo_de_investigación/PerfilesVerticalesContaminantesAtmosfera/Datos/readNetdcf42003_2020CMAS.py', wdir='/home/leo/Documentos/Universidad/Trabajo_de_investigación/PerfilesVerticalesContaminantesAtmosfera/Datos')
File "/snap/pycharm-professional/230/plugins/python/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile
pydev_imports.execfile(filename, global_vars, local_vars) # execute the script
File "/snap/pycharm-professional/230/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/home/leo/Documentos/Universidad/Trabajo_de_investigación/PerfilesVerticalesContaminantesAtmosfera/Datos/readNetdcf42003_2020CMAS.py", line 65, in
plt.contourf(lon, lat, r_ds, transform=ccrs.PlateCarree())
File "/home/leo/.local/lib/python3.8/site-packages/matplotlib/pyplot.py", line 2577, in contourf
__ret = gca().contourf(
File "/home/leo/.local/lib/python3.8/site-packages/cartopy/mpl/geoaxes.py", line 321, in wrapper
return func(self, *args, **kwargs)
File "/home/leo/.local/lib/python3.8/site-packages/cartopy/mpl/geoaxes.py", line 1586, in contourf
result = matplotlib.axes.Axes.contourf(self, *args, **kwargs)
File "/home/leo/.local/lib/python3.8/site-packages/matplotlib/init.py", line 1447, in inner
return func(ax, *map(sanitize_sequence, args), **kwargs)
File "/home/leo/.local/lib/python3.8/site-packages/matplotlib/axes/_axes.py", line 6335, in contourf
contours = mcontour.QuadContourSet(self, *args, **kwargs)
File "/home/leo/.local/lib/python3.8/site-packages/matplotlib/contour.py", line 816, in init
kwargs = self._process_args(*args, **kwargs)
File "/home/leo/.local/lib/python3.8/site-packages/matplotlib/contour.py", line 1430, in _process_args
x, y, z = self._contour_args(args, kwargs)
File "/home/leo/.local/lib/python3.8/site-packages/matplotlib/contour.py", line 1488, in _contour_args
x, y, z = self._check_xyz(args[:3], kwargs)
File "/home/leo/.local/lib/python3.8/site-packages/matplotlib/contour.py", line 1514, in _check_xyz
x = np.asarray(x, dtype=np.float64)
File "/usr/local/lib/python3.8/dist-packages/numpy/core/_asarray.py", line 102, in asarray
return array(a, dtype, copy=False, order=order)
TypeError: array() takes no arguments (1 given)
You should be able to do this using my ncplot package (https://pypi.org/project/ncplot/):
from ncplot import view
view('relative_humidityEne_Dic2003_2020CMAS.nc', 'r')
This will create an interactive plot.
I am making an insurance fee predictor(gives single output,which is a float) on tensorflow taking data like age,sex etc(6 values total).
this is the code
import tensorflow as tf
import numpy as np
from tensorflow import keras
import pandas as pd
a=0
file=pd.read_csv(r"""C:\Users\lavni\OneDrive\Desktop\proj.csv""",sep=',',index_col=False)
Data = file[['age', 'sex', 'bmi', 'children', 'smoker', 'region']].to_numpy()
Charges=file[['charges']].to_numpy()
model = keras.Sequential()
model.add(keras.layers.Dense(units=6, input_shape=(6,)))
model.compile(optimizer='sgd',loss='mean_squared_error')
model.fit(Data,Charges)
When i run the code,I get the following error:
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\ML.py", line 14, in <module>
model.fit(Data,Charges)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\training.py", line 108, in _method_wrapper
return method(self, *args, **kwargs)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1063, in fit
steps_per_execution=self._steps_per_execution)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1117, in __init__
model=model)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 265, in __init__
x, y, sample_weights = _process_tensorlike((x, y, sample_weights))
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1021, in _process_tensorlike
inputs = nest.map_structure(_convert_numpy_and_scipy, inputs)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\util\nest.py", line 635, in map_structure
structure[0], [func(*x) for x in entries],
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\util\nest.py", line 635, in <listcomp>
structure[0], [func(*x) for x in entries],
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1016, in _convert_numpy_and_scipy
return ops.convert_to_tensor(x, dtype=dtype)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\framework\ops.py", line 1499, in convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\framework\tensor_conversion_registry.py", line 52, in _default_conversion_function
return constant_op.constant(value, dtype, name=name)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\framework\constant_op.py", line 264, in constant
allow_broadcast=True)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\framework\constant_op.py", line 275, in _constant_impl
return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\framework\constant_op.py", line 300, in _constant_eager_impl
t = convert_to_eager_tensor(value, ctx, dtype)
File "C:\Users\lavni\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\framework\constant_op.py", line 98, in convert_to_eager_tensor
return ops.EagerTensor(value, ctx.device_name, dtype)
ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type int).
Any help is appreciated and thank you in advance.
Charges = np.array([file.loc[:,'charges']])
This probably solve your problem. First, filter dataframe with loc method. After, convert it to numpy array.
I'm trying to read in data using the Tensorflow Dataset API. I have loaded filenames and label filenames into arrays which I load into a dataset. I then try to map these filenames to the actual image files, but get an error that seems to state that the input to the mapping function recieves placeholders rather than actual tensors.
class DatasetReader:
def __init__(self, records_list, batch_size=1):
self.batch_size = batch_size
self.records = {}
self.records["image"] = tf.convert_to_tensor([record['image'] for record in records_list])
self.records["filename"] = tf.convert_to_tensor([record['filename'] for record in records_list])
self.records["annotation"] = tf.convert_to_tensor([record['annotation'] for record in records_list])
self.dataset = Dataset.from_tensor_slices(self.records)
self.dataset = self.dataset.map(self._input_parser)
self.dataset = self.dataset.batch(batch_size)
self.dataset = self.dataset.repeat()
def _input_parser(self, record):
filename = record['filename']
image_name = record['image']
annotation_file = record['annotation']
image = tf.image.decode_image(tf.read_file(filename))
annotation = tf.image.decode_image(tf.read_file(annotation_file))
return self._augment_image(image, annotation)
The error I'm getting is in the line image = tf.image.decode_image(tf.read_file(filename)). The stack trace is below.
File "FCN.py", line 269, in <module>
tf.app.run()
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 48, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "FCN.py", line 179, in main
train_records, valid_records, image_options_train, image_options_val, FLAGS.batch_size, FLAGS.batch_size)
File "/home/ubuntu/FCN.tensorflow/TFReader.py", line 89, in from_records
train_reader = DatasetReader(train_records, train_image_options, train_batch_size)
File "/home/ubuntu/FCN.tensorflow/TFReader.py", line 34, in __init__
self.dataset = self.dataset.map(self._input_parser)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/ops/dataset_ops.py", line 964, in map
return MapDataset(self, map_func, num_threads, output_buffer_size)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/ops/dataset_ops.py", line 1735, in __init__
self._map_func.add_to_graph(ops.get_default_graph())
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/function.py", line 449, in add_to_graph
self._create_definition_if_needed()
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/framework/function.py", line 168, in _create_definition_if_needed
outputs = self._func(*inputs)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/ops/dataset_ops.py", line 1723, in tf_map_func
ret = map_func(nested_args)
File "/home/ubuntu/FCN.tensorflow/TFReader.py", line 42, in _input_parser
image = tf.image.decode_image(tf.read_file(filename))
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_io_ops.py", line 223, in read_file
result = _op_def_lib.apply_op("ReadFile", filename=filename, name=name)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op
op_def=op_def)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/framework/function.py", line 80, in create_op
data_types, **kwargs)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/function.py", line 665, in create_op
**kwargs)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2632, in create_op
set_shapes_for_outputs(ret)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1911, in set_shapes_for_outputs
shapes = shape_func(op)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1861, in call_with_requiring
return call_cpp_shape_fn(op, require_shape_fn=True)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.py", line 595, in call_cpp_shape_fn
require_shape_fn)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.py", line 659, in _call_cpp_shape_fn_impl
raise ValueError(err.message)
ValueError: Shape must be rank 0 but is rank 1 for 'ReadFile' (op: 'ReadFile') with input shapes: [?].
You cannot pass in a rank-1 tensor to tf.read_file. Here are some examples:
import tensorflow as tf
# Correct: input can be a string.
tf.image.decode_image(tf.read_file("filename"))
# Correct: input can be a rank-0 tensor.
tf.image.decode_image(tf.read_file(tf.convert_to_tensor("filename")))
# Wrong: input cannot be a list.
tf.image.decode_image(tf.read_file(["filename"]))
# Wrong: input cannot be a rank-1 tensor
tf.image.decode_image(tf.read_file(tf.convert_to_tensor(["filename"])))
In your code, it seems like self.records["filename"] is a rank-1 tensor; you might mistakenly passed it as a parameter to tf.read_file in _input_parser
when I running the Tensorflow's official sample code: 'tensorflow-mnist-tutorial':
$python3 mnist_1.0_softmax.py
and something wrong:
Traceback (most recent call last):
File "/Users/holmes/Desktop/untitled/gitRepository/tensorflow-mnist-tutorial/mnist_1.0_softmax.py", line 78, in <module>
I = tensorflowvisu.tf_format_mnist_images(X, Y, Y_) # assembles 10x10 images by default
File "/Users/holmes/Desktop/untitled/gitRepository/tensorflow-mnist-tutorial/tensorflowvisu.py", line 42, in tf_format_mnist_images
everything_incorrect_first = tf.concat(0, [incorrectly_recognised_indices, correctly_recognised_indices]) # images reordered with indeces of unrecognised images first
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py", line 1047, in concat
dtype=dtypes.int32).get_shape(
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 651, in convert_to_tensor
as_ref=False)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 716, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/framework/constant_op.py", line 176, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/framework/constant_op.py", line 165, in constant
tensor_util.make_tensor_proto(value, dtype=dtype, shape=shape, verify_shape=verify_shape))
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/framework/tensor_util.py", line 367, in make_tensor_proto
_AssertCompatible(values, dtype)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/framework/tensor_util.py", line 302, in _AssertCompatible
(dtype.name, repr(mismatch), type(mismatch).__name__))
TypeError: Expected int32, got list containing Tensors of type '_Message' instead.