python memory error when loading MNIST.pkl.gz - python

I am new to Python and I have downloaded the code DBN.py but there is a problem:when I was trying to load the dataset MNIST.pkl.gz.there is always an meomory error..
my code is very simple:
import cPickle, gzip, numpy
# Load the dataset
f = gzip.open('C:\Users\MAC\Desktop\mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = cPickle.load(f)
f.close()
and the error is as follows:
Traceback (most recent call last):
File "<ipython-input-17-528eea6bbfdd>", line 1, in <module>
runfile('C:/Users/MAC/Documents/Python Scripts/untitled0.py', wdir='C:/Users/MAC/Documents/Python Scripts')
File "C:\Users\MAC\Anaconda\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 699, in runfile
execfile(filename, namespace)
File "C:\Users\MAC\Anaconda\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 74, in execfile
exec(compile(scripttext, filename, 'exec'), glob, loc)
File "C:/Users/MAC/Documents/Python Scripts/untitled0.py", line 19, in <module>
train_set, valid_set, test_set = cPickle.load(f)
File "C:\Users\MAC\Anaconda\lib\gzip.py", line 268, in read
self._read(readsize)
File "C:\Users\MAC\Anaconda\lib\gzip.py", line 320, in _read
self._add_read_data( uncompress )
File "C:\Users\MAC\Anaconda\lib\gzip.py", line 338, in _add_read_data
self.extrabuf = self.extrabuf[offset:] + data
MemoryError
I really have no idea,is it because the memory of my computer is too small? it is on windows 7,32 bits

I suspect the problem to be Spyder in this case.
As to why, I have no idea but either the process isn't allowed to allocate enugh memory outside of it's own script or it simply gets stuck in a loop some how.
Try running your code without Spyder by pasting your code into myscript.py for instance and open a terminal and navigate to the folder where you saved your script and run python myscript.py and see if that works or gives the same output.
This is based on a conversation in the comments above.

Related

Using Flopy to create vtk output

I'm trying to write the model grid from a Modflow model to a vtk output to open in Paraview.
I've used Groundwater Vistas and created the MODFLOW - USG (Unstructured Grids)
When I try to use the export function I get the following error:
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm Community Edition 2019.2.5\helpers\pydev\pydevd.py", line 1415, in _exec
pydev_imports.execfile(file, globals, locals) # execute the script
File "C:\Program Files\JetBrains\PyCharm Community Edition 2019.2.5\helpers\pydev_pydev_imps_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "C:/Work/Models/Work/Projects/Job 302/Export grid.py", line 16, in
vtkobj = fp.export.vtk.Vtk(mf)
File "C:\ProgramData\Anaconda3\lib\site-packages\flopy\export\vtk.py", line 206, in init
self.shape = (self.modelgrid.nlay, self.modelgrid.nrow,
AttributeError: 'Grid' object has no attribute 'nlay'
Below is the code I've used:
import os
import flopy as fp
from flopy.export import vtk as fv
model_path = r'C:\Work\Models\302'
os.chdir(model_path)
# MODFLOW name file:
namefile = 'westend_Alluv3_11yr_A.nam'
mf = fp.modflow.Modflow.load(namefile,verbose=False,check=False)
# create the vtk object
vtkobj = fp.export.vtk.Vtk(mf)
Thanks

Having issue with subprocess library in python during text extraction from image using OCR

I have been using python OCR code to extract text from the image but i am getting some error. I think error is with subprocess library, however it is built in library. So i couldn't figure out the error exactly. can anyone please help me in resolving this error. My code and error are as below.
OCR code
import os
import tempfile
import subprocess
def ocr(path):
temp = tempfile.NamedTemporaryFile(delete=False)
process = subprocess.Popen(['tesseract', path, temp.name], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
process.communicate()
with open(temp.name + '.txt', 'r') as handle:
contents = handle.read()
os.remove(temp.name + '.txt')
os.remove(temp.name)
return contents
str = ocr("C:\\Users\\hp\\Desktop\\MS Thesis\\opencv-text-detection\\opencv-text-detection\\images\\sign.jpg")
print(str)
by executing th above code I got following error
[7]: runfile('C:/Users/hp/Desktop/MS Thesis/python text detection.py', wdir='C:/Users/hp/Desktop/MS Thesis')
Traceback (most recent call last):
File "", line 1, in
runfile('C:/Users/hp/Desktop/MS Thesis/python text detection.py', wdir='C:/Users/hp/Desktop/MS Thesis')
File "C:\Users\hp\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 704, in runfile
execfile(filename, namespace)
File "C:\Users\hp\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 108, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/hp/Desktop/MS Thesis/python text detection.py", line 26, in
str = ocr("C:\Users\hp\Desktop\MS Thesis\opencv-text-detection\opencv-text-detection\images\sign.jpg")
File "C:/Users/hp/Desktop/MS Thesis/python text detection.py", line 15, in ocr
process = subprocess.Popen(['tesseract', path, temp.name], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
File "C:\Users\hp\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 171, in init
super(SubprocessPopen, self).init(*args, **kwargs)
File "C:\Users\hp\Anaconda3\lib\subprocess.py", line 769, in init
restore_signals, start_new_session)
File "C:\Users\hp\Anaconda3\lib\subprocess.py", line 1172, in _execute_child
startupinfo)
FileNotFoundError: [WinError 2] The system cannot find the file specified
I have been using Python 3.7.1 with anaconda on window 7
Look like there is an error because it cannot found the file
if the file is in the same directory I will recommend you use:
import os
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.path.join(BASE_DIR, 'file_dir')

File used by another process error: Python

I am trying to change the contents of a destination file with the help of some source file. When the work is done, I am making sure that I have closed all the files in code (ofcourse, I have made sure that it is not opened in any editor too.)
def write_to_file(self, _source_path, _destination_path):
f_source = open(_source_path, 'r')
f_destination = open(_destination_path, 'r')
f_temp = open(self.temp_path, 'w+')
while source_line or destination_line:...
f_temp.close()
f_destination.close()
f_source.close()
shutil.move(self.temp_path, _destination_path)
return
I am getting below error:
Traceback (most recent call last):
File "<input>", line 1, in <module>
File "C:\Program Files\JetBrains\PyCharm\plugins\python-ce\helpers\pydev\_pydev_bundle\pydev_umd.py", line 197, in runfile
pydev_imports.execfile(filename, global_vars, local_vars) # execute the script
File "C:\Program Files\JetBrains\PyCharm\plugins\python-ce\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "C:/Users/rohit.tayal/PycharmProjects/AutoNVMerge/MainActivity.py", line 151, in <module>
script.write_to_file(source_path, destination_path)
File "C:/Users/rohit.tayal/PycharmProjects/AutoNVMerge/MainActivity.py", line 62, in write_to_file
shutil.move(self.temp_path, _destination_path)
File "C:\Anaconda\envs\AutoNVMerge\lib\shutil.py", line 581, in move
os.unlink(src)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'C:\\auto_nv_merge.xml'
C:\auto_nv_merge.xml is my temp file.
When working with
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'C:\\auto_nv_merge.xml'
you have 2 options, be a hero and try to hunt down the process that is holding that file hostage or just restart your computer.
I usually go with the latter, restart

Error using uic to convert .ui file to .py file in Python

I am trying to write a program in python that will convert a .ui file in the same folder (created in Qt Designer) into a .py file. This is the code for this extremely basic program:
# -*- coding: utf-8 -*-
from PyQt4 import uic
with open('exampleinterface.py', 'w') as fout:
uic.compileUi('exampleinterface.ui', fout)
It gives the following error (with long path names shortened):
Traceback (most recent call last):
File "", line 1, in
File "...\Python32_3.5\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 699, in runfile
execfile(filename, namespace)
File "...\Python32_3.5\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 88, in execfile
exec(compile(open(filename, 'rb').read(), filename, 'exec'), namespace)
File ".../Documents/Python/UiToPy/minimalconvert.py", line 11, in
uic.compileUi('exampleinterface.ui', fout)
File "...\Python32_3.5\lib\site-packages\PyQt4\uic__init__.py", line 173, in compileUi
winfo = compiler.UICompiler().compileUi(uifile, pyfile, from_imports, resource_suffix)
File "...\Python32_3.5\lib\site-packages\PyQt4\uic\Compiler\compiler.py", line 140, in compileUi
w = self.parse(input_stream, resource_suffix)
File "...\Python32_3.5\lib\site-packages\PyQt4\uic\uiparser.py", line 974, in parse
document = parse(filename)
File "...\Python32_3.5\lib\xml\etree\ElementTree.py", line 1182, in parse
tree.parse(source, parser)
File "...\Python32_3.5\lib\xml\etree\ElementTree.py", line 594, in parse
self._root = parser._parse_whole(source)
xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 1, column 1
Can anyone tell me why this isn't working, and if there is a solution?
Note: I know that there are other ways to convert a .ui file into a .py file, but I am looking for one that I can easily integrate into a python program, without calling an outside file.
This error popped because my .ui file was not saved with the recent changes that i had done. The file name showed the asterisk(*) mark in the file name. Once i saved the file with changes it could be converted into a .py file.
Thanks to ekhumoro and mwormser. The problem was indeed the .ui file.
I retried it with a new .ui file and everything worked fine.

python - extract csv files from 7z

I have lots of csv files contained in different 7z files. I want to find specific csv files in those 7z files and save them decompressed in a different directory.
I have tried
import os
import py7zlib
tree = r'Where_the_7zfiles_are_stored'
dst = r'Where_I_want_to_store_the_csvfiles'
for dirpath, dirname, filename in os.walk(tree):
for myfile in filename:
if myfile.endswith('2008-01-01_2008-04-30_1.7z'):
myZip = py7zlib.Archive7z(open(os.path.join(dirpath,myfile), 'rb'))
csvInZipFile = zip(myZip.filenames,myZip.files)
for myCsvFileName, myCsvFile in csvInZipFile:
if '2008-01' in myCsvFileName:
with open(os.path.join(dst,myCsvFileName),'wb') as outfile:
outfile.write(myCsvFile.read())
but I get the following error
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Users\'\Anaconda3\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 682, in runfile
execfile(filename, namespace)
File "C:\Users\'\Anaconda3\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 85, in execfile
exec(compile(open(filename, 'rb').read(), filename, 'exec'), namespace)
File "C:/Users//'/Documents/Example/unzipfiles.py", line 23, in <module>
outfile.write(myCsvFile.read())
File "C:\Users\'\Anaconda3\lib\site-packages\py7zlib.py", line 576, in read
data = getattr(self, decoder)(coder, data)
File "C:\Users\'\Anaconda3\lib\site-packages\py7zlib.py", line 634, in _read_lzma
return self._read_from_decompressor(coder, dec, input, checkremaining=True, with_cache=True)
File "C:\Users\'\Anaconda3\lib\site-packages\py7zlib.py", line 611, in _read_from_decompressor
tmp = decompressor.decompress(data)
ValueError: data error during decompression
The odd thing is that the method seems to work fine for the first two csv files. I have no idea how to get to the root of the problem. At least the data in the csv files do not seem to be different. Manually unpacking the different csv files using IZArc goes without problem. (The problem occurred in both python 2.7 and 3.4).
I have also tried to use the lzma module, but here I could not figure out how to retrieve the different csv files contained in the 7z file.

Categories

Resources