I fear that this is a messy way to approach the problem but...
let's say that I want to make some imports in Python based on some conditions.
For this reason I want to write a function:
def conditional_import_modules(test):
if test == 'foo':
import onemodule, anothermodule
elif test == 'bar':
import thirdmodule, and_another_module
else:
import all_the_other_modules
Now how can I have the imported modules globally available?
For example:
conditional_import_modules(test='bar')
thirdmodule.myfunction()
Imported modules are just variables - names bound to some values. So all you need is to import them and make them global with global keyword.
Example:
>>> math
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
NameError: name 'math' is not defined
>>> def f():
... global math
... import math
...
>>> f()
>>> math
<module 'math' from '/usr/local/lib/python2.6/lib-dynload/math.so'>
You can make the imports global within a function like this:
def my_imports(module_name):
globals()[module_name] = __import__(module_name)
I've just had the similar problem, here is my solution:
class GlobalImport:
def __enter__(self):
return self
def __call__(self):
import inspect
self.collector = inspect.getargvalues(inspect.getouterframes(inspect.currentframe())[1].frame).locals
def __exit__(self, *args):
globals().update(self.collector)
then, anywhere in the code:
with GlobalImport() as gi:
import os, signal, atexit, threading, _thread
# whatever you want it won't remain local
# if only
gi()
# is called before the end of this block
# there you go: use os, signal, ... from whatever place of the module
You can use the built-in function __import__ to conditionally import a module with global scope.
To import a top level module (think: import foo):
def cond_import():
global foo
foo = __import__('foo', globals(), locals())
Import from a hierarchy (think: import foo.bar):
def cond_import():
global foo
foo = __import__('foo.bar', globals(), locals())
Import from a hierarchy and alias (think: import foo.bar as bar):
def cond_import():
global bar
foo = __import__('foo.bar', globals(), locals())
bar = foo.bar
I like #badzil approach.
def global_imports(modulename,shortname = None, asfunction = False):
if shortname is None:
shortname = modulename
if asfunction is False:
globals()[shortname] = __import__(modulename)
else:
globals()[shortname] = eval(modulename + "." + shortname)
So something that is traditionally in a class module:
import numpy as np
import rpy2
import rpy2.robjects as robjects
import rpy2.robjects.packages as rpackages
from rpy2.robjects.packages import importr
Can be transformed into a global scope:
global_imports("numpy","np")
global_imports("rpy2")
global_imports("rpy2.robjects","robjects")
global_imports("rpy2.robjects.packages","rpackages")
global_imports("rpy2.robjects.packages","importr",True)
May have some bugs, which I will verify and update. The last example could also have an alias which would be another "shortname" or a hack like "importr|aliasimportr"
I like #rafał grabie approach. As it even support importing all.
i.e.
from os import *
(Despite it being bad practice XD )
Not allowed to comment, but here is a python 2.7 version.
Also removed the need to call the function at the end.
class GlobalImport:
def __enter__(self):
return self
def __exit__(self, *args):
import inspect
collector = inspect.getargvalues(inspect.getouterframes(inspect.currentframe())[1][0]).locals
globals().update(collector)
def test():
with GlobalImport() as gi:
## will fire a warning as its bad practice for python.
from os import *
test()
print path.exists(__file__)
I like the answer from #maxschlepzig.
There is a bug in the approach that if you directly import a function it will not work.
For example,
global_imports("tqdm", "tqdm, True)
does not work, because the module is not imported. And this
global_imports("tqdm")
global_imports("tqdm", "tqdm, True)
works.
I change #maxschlepzig's answer a bit. Using fromlist so you can load function or module with "From" statement in a uniform way.
def global_imports(object_name: str,
short_name: str = None,
context_module_name: str = None):
"""import from local function as global import
Use this statement to import inside a function,
but effective as import at the top of the module.
Args:
object_name: the object name want to import,
could be module or function
short_name: the short name for the import
context_module_name: the context module name in the import
example usage:
import os -> global_imports("os")
import numpy as np -> global_imports("numpy", "np")
from collections import Counter ->
global_imports("Counter", None, "collections")
from google.cloud import storage ->
global_imports("storage", None, "google.cloud")
"""
if not short_name:
short_name = object_name
if not context_module_name:
globals()[short_name] = __import__(object_name)
else:
context_module = __import__(context_module_name,
fromlist=[object_name])
globals()[short_name] = getattr(context_module, object_name)
You could have this function return the names of the modules you want to import, and then use
mod == __import__(module_name)
Step-1: config.py, config_v2.py, rnd.py in same directory/folder
Step-2: config.py
HIGH_ATTENDANCE_COUNT_MIN = 0
Step-3: config_v2.py
HIGH_ATTENDANCE_COUNT_MIN = 5
Step-4: rnd.py
def versioning_test(v):
global config
if v == 'v1':
config = __import__('config', globals(), locals())
if v == 'v2':
config = __import__('config_v2', globals(), locals())
def version_test_in_another_function():
print('version_test_in_another_function: HIGH_ATTENDANCE_COUNT_MIN: ', config.HIGH_ATTENDANCE_COUNT_MIN)
versioning_test("v2")
version_test_in_another_function()
Step-5: $ python3 rnd.py
<<output>>: version_test_in_another_function: HIGH_ATTENDANCE_COUNT_MIN: 5
It is now recommended (for Python 3), to use the importlib
https://docs.python.org/3/reference/import.html#importlib
eg: globals()["np"] = importlib.import_module("numpy")
and you can now execute "np.array([1,2,3])" afterwards.
There are also other ways of importing that you might prefer. Consider seeing the aforementioned documentation.
Related
I am trying to use Python unittest library to test the following code.
from cv2 import imwrite
import h5py
import lumpy as np
class Myclass:
def __init__(self):
self.data = []
def get_data(self):
return self.data
def load_data(self, path):
count = 10
for i in range(count):
mat_file = path + f'{i}.mat'
with h5py.File(mat_file, 'r') as fin:
data = np.array(fin['data'])
for j in range(len(data)):
filename = path + f'{i}_{j}.jpg'
imwrite(filename, data)
self.data = data
I tried to do the following. But I am getting AttributeError saying Myclass has no attribute 'imwrite'. And I do not know how to mock the nested loops with image writing imwrite.
from mock import MagicMock, patch
def test():
m = MagicMock()
m.__enter__.return_value = data
with patch("h5py.File", return_value=m):
with patch(Myclass.imwrite) as mock_imwrite:
testclass = Myclass()
testclass.load_data(path='test')
assert testclass.get_data() == data
I hope someone can help me out. Any help is very much appreciated
The issue here is that your class Myclass does not have a function imwrite. Only the module (aka the file the class is defined in) is aware of this function.
If you want to patch the imwrite function of the package cv2, you have to write:
(untested code)
with patch('cv2.imwrite') as mock_imwrite:
testclass = Myclass()
...
But the as mock_imwrite part is only needed, if you run an assert on the mock. Otherwise, you may just skip it.
Python Code:
class Importer:
from importlib import __import__, reload
from sys import modules
libname = ""
import_count = 0
module = None
def __init__(self, name):
self.libname = name
self.import_count = 0
def importm(self):
if self.libname not in self.modules:
self.module = __import__(self.libname)
else:
print("must reload")
self.module = self.reload(self.module)
self.import_count += 1
# test out Importer
importer = Importer("module")
importer.importm() # prints Hello
importer.importm() # prints Hello
importer.importm() # prints Hello (again)
print(importer.import_count)
The above Python (3.8.1) code is at OnlineGDB, which if you run, will give an error:
TypeError: reload() takes 1 positional argument but 2 were given
When I open up the importlib library in Python, I see this:
# ... (previous code; unnecessary)
_RELOADING = {}
def reload(module): ## this is where reload is defined (one argument)
"""Reload the module and return it.
The module must have been successfully imported before.
"""
if not module or not isinstance(module, types.ModuleType): ## check for type of module
raise TypeError("reload() argument must be a module")
try:
name = module.__spec__.name
except AttributeError:
name = module.__name__
if sys.modules.get(name) is not module: ## other code you (probably) don't have to care about
msg = "module {} not in sys.modules"
raise ImportError(msg.format(name), name=name)
if name in _RELOADING:
return _RELOADING[name]
_RELOADING[name] = module
try:
parent_name = name.rpartition('.')[0]
if parent_name:
try:
parent = sys.modules[parent_name]
except KeyError:
msg = "parent {!r} not in sys.modules"
raise ImportError(msg.format(parent_name),
name=parent_name) from None
else:
pkgpath = parent.__path__
else:
pkgpath = None
target = module
spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target)
if spec is None:
raise ModuleNotFoundError(f"spec not found for the module {name!r}", name=name)
_bootstrap._exec(spec, module)
# The module may have replaced itself in sys.modules!
return sys.modules[name]
finally:
try:
del _RELOADING[name]
except KeyError:
pass
# ... (After code; unnecessary)
All double hashtag (##) comments are mine
It is clearly visible that reload DOES have 1 argument, and it checks if that argument is a module. In the OGDB (OnineGDB) code, I am only passing one argument (pretty sure) and it is of type module (most likely). If I remove that argument (you can edit the OGDB), it gives:
TypeError: reload() argument must be module
So for some reason, Python keeps thinking I have one more argument than I do actually have. The only way I made it work was editing the importlib file to have reload have two arguments (not a good idea).
I tried running PDB, not helpful.
Can anyone spot anything obviously wrong, like actually having two arguments?
What I needed to do is put the imports outside the class for it to work. Here is the new OGDB. Credits to #L3viathan. Code below:
from importlib import __import__, reload
from sys import modules
class Importer:
libname = ""
import_count = 0
module = None
def __init__(self, name):
self.libname = name
self.import_count = 0
def importm(self):
if self.libname not in modules:
self.module = __import__(self.libname)
else:
print("must reload")
self.module = reload(self.module)
self.import_count += 1
# test out Importer
importer = Importer("module")
importer.importm() # prints Hello
importer.importm() # prints Hello
importer.importm() # prints Hello (again)
print(importer.import_count)
You're having an issue because you're calling self.reload(self.module), which is actually equivalent to calling reload(self, self.module). To see this, try running the following:
class Example:
def some_method(*args):
print(args)
def some_other_method(self):
self.some_method(1)
an_example = Example()
example.some_other_method()
You should see that this prints out 2 arguments, not 1, (the first of which is a reference to self) despite us only passing one argument to some_method, and some_method having no self argument.
It would be better to import the reload method within your importm method (or outside the class altogether!), like so:
def importm(self):
from importlib import __import__, reload
if self.libname not in self.modules:
self.module = __import__(self.libname)
else:
print("must reload")
self.module = reload(self.module)
self.import_count += 1
import mymodule
reload(mymodule)
is how it would work ... Im not sure what your question is from that big wall of text above this is typically used to reset state to its initial state
mymodule.py
x = 5
main.py
from importlib import reload # in py2 you did not need to import it
import mymodule
print(mymodule.x) # 5
mymodule.x = 8
print(mymodule.x) # 8
reload(mymodule)
print(mymodule.x) # 5 again
I want to write a simple tutorial in my Python module foo.
If the module is imported as foo I would like the tutorial to call it foo:
>>> import foo
>>> foo.tutorial()
Please run foo.baz().
However, when the module is imported as bar - I want the tutorial to call it bar:
>>> import foo as bar
>>> bar.tutorial()
Please run bar.baz().
I know that neither __name__ nor __package__ variables would change their value on import ... as ....
To be different from Is it possible to detect way of import of a module in python? - it will be enough for me to know the first alias the module is imported with.
It's not pretty, foolproof, or production worthy, but:
import inspect
import sys
thismod = sys.modules[__name__]
def imported_as():
imported_frame = inspect.currentframe().f_back.f_back
for k, v in imported_frame.f_locals.items():
if v is thismod:
return k
for k, v in imported_frame.f_globals.items():
if v is thismod:
return k
def tutorial():
print('please call {}.baz()'.format(imported_as()))
A tiny explanation:
jump up two stack frames
look at the locals and globals to find the module object that is identical to the module
Why you should really never do this:
it isn't foolproof:
from imports will break this
multiple imports of the same module will nondeterministically give only the first found name
inspect.currentframe is super magical and should generally be avoided in production code
Here is a similar solution to Anthony's. I believe it better handles multiple imports. from imports are still an issue.
import inspect
import re
import readline
def tutorial():
frame = inspect.currentframe().f_back
if "__file__" in frame.f_locals:
f = inspect.getframeinfo(frame)
m = re.search(r"(([^\.]*).*)\.tutorial", f.code_context[0].strip())
if m:
parent,path = m.group(2),m.group(1)
if inspect.ismodule(frame.f_locals[parent]):
print "Please run {}.baz()".format(path)
return
print "Verify import and run foo.baz()"
else:
loc = frame.f_locals
glo = frame.f_globals
local_foo = [(k, v.__name__)for k,v in loc.items() if inspect.ismodule(v) and v.__name__=="foo"]
global_foo = [(k, v.__name__)for k,v in glo.items() if inspect.ismodule(v) and v.__name__=="foo"]
if local_foo and set(local_foo)==set(global_foo):
print "Please run {}.baz()".format(global_foo[-1][0])
elif local_foo and global_foo:
print "Please run {}.baz() in local scope or {}.baz()".format(local_foo[-1][0], global_foo[-1][0])
elif local_foo:
print "Please run {}.baz() in local scope".format(local_foo[-1][0])
elif global_foo:
print "Please run {}.baz()".format(global_foo[-1][0])
else:
n = readline.get_current_history_length()
h = [str(readline.get_history_item(i)) for i in range(n)] + [readline.get_line_buffer()]
h.reverse()
for i in h:
matches = re.findall(r"([A-Za-z0-9_-]*(\.[A-Za-z0-9_-]*)*)\.tutorial", i)
for match in matches:
if _baz_check(glo, match[0]):
print "Please run {}.baz()".format(match[0])
return
print "Verify import and run foo.baz()"
def _baz_check(d, path):
path = path.split(".")
if path[0] not in d:
return False
cur = d[path[0]]
for i in range(1,len(path)):
if not hasattr(cur, path[i]):
return False
cur = getattr(cur, path[i])
return hasattr(cur, "__name__") and cur.__name__ == "foo" and hasattr(cur, "baz")
def baz():
print "Baz"
import foo
foo.tutorial()
import foo as bar
bar.tutorial()
def f():
import foo as local_foo
local_foo.tutorial()
f()
from foo import tutorial
tutorial()
import mod
mod.foo.tutorial()
from mod import foo as dum
dum.tutorial()
Please run foo.baz()
Please run bar.baz()
Please run local_foo.baz()
Verify import and run foo.baz()
Please run mod.foo.baz()
Please run dum.baz()
I'd like my students to be able to check their code as they write it in a Jupyter Notebook by calling a function from an imported module which runs a unittest. This works fine unless the function needs to be checked against objects which are to be picked up in the global scope of the Notebook.
Here's my check_test module:
import unittest
from IPython.display import Markdown, display
def printmd(string):
display(Markdown(string))
class Tests(unittest.TestCase):
def check_add_2(self, add_2):
val = 5
self.assertAlmostEqual(add_2(val), 7)
def check_add_n(self, add_n):
n = 6
val = 5
self.assertAlmostEqual(add_n(val), 11)
check = Tests()
def run_check(check_name, func, hint=False):
try:
getattr(check, check_name)(func)
except check.failureException as e:
printmd('**<span style="color: red;">FAILED</span>**')
if hint:
print('Hint:', e)
return
printmd('**<span style="color: green;">PASSED</span>**')
If the Notebook is:
In [1]: def add_2(val):
return val + 2
In [2]: def add_n(val):
return val + n
In [3]: import test_checks
In [4]: test_checks.run_check('check_add_2', add_2)
PASSED
In [5]: test_checks.run_check('check_add_n', add_n)
!!! ERROR !!!
The error here is not suprising: add_n doesn't know about the n I defined in check_add_n.
So I got to thinking I could do something like:
In [6]: def add_n(val, default_n=None):
if default_n:
n = default_n
return val + n
in the Notebook, and then passing n in the test:
def check_add_n(self, add_n):
val = 5
self.assertAlmostEqual(add_n(val, 6), 11)
But this is causing me UnboundLocalError headaches down the line because of the assignment of n, even within an if clause: this is apparently stopping the Notebook from picking up n in global scope when it's needed.
For the avoidance of doubt, I don't want insist that n is passed as an argument to add_n: there could be many such objects used but not changed by the function being tested and I want them resolved in the outer scope.
Any ideas how to go about this?
You can import __main__ to access the notebook scope:
import unittest
from IPython.display import Markdown, display
import __main__
def printmd(string):
display(Markdown(string))
class Tests(unittest.TestCase):
def check_add_2(self, add_2):
val = 5
self.assertAlmostEqual(add_2(val), 7)
def check_add_n(self, add_n):
__main__.n = 6
val = 5
self.assertAlmostEqual(add_n(val), 11)
check = Tests()
def run_check(check_name, func, hint=False):
try:
getattr(check, check_name)(func)
except check.failureException as e:
printmd('**<span style="color: red;">FAILED</span>**')
if hint:
print('Hint:', e)
return
printmd('**<span style="color: green;">PASSED</span>**')
This gives me a PASSED output.
This works because when you execute a python file that file is stored in sys.modules as the __main__ module. This is precisely why the if __name__ == '__main__': idiom is used. It is possible to import such module and since it is already in the module cache it will not re-execute it or anything.
I wrote a method called buildRegex that, given a name (of type str), returns a regex object that finds a from ... import ... name statement in a Python module.
For example, here is the expected behaviour of buildRegex:
>>> regObj = buildRegex('foo')
>>> regObj.search('from a import fool') is None
True
>>> regObj.search('from a import foo') is not None
True
>>> regObj.search('from a.b.c import foo as food') is None
True
>>> regObj.search('from a.b.c import fool, bar as foo') is not None
True
What I have so far works for all the examples above (and more):
def buildRegex(name):
singleImportedName = r'(\b{0}\b(?!\s+as\s+))'.format(name)
importStatement = r'from\s+(\w+(\.\w+)*)\s+import\s+([^#\n]*)(?={0})'.format(singleImportedName )
return re.compile(importStatement)
buildRegex assumes that the searched module has no SyntaxErrors which is OK.
My problem is, when looking for the imported name foo, I also need to know if it is an alias to a different name. I.e. if a module has the statement:
from a.b.c import bar as foo
I want to know what foo is aliasing, in this case, that would be bar. Currently, due to asserted lookaheads in the regex, that is not possible. So, finally my question:
How can I refactor the regex so that this information is not lost, i.e., if the given name is an alias, then the the name its aliasing is in one of the regex's groups?
I'd recommend that instead of writing complicated regular expressions to parse imports, one would actually use the ast.parse to parse the source code into abstract syntax tree and find the names from there, as ast.parse is guaranteed to parse Python correctly. Something like:
import ast
class ImportFinder(ast.NodeVisitor):
def __init__(self):
self.imports = []
def visit_Import(self, node):
names = []
for i in node.names:
names.append((i.name, i.asname))
self.imports.append(['import', names])
def visit_ImportFrom(self, node):
module = node.module
level = node.level # how many dots
names = []
for i in node.names:
names.append((i.name, i.asname))
self.imports.append(('from', level, module, names))
def parse_imports(source):
tree = ast.parse(source)
finder = ImportFinder()
finder.visit(tree)
return finder.imports
Example usage:
import pprint
pprint.pprint(parse_imports('''
from foo import bar, baz, frob
from .. import bar as spam, baz as ham, frob
import bar.baz
import bar.foo as baf
'''))
Prints out:
[('from', 0, 'foo', [('bar', None), ('baz', None), ('frob', None)]),
('from', 2, None, [('bar', 'spam'), ('baz', 'ham'), ('frob', None)]),
['import', [('bar.baz', None)]],
['import', [('bar.foo', 'baf')]]]
The integer on the from lines gives the number of . before the module name.
import inspect
import importlib
import ast
class Imports(ast.NodeVisitor):
def visit_Import(self, node):
print("In Import")
for imp in node.names:
if imp.asname is not None:
print("module name = {}, alias = {}".format(imp.name, imp.asname))
else:
print("module name = {}".format(imp.name))
print()
def visit_ImportFrom(self, node):
print("In ImportFrom")
for imp in node.names:
if imp.asname is not None:
print("module = {}\nname = {}\nalias = {}\nlevel = {}\n".
format(node.module, imp.name, imp.asname, node.level))
else:
print("module = {}\nname = {}\nlevel = {}\n".
format(node.module, imp.name, node.level))
print()
mod = "temp_test"
mod = importlib.import_module(mod)
p = ast.parse(inspect.getsource(mod))
Imports().visit(p)
Input:
from bisect import bisect_left as bs
import datetime
import time
import numpy as np
def foo():
from re import findall
class Foo():
def test(self):
from re import compile as cp, finditer as ft
Output:
In ImportFrom
module = bisect
name = bisect_left
alias = bs
level = 0
In Import
module name = datetime
In Import
module name = time
In Import
module name = numpy, alias = np
In ImportFrom
module = re
name = findall
level = 0
In ImportFrom
module = re
name = compile
alias = cp
level = 0
module = re
name = finditer
alias = ft
level = 0
class Import(names)
An import statement. names is a list of alias nodes.
class ImportFrom(module, names, level)
Represents from x import y. module is a raw string of the ‘from’ name, without any leading dots, or None for statements such as from . import foo. level is an integer holding the level of the relative import (0 means absolute import).
The greentreesnakes documentation for me at least has a much better explanation of what all the nodes do and how to use the ast module than the actual ast documentation itself.
You can use also pass the module directly or open the py file and pass the content to ast.parse:
with open("temp_test.py") as f:
p = ast.parse(f.read(), filename="<ast>", mode="exec")
Imports().visit(p)
And passing the module:
import temp_test
p = ast.parse(inspect.getsource(temp_test))
Imports().visit(p)