I have a package that sort of looks like this:
- package
-- module1.py
-- module2.py
-- __init__.py
In init.py I am programmatically scanning the package and importing the modules that are inside.
import importlib
import pkgutil
registry = {}
def creatable(cls):
if cls.handles_type() in registry:
raise KeyError("Duplicate string representations found for string: " + cls.handles_type())
registry[cls.handles_type()] = cls
return cls
def import_submodules(package, recursive=False):
""" Import all submodules of a module, recursively, including subpackages
"""
if isinstance(package, str):
package = importlib.import_module(package)
results = {}
for loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
full_name = package.__name__ + '.' + name
results[full_name] = importlib.import_module(full_name)
if recursive and is_pkg:
results.update(import_submodules(full_name))
return results
import_submodules(__name__)
Inside the modules, there are classes annotated with the #creatable decorator that I define here. The idea is to have a registry dictionary with the key being the class name and the value being the class (so I can create instances using the string representation).
The issue is, the registry is empty when using Pyinstaller.
UPDATE:
The issue is that package.__path__ does not contain anything. It can be fixed by adding the .py files on the package path i.e.
datas=[
('package/*.py', 'package'),
]
But this doesn't look like a good solution - i.e. I'd be sending code to the end user.
This isn't an answer but is the only way I can show some code - this is my hook file for package mypkg, in the hooks folder, called hook-mypkg.py
import os
imports = []
for root, dirs, files in os.walk(os.path.join(os.getcwd(),"mypkg" )):
print(root)
for file in files:
if file.endswith( ".py") and not file.endswith( "__init__.py"):
print( " ",file)
imports.append("mypkg."+file[:-3])
print( "........." )
print( "hi=",imports )
hiddenimports = imports
This definitely works to include the .py files it finds in the bundle - they appear in globals()['__loader__'].toc in init.py as e.g. "mypkg.file1" for module mypkg\file1.py when sys.frozen is True.
Note after editing the hook you have to delete the dist and build folders then re-run pyinstaller
Related
I have a directory of code filled in like this:
root_dir
├── root.py
├── same_step_0.py
├── same_step_1.py
├── same_step_2.py
└── utils
├── __init__.py
├── file_one.py
└── file_two.py
The root.py file references the functions in the same_step files. For example:
# In root.py
same_step_0.generated_function()
The "step" files have references to functions in the 'utils' directory, with a simple import and call - e.g.
from utils.file_one import function_one
function_one(val="somelongstring")
Ok, now we get to the problem - I'm working against an API which effectively requires each "step" to be a single python file. So, what I need to do is, somehow, append all functions pulled in by each step file into a single file which I can then submit to the service.
IDEALLY, I'd love to do this with as little changes to the underlying code. So, if possible, in same_step_0 I'd like to leave it as:
from utils.file_one import function_one
function_one(val="somelongstring")
and not have to change it to
# Note removing the 'utils' prefix
from file_one import function_one
function_one(val="somelongstring")
So, the question is, is there a way I can append all the dependent files or functions together into a single file such that it mimic'd import behavior? Or at least function call behavior? Ideas I've thought about:
Pickling all subfiles into a serialization format and inserting at the end of the code
Raw appending of all python files - perhaps inside an inline module?
Scraping all files for functions and attaching them manually (however, this would not preserve namespacing)
I haven't dug into these yet - I'm curious if this is even possible
I can do a lot of generation and wrapping of code, but I'd prefer not to touch the inner code (the function_one call, for example).
Goal
To create a minimal set of scripts, and an interchange format that can be stored in a Python executable to recreate a module structure.
Source
Ok here's a simple example that abuses types.ModuleType, sys, and globals() to do this:
import os
import sys
import types
class Module:
def __init__(self, name, source=None, path=None, modules=None):
self.name = name
self.source = source
self.path = path
self.modules = modules
def from_file(name, path):
return Module(name, open(path).read(), path)
def from_dir(name, path):
return Module(name, None, path, [])
def __repr__(self):
return f'Module(name={repr(self.name)}, source={repr(self.source)}, path={repr(self.path)}, modules={self.modules})'
def to_dict(self):
data = { 'name': self.name }
if self.source is not None:
data['source'] = self.source
if self.path is not None:
data['path'] = self.path
if self.modules is not None:
data['modules'] = [i.to_dict() for i in self.modules]
return data
#staticmethod
def from_dict(data):
modules = None
if 'modules' in data:
modules = [Module.from_dict(i) for i in data.pop('modules')]
return Module(**data, modules=modules)
def get_modname(parent, module):
if parent is None:
return module
if module == '__init__':
return parent
return f'{parent}.{module}'
def compile_module(module, parent=None):
modname = get_modname(parent, module.name)
mod = types.ModuleType(modname)
exec(module.source, mod.__dict__)
path = os.path.realpath(module.path)
mod.__path__ = os.path.dirname(path)
mod.__file__ = path
sys.modules[modname] = mod
globals()[modname] = mod
return mod
def compile_module_recursive(package, parent=None):
# Need to do this recursively.
mod = compile_module(package, parent)
if not package.modules:
return mod
for submodule in package.modules:
submod = compile_module_recursive(submodule, parent=get_modname(parent, package.name))
if not hasattr(mod, submodule.name):
setattr(mod, submodule.name, submod)
return mod
def read_module_recursive(directory, parent=None):
# The module order is first the:
# 1. All submodules
# 2. Then definitions inside
# Then, need to define `__name__`, `__path__` and `__file__`.
cwd = os.getcwd()
realpath = os.path.realpath(directory)
parent_dir = os.path.dirname(realpath)
base_dir = os.path.basename(realpath)
if parent is None:
parent = Module.from_dir(base_dir, realpath)
os.chdir(realpath)
for entry in os.listdir(realpath):
path = os.path.join(realpath, entry)
if os.path.isfile(path):
name, ext = os.path.splitext(entry)
if not ext == '.py':
continue
if name == '__init__':
parent.path = path
parent.source = open(path).read()
else:
parent.modules.append(Module.from_file(name, path))
elif os.path.isdir(path):
if entry == '__pycache__':
continue
path = os.path.join(realpath, entry)
# Must have processed __init__.py
if not os.path.isfile(f'{path}/__init__.py'):
continue
module = Module.from_dir(entry, path)
parent.modules.append(module)
read_module_recursive(entry, module)
os.chdir(cwd)
return parent
# SAMPLE USE
# ----------
# module = read_module_recursive('mylib')
# data = module.to_dict() # can store as a Python dict.
# module = Module.from_dict(data)
# compile_module_recursive(module)
How it works
Basically, it reads all .py files from all modules, recursively. It currently does not support extension types, only pure Python files. It then creates an intermediate, tree-like type for the entire module tree. It provides easy serialization to and from dict, so it's much easier to copy-and-paste it into your executable file.
Example use
Say I have the following directory structure:
mylib/
__init__.py
a.py
b.py
c/
__init__.py
d.py
It properly respects the import order, and how it variable assignments override imports or submodules. Please note that it imports everything recursively all at once, which differs than the traditional import structure.
mylib/init.py
a = 'e'
mylib/a.py
def afunc(a):
return str(a)
class AClass:
def x(self):
return 1
mylib/b.py
def bfunc(b):
return repr(b)
class BClass:
def x(self):
return 1
mylib/c/init.py
mylib/c/d.py
def dfunc(d):
return str(d)
class DClass:
def x(self):
return 1
Example
>>> module = read_module_recursive('mylib')
>>> data = module.to_dict() # can store as a Python dict.
>>> module = Module.from_dict(data)
>>> compile_module_recursive(module)
>>> mylib
<module 'mylib' from 'C:\\Users\\user\\OneDrive\\Desktop\\lib\\mylib\\__init__.py'>
>>> mylib.a
'e'
>>> from mylib.a import afunc # still works
>>> afunc(54)
'54'
>>> import mylib # works because `'mylib'` is in `sys.modules`.
Congrats: this is a hack, but it works, and it works nicely, and it uses Python's own packaging system to do everything, so it's much more resilient than copying and pasting. After compiling, it also respects imports (due to adding them to sys.modules). It also respects __doc__, and other attributes.
License
Public Domain or Unlicensed. Use as you see fit. Attribution would be nice, but not required.
Problem
I have a folder structure like this:
- modules
- root
- abc
hello.py
__init__.py
- xyz
hi.py
__init__.py
blah.py
__init__.py
foo.py
bar.py
__init_.py
Here is the same thing in string format:
"modules",
"modues/__init__.py",
"modules/foo.py",
"modules/bar.py",
"modules/root",
"modules/root/__init__.py",
"modules/root/blah,py",
"modules/root/abc",
"modules/root/abc/__init__.py",
"modules/root/abc/hello.py",
"modules/root/xyz",
"modules/root/xyz/__init__.py",
"modules/root/xyz/hi.py"
I am trying to print out all the modules in the python import style format.
An example output would like this:
modules.foo
modules.bar
modules.root.blah
modules.root.abc.hello
modules.root.xyz.hi
How can I do this is in python(if possible without third party libraries) easily?
What I tried
Sample Code
import pkgutil
import modules
absolute_modules = []
def find_modules(module_path):
for package in pkgutil.walk_packages(module_path):
print(package)
if package.ispkg:
find_modules([package.name])
else:
absolute_modules.append(package.name)
if __name__ == "__main__":
find_modules(modules.__path__)
for module in absolute_modules:
print(module)
However, this code will only print out 'foo' and 'bar'. But not 'root' and it's sub packages. I'm also having trouble figuring out how to convert this to preserve it's absolute import style. The current code only gets the package/module name and not the actual absolute import.
This uses setuptools.find_packages (for the packages) and pkgutil.iter_modules for their submodules. Python2 is supported as well. No need for recursion, it's all handled by these two functions used together.
import sys
from setuptools import find_packages
from pkgutil import iter_modules
def find_modules(path):
modules = set()
for pkg in find_packages(path):
modules.add(pkg)
pkgpath = path + '/' + pkg.replace('.', '/')
if sys.version_info.major == 2 or (sys.version_info.major == 3 and sys.version_info.minor < 6):
for _, name, ispkg in iter_modules([pkgpath]):
if not ispkg:
modules.add(pkg + '.' + name)
else:
for info in iter_modules([pkgpath]):
if not info.ispkg:
modules.add(pkg + '.' + info.name)
return modules
So I finally figured out how to do this cleanly and get pkgutil to take care of all the edge case for you. This code was based off python's help() function which only displays top level modules and packages.
import importlib
import pkgutil
import sys
import modules
def find_abs_modules(module):
path_list = []
spec_list = []
for importer, modname, ispkg in pkgutil.walk_packages(module.__path__):
import_path = f"{module.__name__}.{modname}"
if ispkg:
spec = pkgutil._get_spec(importer, modname)
importlib._bootstrap._load(spec)
spec_list.append(spec)
else:
path_list.append(import_path)
for spec in spec_list:
del sys.modules[spec.name]
return path_list
if __name__ == "__main__":
print(sys.modules)
print(find_abs_modules(modules))
print(sys.modules)
This will work even for builtin packages.
The below code will give you the relative package module from the codes current working directory.
import os
import re
for root,dirname,filename in os.walk(os.getcwd()):
pth_build=""
if os.path.isfile(root+"/__init__.py"):
for i in filename:
if i <> "__init__.py" and i <> "__init__.pyc":
if i.split('.')[1] == "py":
slot = list(set(root.split('\\')) -set(os.getcwd().split('\\')))
pth_build = slot[0]
del slot[0]
for j in slot:
pth_build = pth_build+"."+j
print pth_build +"."+ i.split('.')[0]
This code will display:
modules.foo
modules.bar
modules.root.blah
modules.root.abc.hello
modules.root.xyz.hi
If you run it outside the modules folder.
This question already has answers here:
How to import a module in Python with importlib.import_module
(3 answers)
Closed 8 years ago.
I have the following directory structure
+ code
|
--+ plugins
|
-- __init__.py
-- test_plugin.py (has a class TestPlugin)
-- another_test_plugin.py (has a class AnotherTestPlugin)
--+ load.py
--+ __init__.py
In load.py, I want to be able to initialize only those classes that the user specifies. For example, lets say I do something like
$ python load.py -c test_plugin # Should only import test_plugin.py and initialize an object of the TestPlugin class
I am having trouble trying to use the "imp" module to do it. It keeps on saying "No such file or directory". My understanding is that it is somehow not understanding the path properly. Can someone help me out with this?
ok, your problem is a path related problem. You expect that the script is being run in the same directory as where load.py is, where it is not the case.
what you have to do is something like:
import imp, os, plugins
path = os.path.dirname(plugins.__file__)
imp.load_source('TestPlugin', os.path.join(path, 'test_plugin.py')
where plugins is the module containing all your plugins (i.e. just the empty __init__.py), that will help you get the full path to your plugin modules' files.
Another solution, if you want a "plugins" discovery tool:
import imp, os
import glob
def load_plugins(path):
"""
Assuming `path` is the only directory in which you store your plugins,
and assuming each name follows the syntax:
plugin_file.py -> PluginFile
Please note that we don't import files starting with an underscore.
"""
plugins = {}
plugin_files = glob.glob(path + os.sep + r'[!_]*.py')
for plugin_path in plugin_files:
module_name, ext = os.path.splitext(plugin_path)
module_name = os.path.basename(module_name)
class_name = module_name.title().replace('_', '')
loaded_module = imp.load_source(class_name, plugin_path) # we import the plugin
plugins[module_name] = getattr(loaded_module, class_name)
return plugins
plugins = load_plugins(your_path_here)
plugin_name = sys.argv[3]
plugin = plugins.get(plugin_name)
if not plugin:
# manage a not existing plugin
else:
plugin_instance = plugin() # creates an instance of your plugin
This way, you can also specify different names by changing your keys, e.g., 'test_plugins' => 'tp'. You don't have to initialize your plugins, but you can still run this function whenever you want to load your plugins at runtime.
exec('import ' + sys.argv[2])
obj = test_plugin.TestPlugin()
Here sys.argv[2] is 'test_plugin' string from command line arguments.
EDIT: Another way to avoid using exec:
import importlib
mod = importlib.import_module(sys.argv[2])
I have a package of plug-in style modules. It looks like this:
/Plugins
/Plugins/__init__.py
/Plugins/Plugin1.py
/Plugins/Plugin2.py
etc...
Each .py file contains a class that derives from PluginBaseClass. So I need to list every module in the Plugins package and then search for any classes that implement PluginBaseClass. Ideally I want to be able to do something like this:
for klass in iter_plugins(project.Plugins):
action = klass()
action.run()
I have seen some other answers out there, but my situation is different. I have an actual import to the base package (ie: import project.Plugins) and I need to find the classes after discovering the modules.
Edit: here's a revised solution. I realised I was making a mistake while testing my previous one, and it doesn't really work the way you would expect. So here is a more complete solution:
import os
from imp import find_module
from types import ModuleType, ClassType
def iter_plugins(package):
"""Receives package (as a string) and, for all of its contained modules,
generates all classes that are subclasses of PluginBaseClass."""
# Despite the function name, "find_module" will find the package
# (the "filename" part of the return value will be None, in this case)
filename, path, description = find_module(package)
# dir(some_package) will not list the modules within the package,
# so we explicitly look for files. If you need to recursively descend
# a directory tree, you can adapt this to use os.walk instead of os.listdir
modules = sorted(set(i.partition('.')[0]
for i in os.listdir(path)
if i.endswith(('.py', '.pyc', '.pyo'))
and not i.startswith('__init__.py')))
pkg = __import__(package, fromlist=modules)
for m in modules:
module = getattr(pkg, m)
if type(module) == ModuleType:
for c in dir(module):
klass = getattr(module, c)
if (type(klass) == ClassType and
klass is not PluginBaseClass and
issubclass(klass, PluginBaseClass)):
yield klass
My previous solution was:
You could try something like:
from types import ModuleType
import Plugins
classes = []
for item in dir(Plugins):
module = getattr(Plugins, item)
# Get all (and only) modules in Plugins
if type(module) == ModuleType:
for c in dir(module):
klass = getattr(module, c)
if isinstance(klass, PluginBaseClass):
classes.append(klass)
Actually, even better, if you want some modularity:
from types import ModuleType
def iter_plugins(package):
# This assumes "package" is a package name.
# If it's the package itself, you can remove this __import__
pkg = __import__(package)
for item in dir(pkg):
module = getattr(pkg, item)
if type(module) == ModuleType:
for c in dir(module):
klass = getattr(module, c)
if issubclass(klass, PluginBaseClass):
yield klass
You may (and probably should) define __all__ in __init__.py as a list of the submodules in your package; this is so that you support people doing from Plugins import *. If you have done so, you can iterate over the modules with
import Plugins
import sys
modules = { }
for module in Plugins.__all__:
__import__( module )
modules[ module ] = sys.modules[ module ]
# iterate over dir( module ) as above
The reason another answer posted here fails is that __import__ imports the lowest-level module, but returns the top-level one (see the docs). I don't know why.
Scanning modules isn't good idea. If you need class registry you should look at metaclasses or use existing solutions like zope.interface.
Simple solution through metaclasses may look like that:
from functools import reduce
class DerivationRegistry(type):
def __init__(cls,name,bases,cls_dict):
type.__init__(cls,name,bases,cls_dict)
cls._subclasses = set()
for base in bases:
if isinstance(base,DerivationRegistry):
base._subclasses.add(cls)
def getSubclasses(cls):
return reduce( set.union,
( succ.getSubclasses() for succ in cls._subclasses if isinstance(succ,DerivationRegistry)),
cls._subclasses)
class Base(object):
__metaclass__ = DerivationRegistry
class Cls1(object):
pass
class Cls2(Base):
pass
class Cls3(Cls2,Cls1):
pass
class Cls4(Cls3):
pass
print(Base.getSubclasses())
If you don't know what's going to be in Plugins ahead of time, you can get a list of python files in the package's directory, and import them like so:
# compute a list of modules in the Plugins package
import os
import Plugins
plugin_modules = [f[:-3] for f in os.listdir(os.path.dirname(Plugins.__file__))
if f.endswith('.py') and f != '__init__.py']
Sorry, that comprehension might be a mouthful for someone relatively new to python. Here's a more verbose version (might be easier to follow):
plugin_modules = []
package_path = Plugins.__file__
file_list = os.listdir(os.path.dirname(package_path))
for file_name in file_list:
if file_name.endswith('.py') and file_name != '__init__.py':
plugin_modules.append(file_name)
Then you can use __import__ to get the module:
# get the first one
plugin = __import__('Plugins.' + plugin_modules[0])
I have a directory structure as follows:
| main.py
| scripts
|--| __init__.py
| script1.py
| script2.py
| script3.py
From main.py, the module scripts is imported. I tried using pkgutils.walk_packages in combination with __all__, but using that, I can only import all the submodules directly under main using from scripts import *. I would like to get them all under scripts. What would be the cleanest way to import all the submodules of scripts so that I could access scripts.script1 from main?
EDIT: I am sorry that I was a bit vague. I would like to import the submodules on run-time without specifying them explicitly in __init__.py. I can use pkgutils.walk_packages to get the submodule names (unless someone knows of a better way), but I am not sure of the cleanest way to use these names (or maybe the ImpImporters that walk_packages returns?) to import them.
Edit: Here's one way to recursively import everything at runtime...
(Contents of __init__.py in top package directory)
import pkgutil
__all__ = []
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
__all__.append(module_name)
_module = loader.find_module(module_name).load_module(module_name)
globals()[module_name] = _module
I'm not using __import__(__path__+'.'+module_name) here, as it's difficult to properly recursively import packages using it. If you don't have nested sub-packages, and wanted to avoid using globals()[module_name], though, it's one way to do it.
There's probably a better way, but this is the best I can do, anyway.
Original Answer (For context, ignore othwerwise. I misunderstood the question initially):
What does your scripts/__init__.py look like? It should be something like:
import script1
import script2
import script3
__all__ = ['script1', 'script2', 'script3']
You could even do without defining __all__, but things (pydoc, if nothing else) will work more cleanly if you define it, even if it's just a list of what you imported.
This is based on the answer that kolypto provided, but his answer does not perform recursive import of packages, whereas this does. Although not required by the main question, I believe recursive import applies and can be very useful in many similar situations. I, for one, found this question when searching on the topic.
This is a nice, clean way of performing the import of the subpackage's modules, and should be portable as well, and it uses the standard lib for python 2.7+ / 3.x.
import importlib
import pkgutil
def import_submodules(package, recursive=True):
""" Import all submodules of a module, recursively, including subpackages
:param package: package (name or actual module)
:type package: str | module
:rtype: dict[str, types.ModuleType]
"""
if isinstance(package, str):
package = importlib.import_module(package)
results = {}
for loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
full_name = package.__name__ + '.' + name
results[full_name] = importlib.import_module(full_name)
if recursive and is_pkg:
results.update(import_submodules(full_name))
return results
Usage:
# from main.py, as per the OP's project structure
import scripts
import_submodules(scripts)
# Alternatively, from scripts.__init__.py
import_submodules(__name__)
Simply works, and allows relative import inside packages:
def import_submodules(package_name):
""" Import all submodules of a module, recursively
:param package_name: Package name
:type package_name: str
:rtype: dict[types.ModuleType]
"""
package = sys.modules[package_name]
return {
name: importlib.import_module(package_name + '.' + name)
for loader, name, is_pkg in pkgutil.walk_packages(package.__path__)
}
Usage:
__all__ = import_submodules(__name__).keys()
Not nearly as clean as I would like, but none of the cleaner methods worked for me. This achieves the specified behaviour:
Directory structure:
| pkg
|--| __init__.py
| main.py
| scripts
|--| __init__.py
| script1.py
| script2.py
| script3.py
Where pkg/scripts/__init__.py is empty, and pkg/__init__.py contains:
import importlib as _importlib
import pkgutil as _pkgutil
__all__ = [_mod[1].split(".")[-1] for _mod in
filter(lambda _mod: _mod[1].count(".") == 1 and not
_mod[2] and __name__ in _mod[1],
[_mod for _mod in _pkgutil.walk_packages("." + __name__)])]
__sub_mods__ = [".".join(_mod[1].split(".")[1:]) for _mod in
filter(lambda _mod: _mod[1].count(".") > 1 and not
_mod[2] and __name__ in _mod[1],
[_mod for _mod in
_pkgutil.walk_packages("." + __name__)])]
from . import *
for _module in __sub_mods__:
_importlib.import_module("." + _module, package=__name__)
Although it's messy, it should be portable. I've used this code for several different packages.
I got tired of this problem myself, so I wrote a package called automodinit to fix it. You can get it from http://pypi.python.org/pypi/automodinit/. Usage is like this:
Include the automodinit package into your setup.py dependencies.
Add the following to the beginning of the __init__.py file:
__all__ = ["I will get rewritten"]
# Don't modify the line above, or this line!
import automodinit
automodinit.automodinit(__name__, __file__, globals())
del automodinit
# Anything else you want can go after here, it won't get modified.
That's it! From now on importing a module will set __all__ to
a list of .py[co] files in the module and will also import each
of those files as though you had typed:
for x in __all__: import x
Therefore the effect of from M import * matches exactly import M.
automodinit is happy running from inside ZIP archives and is therefore ZIP safe.
To just load all submodules of a package, you can use this simple function:
import importlib
import pkgutil
def import_submodules(module):
"""Import all submodules of a module, recursively."""
for loader, module_name, is_pkg in pkgutil.walk_packages(
module.__path__, module.__name__ + '.'):
importlib.import_module(module_name)
Use case: load all database models of a Flask app, so that Flask-Migrate could detect changes to the schema. Usage:
import myproject.models
import_submodules(myproject.models)
I was writing a small personal library and adding new modules all the time so I wrote a shell script to look for scripts and create the __init__.py's. The script is executed just outside of the main directory for my package, pylux.
I know it probably isn't the answer you're looking for, but it servered its purpose for me and it might be useful to someone else, too.
#!/bin/bash
echo 'Traversing folder hierarchy...'
CWD=`pwd`
for directory in `find pylux -type d -exec echo {} \;`;
do
cd $directory
#echo Entering $directory
echo -n "" > __init__.py
for subdirectory in `find . -type d -maxdepth 1 -mindepth 1`;
do
subdirectory=`echo $subdirectory | cut -b 3-`
#echo -n ' ' ...$subdirectory
#echo -e '\t->\t' import $subdirectory
echo import $subdirectory >> __init__.py
done
for pyfile in *.py ;
do
if [ $pyfile = $(echo __init__.py) ]; then
continue
fi
#echo -n ' ' ...$pyfile
#echo -e '\t->\t' import `echo $pyfile | cut -d . -f 1`
echo import `echo $pyfile | cut -d . -f 1` >> __init__.py
done
cd $CWD
done
for directory in `find pylux -type d -exec echo {} \;`;
do
echo $directory/__init__.py:
cat $directory/__init__.py | awk '{ print "\t"$0 }'
done
I've played around with Joe Kington's Answer and have built a solution that uses globals and get/setattr and thus doesn't need eval. A slight modification is that instead of directly using the packages __path__ for walk_packages, I use the packages parent directory and then only import modules starting with __name__ + ".". This was done to reliably get all subpackages from walk_packages - in my use case I had a subpackage named test which caused pkgutil to iterate over the test package from python's library; furthermore, using __path__ would not recurse into the packages subdirectories. All these issues were observed using jython and python2.5, the code below is only tested in jython thus far.
Also note that OPs question only talks about importing all modules from a package, this code recursively imports all packages too.
from pkgutil import walk_packages
from os import path
__all__ = []
__pkg_prefix = "%s." % __name__
__pkg_path = path.abspath(__path__[0]).rsplit("/", 1)[0] #parent directory
for loader, modname, _ in walk_packages([__pkg_path]):
if modname.startswith(__pkg_prefix):
#load the module / package
module = loader.find_module(modname).load_module(modname)
modname = modname[len(__pkg_prefix):] #strip package prefix from name
#append all toplevel modules and packages to __all__
if not "." in modname:
__all__.append(modname)
globals()[modname] = module
#set everything else as an attribute of their parent package
else:
#get the toplevel package from globals()
pkg_name, rest = modname.split(".", 1)
pkg = globals()[pkg_name]
#recursively get the modules parent package via getattr
while "." in rest:
subpkg, rest = rest.split(".", 1)
pkg = getattr(pkg, subpkg)
#set the module (or package) as an attribute of its parent package
setattr(pkg, rest, module)
As a future improvement I'll try to make this dynamic with a __getattr__ hook on the package, so the actual modules are only imported when they are accessed...
This works nicely for me in Python 3.3. Note that this works only for submodules which are in files in the same directory as the __init__.py. With some work however it can be enhanced for supporting submodules in directories too.
from glob import iglob
from os.path import basename, relpath, sep, splitext
def import_submodules(__path__to_here):
"""Imports all submodules.
Import this function in __init__.py and put this line to it:
__all__ = import_submodules(__path__)"""
result = []
for smfile in iglob(relpath(__path__to_here[0]) + "/*.py"):
submodule = splitext(basename(smfile))[0]
importstr = ".".join(smfile.split(sep)[:-1])
if not submodule.startswith("_"):
__import__(importstr + "." + submodule)
result.append(submodule)
return result