I'd like to embed pylint in a program. The user enters python programs (in Qt, in a QTextEdit, although not relevant) and in the background I call pylint to check the text he enters. Finally, I print the errors in a message box.
There are thus two questions: First, how can I do this without writing the entered text to a temporary file and giving it to pylint ? I suppose at some point pylint (or astroid) handles a stream and not a file anymore.
And, more importantly, is it a good idea ? Would it cause problems for imports or other stuffs ? Intuitively I would say no since it seems to spawn a new process (with epylint) but I'm no python expert so I'm really not sure. And if I use this to launch pylint, is it okay too ?
Edit:
I tried tinkering with pylint's internals, event fought with it, but finally have been stuck at some point.
Here is the code so far:
from astroid.builder import AstroidBuilder
from astroid.exceptions import AstroidBuildingException
from logilab.common.interface import implements
from pylint.interfaces import IRawChecker, ITokenChecker, IAstroidChecker
from pylint.lint import PyLinter
from pylint.reporters.text import TextReporter
from pylint.utils import PyLintASTWalker
class Validator():
def __init__(self):
self._messagesBuffer = InMemoryMessagesBuffer()
self._validator = None
self.initValidator()
def initValidator(self):
self._validator = StringPyLinter(reporter=TextReporter(output=self._messagesBuffer))
self._validator.load_default_plugins()
self._validator.disable('W0704')
self._validator.disable('I0020')
self._validator.disable('I0021')
self._validator.prepare_import_path([])
def destroyValidator(self):
self._validator.cleanup_import_path()
def check(self, string):
return self._validator.check(string)
class InMemoryMessagesBuffer():
def __init__(self):
self.content = []
def write(self, st):
self.content.append(st)
def messages(self):
return self.content
def reset(self):
self.content = []
class StringPyLinter(PyLinter):
"""Does what PyLinter does but sets checkers once
and redefines get_astroid to call build_string"""
def __init__(self, options=(), reporter=None, option_groups=(), pylintrc=None):
super(StringPyLinter, self).__init__(options, reporter, option_groups, pylintrc)
self._walker = None
self._used_checkers = None
self._tokencheckers = None
self._rawcheckers = None
self.initCheckers()
def __del__(self):
self.destroyCheckers()
def initCheckers(self):
self._walker = PyLintASTWalker(self)
self._used_checkers = self.prepare_checkers()
self._tokencheckers = [c for c in self._used_checkers if implements(c, ITokenChecker)
and c is not self]
self._rawcheckers = [c for c in self._used_checkers if implements(c, IRawChecker)]
# notify global begin
for checker in self._used_checkers:
checker.open()
if implements(checker, IAstroidChecker):
self._walker.add_checker(checker)
def destroyCheckers(self):
self._used_checkers.reverse()
for checker in self._used_checkers:
checker.close()
def check(self, string):
modname = "in_memory"
self.set_current_module(modname)
astroid = self.get_astroid(string, modname)
self.check_astroid_module(astroid, self._walker, self._rawcheckers, self._tokencheckers)
self._add_suppression_messages()
self.set_current_module('')
self.stats['statement'] = self._walker.nbstatements
def get_astroid(self, string, modname):
"""return an astroid representation for a module"""
try:
return AstroidBuilder().string_build(string, modname)
except SyntaxError as ex:
self.add_message('E0001', line=ex.lineno, args=ex.msg)
except AstroidBuildingException as ex:
self.add_message('F0010', args=ex)
except Exception as ex:
import traceback
traceback.print_exc()
self.add_message('F0002', args=(ex.__class__, ex))
if __name__ == '__main__':
code = """
a = 1
print(a)
"""
validator = Validator()
print(validator.check(code))
The traceback is the following:
Traceback (most recent call last):
File "validator.py", line 16, in <module>
main()
File "validator.py", line 13, in main
print(validator.check(code))
File "validator.py", line 30, in check
self._validator.check(string)
File "validator.py", line 79, in check
self.check_astroid_module(astroid, self._walker, self._rawcheckers, self._tokencheckers)
File "c:\Python33\lib\site-packages\pylint\lint.py", line 659, in check_astroid_module
tokens = tokenize_module(astroid)
File "c:\Python33\lib\site-packages\pylint\utils.py", line 103, in tokenize_module
print(module.file_stream)
AttributeError: 'NoneType' object has no attribute 'file_stream'
# And sometimes this is added :
File "c:\Python33\lib\site-packages\astroid\scoped_nodes.py", line 251, in file_stream
return open(self.file, 'rb')
OSError: [Errno 22] Invalid argument: '<?>'
I'll continue digging tomorrow. :)
I got it running.
the first one (NoneType …) is really easy and a bug in your code:
Encountering an exception can make get_astroid “fail”, i.e. send one syntax error message and return!
But for the secong one… such bullshit in pylint’s/logilab’s API… Let me explain: Your astroid object here is of type astroid.scoped_nodes.Module.
It’s also created by a factory, AstroidBuilder, which sets astroid.file = '<?>'.
Unfortunately, the Module class has following property:
#property
def file_stream(self):
if self.file is not None:
return open(self.file, 'rb')
return None
And there’s no way to skip that except for subclassing (Which would render us unable to use the magic in AstroidBuilder), so… monkey patching!
We replace the ill-defined property with one that checks an instance for a reference to our code bytes (e.g. astroid._file_bytes) before engaging in above default behavior.
def _monkeypatch_module(module_class):
if module_class.file_stream.fget.__name__ == 'file_stream_patched':
return # only patch if patch isn’t already applied
old_file_stream_fget = module_class.file_stream.fget
def file_stream_patched(self):
if hasattr(self, '_file_bytes'):
return BytesIO(self._file_bytes)
return old_file_stream_fget(self)
module_class.file_stream = property(file_stream_patched)
That monkeypatching can be called just before calling check_astroid_module. But one more thing has to be done. See, there’s more implicit behavior: Some checkers expect and use astroid’s file_encoding field. So we now have this code in the middle of check:
astroid = self.get_astroid(string, modname)
if astroid is not None:
_monkeypatch_module(astroid.__class__)
astroid._file_bytes = string.encode('utf-8')
astroid.file_encoding = 'utf-8'
self.check_astroid_module(astroid, self._walker, self._rawcheckers, self._tokencheckers)
One could say that no amount of linting creates actually good code. Unfortunately pylint unites enormous complexity with a specialization of calling it on files. Really good code has a nice native API and wraps that with a CLI interface. Don’t ask me why file_stream exists if internally, Module gets built from but forgets the source code.
PS: i had to change sth else in your code: load_default_plugins has to come before some other stuff (maybe prepare_checkers, maybe sth. else)
PPS: i suggest subclassing BaseReporter and using that instead of your InMemoryMessagesBuffer
PPPS: this just got pulled (3.2014), and will fix this: https://bitbucket.org/logilab/astroid/pull-request/15/astroidbuilderstring_build-was/diff
4PS: this is now in the official version, so no monkey patching required: astroid.scoped_nodes.Module now has a file_bytes property (without leading underscore).
Working with an unlocatable stream may definitly cause problems in case of relative imports, since the location is then needed to find the actually imported module.
Astroid support building an AST from a stream, but this is not used/exposed through Pylint which is a level higher and designed to work with files. So while you may acheive this it will need a bit of digging into the low-level APIs.
The easiest way is definitly to save the buffer to the file then to use the SA answer to start pylint programmatically if you wish (totally forgot this other account of mine found in other responses ;). Another option being to write a custom reporter to gain more control.
Related
I am trying to call a method on an other class and give the called class a reference of the current class along with some other parameters. But somehow it takes the self given as a parameter as the self of the called class.
Let me show you:
import os, sys
from wsPart import wsPart
class thermo(wsPart):
functional = False ## see line 8
file = '/sys/bus/w1/devices/28-00000833e8ff/w1_slave'
def __init__(self, name, logger):
super().__init__(name, logger)
functional = True
def read(self):
fileobject = open(self.file)
filecontent = fileobject.read()
fileobject.close()
self.logger.writeLog(self,"Completed Meassurement") ##Problem on this line
return filecontent
So I call the class logger and the method writeLog on it. Giving the Parameters message and a reference of the class thermo (self).
import datetime
from wsPart import wsPart
class logger():
logfile = "/var/log/wheaterstation.log"
name = "Logger"
def writeLog(self, sender, message):
conn = open(self.logfile, "w")
now = str(datetime.datetime.now().isoformat())
conn.write("[" + now + "]" + " (" + sender.getName() + "): " + message + "\n") ##Problem on this line
conn.close()
As you can see I put the parameters self because its a method that belongs to a class, the sender should be the reference to the class thermo that was passed as self in the thermo class. Lastly there is the message which was passed in the thermo class as well.
But this just gives me the error:
Traceback (most recent call last):
File "scrLib/wsControl.py", line 61, in <module>
controller = controller()
File "scrLib/wsControl.py", line 22, in __init__
self.thermo = thermo("Thermometer", logger)
File "/home/joco/git/wheaterstation/scrLib/thermo.py", line 10, in __init__
super().__init__(name, logger)
File "/home/joco/git/wheaterstation/scrLib/wsPart.py", line 8, in __init__
self.logger.writeLog(self, "created")
TypeError: writeLog() missing 1 required positional argument: 'message'
So it seems that the self parameter which was passed in the thermo class is interpeted as the self of the class logger which gets it all mixed up.
Can you guys help me here?
The full code + additional comments can be viewed Here
Edit:
Both the logger and the thermo class get initialized in the file wsPart.py:
class controller():
name = ""
logger = None
thermo = None
dbConnector = None
def __init__(self):
##THis created the controller and all the other objects
self.name = "Controller"
##Create Objects
self.logger = logger()
self.logger.writeLog(self,"logger created") ##This line Works
self.thermo = thermo("Thermometer", logger)
self.dbConnector = dbConnector("DBConnector",logger)
yes, bad idea to call the instance and the class name the same. Here:
self.logger = logger()
self.logger.writeLog(self,"logger created") ##This line Works
self.thermo = thermo("Thermometer", logger)
self.dbConnector = dbConnector("DBConnector",logger)
You're passing the class itself to your constructors. So the methods are seen as static/expect one more parameter. You need to change the 2 last lines to pass the instance you just created:
self.thermo = thermo("Thermometer", self.logger)
self.dbConnector = dbConnector("DBConnector", self.logger)
more importantly, you need to use different names for classes and instances of the same objects to avoid that confusion (python convention for class names is starting each word with upper case (camelcase) ex: Logger. Other languages don't use that convention, but python is a lot about conventions).
With a different name you'd have gotten a NameError exception and you would have fixed the error yourself.
Aside: don't "initialize" members like this in the class definition:
name = ""
logger = None
thermo = None
dbConnector = None
those are creating class members, not instance members. Remove those, and let __init__ create instance members like you're currently doing. __init__ is called no matter what, and those lines above just add to the confusion (except for some corner cases, only constants should be declared that way)
Totally unrelated but code in comments is unreadable so I post this as an answer:
this does not work as you seem to expect:
class Whatever():
functional = False ## see line 8
def __init__(self, name, logger):
super().__init__(name, logger)
functional = True
Python has no "implied this" sor here in __init__ you're not creating an instance attribute but a local variable. You want self.functional = True
Make sure you close files
def read(self):
fileobject = open(self.file)
filecontent = fileobject.read()
fileobject.close()
If anything wrong happens between open() and fileobject.close(), the file is not garanteed to be properly closed. You want eiher a try/finally block ie
f = open(path)
try:
do_something_with(f)
finally:
f.close()
or much better a with block:
with open(path) as f:
do_something_with(f)
which will ensure the file is closed whatever happens.
write mode truncates the file
def writeLog(self, sender, message):
conn = open(self.logfile, "w")
now = str(datetime.datetime.now().isoformat())
conn.write("[" + now + "]" + " (" + sender.getName() + "): " + message + "\n") ##Problem on this line
conn.close()
as documented, opening a file in write mode truncates the file. You probably want the "append" mode instead here.
Don't reinvent the squared wheel when there's a round one already
Logging is not as trivial as just writing to a file (concurrency issues, need to send the log message to some other destination, logging levels etc), and even if you don't need more (at least the moment) your solution is quite inefficient (opening a file is expensive).
Python has a very comprehensive logging package in it's standard lib. I wholefully agree that it requires a bit of learning to configure and use properly but that's still a huge win compared to the time you'll spend trying to make a naive half-backed custom implementation works properly on production, AND this is a knowledge that you will need for just any serious project anyway.
Environment
Windows 10 + python 3.6.3 64 bit (also tried 32 bit). I am a python developer trying to use COM for (nearly) the first time and hit this huge blocker.
Problem
I have had various errors when trying to use an IRTDServer implemented in a dll (not written by me), via either win32com or comtypes. Using win32com turned out to be more difficult. I have an included an example unittest for both libraries below.
Accessing the server from Excel 2016 works as expected; this returns the expected value:
=RTD("foo.bar", , "STAT1", "METRIC1")
Code using win32com library
Here is a simple test case which should connect to the server but doesn't. (This is just one version, as I have changed it many times trying to debug the problem.)
from unittest import TestCase
class COMtest(TestCase):
def test_win32com(self):
import win32com.client
from win32com.server.util import wrap
class RTDclient:
# are these only required when implementing the server?
_com_interfaces_ = ["IRTDUpdateEvent"]
_public_methods_ = ["Disconnect", "UpdateNotify"]
_public_attrs_ = ["HeartbeatInterval"]
def __init__(self, *args, **kwargs):
self._comObj = win32com.client.Dispatch(*args, **kwargs)
def connect(self):
self._rtd = win32com.client.CastTo(self._comObj, 'IRtdServer')
result = self._rtd.ServerStart(wrap(self))
assert result > 0
def UpdateNotify(self):
print("UpdateNotify() callback")
def Disconnect(self):
print("Disconnect() called")
HeartbeatInterval = -1
_rtd = RTDclient("foo.bar")
_rtd.connect()
Result:
Traceback (most recent call last):
File "env\lib\site-packages\win32com\client\gencache.py", line 532, in EnsureDispatch
ti = disp._oleobj_.GetTypeInfo()
pywintypes.com_error: (-2147467263, 'Not implemented', None, None)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "test\test.py", line 23, in test_win32com
_rtd.connect()
File "test\test.py", line 16, in connect
self._rtd = win32com.client.CastTo(dispatch, 'IRtdServer')
File "env\lib\site-packages\win32com\client\__init__.py", line 134, in CastTo
ob = gencache.EnsureDispatch(ob)
File "env\lib\site-packages\win32com\client\gencache.py", line 543, in EnsureDispatch
raise TypeError("This COM object can not automate the makepy process - please run makepy manually for this object")
TypeError: This COM object can not automate the makepy process - please run makepy manually for this object
Following those directions, I ran the makepy script successfully:
> env\Scripts\python.exe env\lib\site-packages\win32com\client\makepy.py "foo.bar"
Generating to C:\Users\user1\AppData\Local\Temp\gen_py\3.5\longuuid1x0x1x0.py
Building definitions from type library...
Generating...
Importing module
(I replaced the UUID on stackoverflow for privacy. This UUID is the same as the typelib UUID for "foo.bar".)
The generated file contains various the function and type definitions of both IRtdServer and IRTDUpdateEvent. But in this file, both interfaces are subclasses of win32com.client.DispatchBaseClass, while according to OleViewDotNet, they should be subclasses of IUnknown?
However, when I attempted to run the unittest again, I received the exact same error as before. It is as if the lookup mechanism is not finding the generated module?
Also, GetTypeInfo returning Not implemented is alarming me. From my understanding, win32com uses that method (part of IDispatch COM interface) to determine the argument and return types for all other functions in other interfaces, including IRtdServer. If it's not implemented, it would be unable to determine the types correctly. Yet, the generated file seems to include this information, which is also perplexing.
Code using comtypes library
from unittest import TestCase
class COMtest(TestCase):
def test_comtypes(self):
import comtypes.client
class RTDclient:
# are these for win32com only?
_com_interfaces_ = ["IRTDUpdateEvent"]
_public_methods_ = ["Disconnect", "UpdateNotify"]
_public_attrs_ = ["HeartbeatInterval"]
def __init__(self, clsid):
self._comObj = comtypes.client.CreateObject(clsid)
def connect(self):
self._rtd = self._comObj.IRtdServer()
result = self._rtd.ServerStart(self)
assert result > 0
def UpdateNotify(self):
print("UpdateNotify() callback")
def Disconnect(self):
print("Disconnect() called")
HeartbeatInterval = -1
_rtd = RTDclient("foo.bar")
_rtd.connect()
Result:
File "test\test.py", line 27, in test_comtypes
_rtd.connect()
File "test\test.py", line 16, in connect
self._rtd = self._comObj.IRTDServer()
File "env\lib\site-packages\comtypes\client\dynamic.py", line 110, in __getattr__
dispid = self._comobj.GetIDsOfNames(name)[0]
File "env\lib\site-packages\comtypes\automation.py", line 708, in GetIDsOfNames
self.__com_GetIDsOfNames(riid_null, arr, len(names), lcid, ids)
_ctypes.COMError: (-2147352570, 'Unknown name.', (None, None, None, 0, None))
Some other solutions I've tried
(Based on googling and answers in the comments below)
(Re-)Registered the DLL
Registered the 32 bit version of the DLL and tried python 32 bit
Set compatibility mode of python.exe to Windows XP SP3
Tried not instantiating IRtdServer, that is, replacing these two lines:
self._rtd = self._comObj.IRtdServer()
result = self._rtd.ServerStart(self)
with:
result = self._comObj.ServerStart(self)
The error this time is:
TypeError: 'NoneType' object is not callable
That would seem to indicate that the ServerStart function exists, but is undefined? (Seems really weird. There must be more to this mystery.)
Tried passing interface="IRtdServer" parameter to CreateObject:
def __init__(self, clsid):
self._comObj = comtypes.client.CreateObject(clsid, interface="IRtdServer")
def connect(self):
result = self._comObj.ServerStart(self)
...
The error received is:
File "test\test.py", line 13, in __init__
self._comObj = comtypes.client.CreateObject(clsid, interface="IRtdServer")
File "env\lib\site-packages\comtypes\client\__init__.py", line 238, in CreateObject
obj = comtypes.CoCreateInstance(clsid, clsctx=clsctx, interface=interface)
File "env\lib\site-packages\comtypes\__init__.py", line 1223, in CoCreateInstance
p = POINTER(interface)()
TypeError: Cannot create instance: has no _type_
Tracing code in the comtypes library, that would seem to indicate that the interface parameter wants an interface class, not a string. I found various interfaces defined in the comtypes library: IDispatch, IPersist, IServiceProvider. All are subclasses of IUnknown. According to OleViewDotNet, IRtdServer is also a subclass of IUnknown. This leads me to believe that I need to similarly write an IRtdServer class in python in order to use the interface, but I don't know how to do that.
I noticed the dynamic parameter of CreateObject. The code indicates this is mutually exclusive to the interface parameter, so I tried that:
def __init__(self, clsid):
self._comObj = comtypes.client.CreateObject(clsid, dynamic=True)
def connect(self):
self._rtd = self._comObj.IRtdServer()
result = self._rtd.ServerStart(self)
But the error is the same as my original error: IRtdServer has _ctypes.COMError: (-2147352570, 'Unknown name.', (None, None, None, 0, None))
Any help or clues would be greatly be appreciated. Thank you in advance.
(Not really knowing what I'm doing,) I tried to use OleViewDotNet to look at the DLL:
I ran into same problem.
I also tried using win32com to get excel run that for me, that's a bit unstable to be honest...I cannot even touch my Excel.
Therefore I spent some time looking into this. The problem lies with CastTo. Think that COM object you (and I) loaded just does not contain enough information to be casted (some methods like GetTypeInfo are not implemented etc...)
Therefore I created a wrapper that makes methods of those COM objects callable...not obvious. And this seems working for me.
Client code is modified from a project called pyrtd, which didn't work for various reasons (think due to change of RTD model...return of RefreshData is just completely different now).
import functools
import pythoncom
import win32com.client
from win32com import universal
from win32com.client import gencache
from win32com.server.util import wrap
EXCEL_TLB_GUID = '{00020813-0000-0000-C000-000000000046}'
EXCEL_TLB_LCID = 0
EXCEL_TLB_MAJOR = 1
EXCEL_TLB_MINOR = 4
gencache.EnsureModule(EXCEL_TLB_GUID, EXCEL_TLB_LCID, EXCEL_TLB_MAJOR, EXCEL_TLB_MINOR)
universal.RegisterInterfaces(EXCEL_TLB_GUID,
EXCEL_TLB_LCID, EXCEL_TLB_MAJOR, EXCEL_TLB_MINOR,
['IRtdServer', 'IRTDUpdateEvent'])
# noinspection PyProtectedMember
class ObjectWrapperCOM:
"""
This object can act as a wrapper for an object dispatched using win32com.client.Dispatch
Sometimes the object written by 3rd party is not well constructed that win32com will not be able to obtain
type information etc in order to cast the object to a certain interface. win32com.client.CastTo will fail.
This wrapper class will enable the object to call its methods in this case, even if we do not know what exactly
the wrapped object is.
"""
LCID = 0x0
def __init__(self, obj):
self._impl = obj # type: win32com.client.CDispatch
def __getattr__(self, item):
flags, dispid = self._impl._find_dispatch_type_(item)
if dispid is None:
raise AttributeError("{} is not a valid property or method for this object.".format(item))
return functools.partial(self._impl._oleobj_.Invoke, dispid, self.LCID, flags, True)
# noinspection PyPep8Naming
class RTDUpdateEvent:
"""
Implements interface IRTDUpdateEvent from COM imports
"""
_com_interfaces_ = ['IRTDUpdateEvent']
_public_methods_ = ['Disconnect', 'UpdateNotify']
_public_attrs_ = ['HeartbeatInterval']
# Implementation of IRTDUpdateEvent.
HeartbeatInterval = -1
def __init__(self, event_driven=True):
self.ready = False
self._event_driven = event_driven
def UpdateNotify(self):
if self._event_driven:
self.ready = True
def Disconnect(self):
pass
class RTDClient:
"""
Implements a Real-Time-Data (RTD) client for accessing COM data sources that provide an IRtdServer interface.
"""
MAX_REGISTERED_TOPICS = 1024
def __init__(self, class_id):
"""
:param classid: can either be class ID or program ID
"""
self._class_id = class_id
self._rtd = None
self._update_event = None
self._topic_to_id = {}
self._id_to_topic = {}
self._topic_values = {}
self._last_topic_id = 0
def connect(self, event_driven=True):
"""
Connects to the RTD server.
Set event_driven to false if you to disable update notifications.
In this case you'll need to call refresh_data manually.
"""
dispatch = win32com.client.Dispatch(self._class_id)
self._update_event = RTDUpdateEvent(event_driven)
try:
self._rtd = win32com.client.CastTo(dispatch, 'IRtdServer')
except TypeError:
# Automated makepy failed...no detailed construction available for the class
self._rtd = ObjectWrapperCOM(dispatch)
self._rtd.ServerStart(wrap(self._update_event))
def update(self):
"""
Check if there is data waiting and call RefreshData if necessary. Returns True if new data has been received.
Note that you should call this following a call to pythoncom.PumpWaitingMessages(). If you neglect to
pump the message loop you'll never receive UpdateNotify callbacks.
"""
# noinspection PyUnresolvedReferences
pythoncom.PumpWaitingMessages()
if self._update_event.ready:
self._update_event.ready = False
self.refresh_data()
return True
else:
return False
def refresh_data(self):
"""
Grabs new data from the RTD server.
"""
(ids, values) = self._rtd.RefreshData(self.MAX_REGISTERED_TOPICS)
for id_, value in zip(ids, values):
if id_ is None and value is None:
# This is probably the end of message
continue
assert id_ in self._id_to_topic, "Topic ID {} is not registered.".format(id_)
topic = self._id_to_topic[id_]
self._topic_values[topic] = value
def get(self, topic: tuple):
"""
Gets the value of a registered topic. Returns None if no value is available. Throws an exception if
the topic isn't registered.
"""
assert topic in self._topic_to_id, 'Topic %s not registered.' % (topic,)
return self._topic_values.get(topic)
def register_topic(self, topic: tuple):
"""
Registers a topic with the RTD server. The topic's value will be updated in subsequent data refreshes.
"""
if topic not in self._topic_to_id:
id_ = self._last_topic_id
self._last_topic_id += 1
self._topic_to_id[topic] = id_
self._id_to_topic[id_] = topic
self._rtd.ConnectData(id_, topic, True)
def unregister_topic(self, topic: tuple):
"""
Un-register topic so that it will not get updated.
:param topic:
:return:
"""
assert topic in self._topic_to_id, 'Topic %s not registered.' % (topic,)
self._rtd.DisconnectData(self._topic_to_id[topic])
def disconnect(self):
"""
Closes RTD server connection.
:return:
"""
self._rtd.ServerTerminate()
There seems to already be both server/client for Excel 2002.
pyrtd
Looking at that source, once you create a dispatch object, then it seems to be cast to IRtdServer.
Extract related parts, it becomes below.
from win32com import client, universal
from win32com.server.util import wrap
def __init__(self, classid):
self._classid = classid
self._rtd = None
def connect(self, event_driven=True):
dispatch = client.Dispatch(self._classid)
self._rtd = client.CastTo(dispatch, 'IRtdServer')
if event_driven:
self._rtd.ServerStart(wrap(self))
else:
self._rtd.ServerStart(None)
Please refer to client.py and examples/rtdtime.py of the following sources.
pyrtd - default
pyrtd/rtd/client.py
pyrtd/examples/rtdtime.py
I hope the following question is not too long. But otherwise I cannot explain by problem and what I want:
Learned from How to use importlib to import modules from arbitrary sources? (my question of yesterday)
I have written a specfic loader for a new file type (.xxx).
(In fact the xxx is an encrypted version of a pyc to protect code from being stolen).
I would like just to add an import hook for the new file type "xxx" without affecting the other types (.py, .pyc, .pyd) in any way.
Now, the loader is ModuleLoader, inheriting from mportlib.machinery.SourcelessFileLoader.
Using sys.path_hooks the loader shall be added as a hook:
myFinder = importlib.machinery.FileFinder
loader_details = (ModuleLoader, ['.xxx'])
sys.path_hooks.append(myFinder.path_hook(loader_details))
Note: This is activated once by calling modloader.activateLoader()
Upon loading a module named test (which is a test.xxx) I get:
>>> import modloader
>>> modloader.activateLoader()
>>> import test
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ImportError: No module named 'test'
>>>
However, when I delete content of sys.path_hooks before adding the hook:
sys.path_hooks = []
sys.path.insert(0, '.') # current directory
sys.path_hooks.append(myFinder.path_hook(loader_details))
it works:
>>> modloader.activateLoader()
>>> import test
using xxx class
in xxxLoader exec_module
in xxxLoader get_code: .\test.xxx
ANALYZING ...
GENERATE CODE OBJECT ...
2 0 LOAD_CONST 0
3 LOAD_CONST 1 ('foo2')
6 MAKE_FUNCTION 0
9 STORE_NAME 0 (foo2)
12 LOAD_CONST 2 (None)
15 RETURN_VALUE
>>>>>> test
<module 'test' from '.\\test.xxx'>
The module is imported correctly after conversion of the files content to a code object.
However I cannot load the same module from a package: import pack.test
Note: __init__.py is of course as an empty file in pack directory.
>>> import pack.test
Traceback (most recent call last):
File "<frozen importlib._bootstrap>", line 2218, in _find_and_load_unlocked
AttributeError: 'module' object has no attribute '__path__'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ImportError: No module named 'pack.test'; 'pack' is not a package
>>>
Not enough, I cannot load plain *.py modules from that package anymore: I get the same error as above:
>>> import pack.testpy
Traceback (most recent call last):
File "<frozen importlib._bootstrap>", line 2218, in _find_and_load_unlocked
AttributeError: 'module' object has no attribute '__path__'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ImportError: No module named 'pack.testpy'; 'pack' is not a package
>>>
For my understanding sys.path_hooks is traversed until the last entry is tried. So why is the first variant (without deleting sys.path_hooks) not recognizing the new extension "xxx" and the second variant (deleting sys.path_hooks) do?
It looks like the machinery is throwing an exception rather than traversing further to the next entry, when an entry of sys.path_hooks is not able to recognize "xxx".
And why is the second version working for py, pyc and xxx modules in the current directory, but not working in the package pack? I would expect that py and pyc is not even working in the current dir, because sys.path_hooks contains only a hook for "xxx"...
The short answer is that the default PathFinder in sys.meta_path isn't meant to have new file extensions and importers added in the same paths it already supports. But there's still hope!
Quick Breakdown
sys.path_hooks is consumed by the importlib._bootstrap_external.PathFinder class.
When an import happens, each entry in sys.meta_path is asked to find a matching spec for the requested module. The PathFinder in particular will then take the contents of sys.path and pass it to the factory functions in sys.path_hooks. Each factory function has a chance to either raise an ImportError (basically the factory saying "nope, I don't support this path entry") or return a finder instance for that path. The first successfully returned finder is then cached in sys.path_importer_cache. From then on PathFinder will only ask those cached finder instances if they can provide the requested module.
If you look at the contents of sys.path_importer_cache, you'll see all of the directory entries from sys.path have been mapped to FileFinder instances. Non-directory entries (zip files, etc) will be mapped to other finders.
Thus, if you append a new factory created via FileFinder.path_hook to sys.path_hooks, your factory will only be invoked if the previous FileFinder hook didn't accept the path. This is unlikely, since FileFinder will work on any existing directory.
Alternatively, if you insert your new factory to sys.path_hooks ahead of the existing factories, the default hook will only be used if your new factory doesn't accept the path. And again, since FileFinder is so liberal with what it will accept, this would lead to only your loader being used, as you've already observed.
Making it Work
So you can either try to adjust that existing factory to also support your file extension and importer (which is difficult as the importers and extension string tuples are held in a closure), or do what I ended up doing, which is add a new meta path finder.
So eg. from my own project,
import sys
from importlib.abc import FileLoader
from importlib.machinery import FileFinder, PathFinder
from os import getcwd
from os.path import basename
from sibilant.module import prep_module, exec_module
SOURCE_SUFFIXES = [".lspy", ".sibilant"]
_path_importer_cache = {}
_path_hooks = []
class SibilantPathFinder(PathFinder):
"""
An overridden PathFinder which will hunt for sibilant files in
sys.path. Uses storage in this module to avoid conflicts with the
original PathFinder
"""
#classmethod
def invalidate_caches(cls):
for finder in _path_importer_cache.values():
if hasattr(finder, 'invalidate_caches'):
finder.invalidate_caches()
#classmethod
def _path_hooks(cls, path):
for hook in _path_hooks:
try:
return hook(path)
except ImportError:
continue
else:
return None
#classmethod
def _path_importer_cache(cls, path):
if path == '':
try:
path = getcwd()
except FileNotFoundError:
# Don't cache the failure as the cwd can easily change to
# a valid directory later on.
return None
try:
finder = _path_importer_cache[path]
except KeyError:
finder = cls._path_hooks(path)
_path_importer_cache[path] = finder
return finder
class SibilantSourceFileLoader(FileLoader):
def create_module(self, spec):
return None
def get_source(self, fullname):
return self.get_data(self.get_filename(fullname)).decode("utf8")
def exec_module(self, module):
name = module.__name__
source = self.get_source(name)
filename = basename(self.get_filename(name))
prep_module(module)
exec_module(module, source, filename=filename)
def _get_lspy_file_loader():
return (SibilantSourceFileLoader, SOURCE_SUFFIXES)
def _get_lspy_path_hook():
return FileFinder.path_hook(_get_lspy_file_loader())
def _install():
done = False
def install():
nonlocal done
if not done:
_path_hooks.append(_get_lspy_path_hook())
sys.meta_path.append(SibilantPathFinder)
done = True
return install
_install = _install()
_install()
The SibilantPathFinder overrides PathFinder and replaces only those methods which reference sys.path_hook and sys.path_importer_cache with similar implementations which instead look in a _path_hook and _path_importer_cache which are local to this module.
During import, the existing PathFinder will try to find a matching module. If it cannot, then my injected SibilantPathFinder will re-traverse the sys.path and try to find a match with one of my own file extensions.
Figuring More Out
I ended up delving into the source for the _bootstrap_external module
https://github.com/python/cpython/blob/master/Lib/importlib/_bootstrap_external.py
The _install function and the PathFinder.find_spec method are the best starting points to seeing why things work the way they do.
#obriencj's analysis of the situation is correct. But I came up with a different solution to this problem that doesn't require putting anything in sys.meta_path. Instead, it installs a special hook in sys.path_hooks that acts almost as a sort of middle-ware between the PathFinder in sys.meta_path, and the hooks in sys.path_hooks where, rather than just using the first hook that says "I can handle this path!" it tries all matching hooks in order, until it finds one that actually returns a useful ModuleSpec from its find_spec method:
#PathEntryFinder.register
class MetaFileFinder:
"""
A 'middleware', if you will, between the PathFinder sys.meta_path hook,
and sys.path_hooks hooks--particularly FileFinder.
The hook returned by FileFinder.path_hook is rather 'promiscuous' in that
it will handle *any* directory. So if one wants to insert another
FileFinder.path_hook into sys.path_hooks, that will totally take over
importing for any directory, and previous path hooks will be ignored.
This class provides its own sys.path_hooks hook as follows: If inserted
on sys.path_hooks (it should be inserted early so that it can supersede
anything else). Its find_spec method then calls each hook on
sys.path_hooks after itself and, for each hook that can handle the given
sys.path entry, it calls the hook to create a finder, and calls that
finder's find_spec. So each sys.path_hooks entry is tried until a spec is
found or all finders are exhausted.
"""
class hook:
"""
Use this little internal class rather than a function with a closure
or a classmethod or anything like that so that it's easier to
identify our hook and skip over it while processing sys.path_hooks.
"""
def __init__(self, basepath=None):
self.basepath = os.path.abspath(basepath)
def __call__(self, path):
if not os.path.isdir(path):
raise ImportError('only directories are supported', path=path)
elif not self.handles(path):
raise ImportError(
'only directories under {} are supported'.format(
self.basepath), path=path)
return MetaFileFinder(path)
def handles(self, path):
"""
Return whether this hook will handle the given path, depending on
what its basepath is.
"""
path = os.path.abspath(path)
return (self.basepath is None or
os.path.commonpath([self.basepath, path]) == self.basepath)
def __init__(self, path):
self.path = path
self._finder_cache = {}
def __repr__(self):
return '{}({!r})'.format(self.__class__.__name__, self.path)
def find_spec(self, fullname, target=None):
if not sys.path_hooks:
return None
last = len(sys.path_hooks) - 1
for idx, hook in enumerate(sys.path_hooks):
if isinstance(hook, self.__class__.hook):
continue
finder = None
try:
if hook in self._finder_cache:
finder = self._finder_cache[hook]
if finder is None:
# We've tried this finder before and got an ImportError
continue
except TypeError:
# The hook is unhashable
pass
if finder is None:
try:
finder = hook(self.path)
except ImportError:
pass
try:
self._finder_cache[hook] = finder
except TypeError:
# The hook is unhashable for some reason so we don't bother
# caching it
pass
if finder is not None:
spec = finder.find_spec(fullname, target)
if (spec is not None and
(spec.loader is not None or idx == last)):
# If no __init__.<suffix> was found by any Finder,
# we may be importing a namespace package (which
# FileFinder.find_spec returns in this case). But we
# only want to return the namespace ModuleSpec if we've
# exhausted every other finder first.
return spec
# Module spec not found through any of the finders
return None
def invalidate_caches(self):
for finder in self._finder_cache.values():
finder.invalidate_caches()
#classmethod
def install(cls, basepath=None):
"""
Install the MetaFileFinder in the front sys.path_hooks, so that
it can support any existing sys.path_hooks and any that might
be appended later.
If given, only support paths under and including basepath. In this
case it's not necessary to invalidate the entire
sys.path_importer_cache, but only any existing entries under basepath.
"""
if basepath is not None:
basepath = os.path.abspath(basepath)
hook = cls.hook(basepath)
sys.path_hooks.insert(0, hook)
if basepath is None:
sys.path_importer_cache.clear()
else:
for path in list(sys.path_importer_cache):
if hook.handles(path):
del sys.path_importer_cache[path]
This is still, depressing, far more complication than should be necessary. I feel like on Python 2, before the import system rewrite, it was much simpler to do this since less of the support for the built-in module types (.py, etc.) was built on top of the import hooks themselves, so it was harder to break importing normal modules by adding hooks to import new modules types. I'm going to start a discussion on python-ideas to see if there's any way we can't improve this situation.
I came up with yet an alternative tweak. I won't say it is beautiful as it does a closure on an already existing one, but at least short :)
It adds loaders to the default FileLoader objects through a new hook. The original path_hook_for_FileFinder is wrapped in a closure and the loaders are injected into the FileFinder objects returned by the original hook.
After the new hook added the path_importer_cache is cleared as that is already filled with the original FileFinder objects. Those could also be updated dynamically, but I did not bother for now.
Disclaimer: not extensively tested yet. It does what I need in the easiest possible way I know, but the import system is complicated enough to produce funny side-effects for a tweak like this.
import sys
import importlib.machinery
def extend_path_hook_for_FileFinder(*loader_details):
orig_hook, orig_pos = None, None
for i, hook in enumerate(sys.path_hooks):
if hook.__name__ == 'path_hook_for_FileFinder':
orig_hook, orig_pos = hook, i
break
sys.path_hooks.remove(orig_hook)
def extended_path_hook_for_FileFinder(path):
orig_finder = orig_hook(path)
loaders = []
for loader, suffixes in loader_details:
loaders.extend((suffix, loader) for suffix in suffixes)
orig_finder._loaders.extend(loaders)
return orig_finder
sys.path_hooks.insert(orig_pos, extended_path_hook_for_FileFinder)
MY_SUFFIXES = ['.pymy']
class MySourceFileLoader(importlib.machinery.SourceFileLoader):
pass
loader_detail = (MySourceFileLoader, MY_SUFFIXES)
extend_path_hook_for_FileFinder(loader_detail)
# empty cache as it is already filled with simple FileFinder
# objects for the most common path elements
sys.path_importer_cache.clear()
sys.path_importer_cache.invalidate_caches()
ContextManager is really useful and it's also make our code more readable, but it seems it only works if the given function is meant to be a context manager, otherwise it will fail (no __exit__ or something else). I' wondering if we can use any function including those in libraries such as Django as a contextmanager, suppose the given code
self.assertEqual(Transaction.objects.filter(account=a, date=b, year=c).count(), 10)
self.assertEqual(Transaction.objects.filter(account=e, date=f, year=g).count(), 15)
self.assertEqual(Transaction.objects.filter(account=h, date=i, year=j).count(), 20)
Can be transformed into:
with Transaction.objects.filter as f:
self.assertEqual(f(account=a, date=b, year=c).count(), 10)
self.assertEqual(f(account=d, date=d, year=e).count(), 15)
self.assertEqual(f(account=h, date=i, yearj).count(), 20)
The way I look at it is that the one below is much readable, cleaner and less verbose. Is this possible?
Just do:
f = Transaction.objects.filter
before your statements!
If you really wanted to use a context manager, you could write one that does what you want:
from contextlib import contextmanager
#contextmanager
def alias(func):
yield func
with alias(Transaction.objects.filter) as f:
...
Note that, however you do this, without an explicit del statement, f will still be around after the with.
You're misunderstanding how the python with works (probably due to experience of Django Template's {% with %} block).
As mentioned in PEP 343, the with statement allows you to easily abstract try/finally blocks. This is really useful for things such as IO, where, if something goes wrong, you want to make sure that you safely close the file no matter what.
In your example, you're really looking to just reduce the length of the variable, which is a nested child of both Transaction and Transaction.objects. In that case you can simple do.
fn = Transaction.objects.filter
Your follow-up question of "how do I invalidate the usage of fn... after we go out of the scope" is also a little bit mis-guided, as common usages of with don't necessarily destroy the variable's reference to the original object.
>>> with open("README.md") as f:
... print f
...
<open file 'README.md', mode 'r' at 0x0055F860>
>>> f
<closed file 'README.md', mode 'r' at 0x0055F860>
>>>
To be honest, I'd suggest that you look for an alternative solution, as I think the semantics here are a little off. That said, if you truly want to do this, and mimic the unassignment as well, you'd need something like this:
class AliasContextManager(object):
"""
Handle temporary function scope within a with block.
"""
def __init__(self, fn):
self.fn = fn
def proxy(self):
"""
Create a proxy to our function, such that we can remove the reference
on exit, and replace it with None.
"""
def _proxy(*args, **kwargs):
fn = getattr(self, "fn", None)
return fn(*args, **kwargs)
return _proxy
def __enter__(self):
return self.proxy()
def __exit__(self, *args):
del self.fn
alias = AliasContextManager
And here's how it can be used:
>>> with alias(sum) as fn:
... print fn([1,2,3])
... print fn([4,5,6])
6
15
>>> print fn([7,8,9])
Traceback (most recent call last):
File "x.py", line 22, in <module>
print fn([7,8,9])
File "x.py", line 8, in _proxy
return fn(*args, **kwargs)
TypeError: 'NoneType' object is not callable
I am using the cmd.Cmd class in Python to offer a simple readline interface to my program.
Self contained example:
from cmd import Cmd
class CommandParser(Cmd):
def do_x(self, line):
pass
def do_xy(self, line):
pass
def do_xyz(self, line):
pass
if __name__ == "__main__":
parser = CommandParser()
parser.cmdloop()
Pressing tab twice will show possibilities. Pressing tab again does the same.
My question is, how do I get the options to cycle on the third tab press? In readline terms I think this is called Tab: menu-complete, but I can't see how to apply this to a Cmd instance.
I already tried:
readline.parse_and_bind('Tab: menu-complete')
Both before and after instantiating the parser instance. No luck.
I also tried passing "Tab: menu-complete" to the Cmd constructor. No Luck here either.
Anyone know how it's done?
Cheers!
The easiest trick would be to add a space after menu-complete:
parser = CommandParser(completekey="tab: menu-complete ")
The bind expression that is executed
readline.parse_and_bind(self.completekey+": complete")
will then become
readline.parse_and_bind("tab: menu-complete : complete")
Everything after the second space is acutally ignored, so it's the same as tab: menu-complete.
If you don't want to rely on that behaviour of readline parsing (I haven't seen it documented) you could use a subclass of str that refuses to be extended as completekey:
class stubborn_str(str):
def __add__(self, other):
return self
parser = CommandParser(completekey=stubborn_str("tab: menu-complete"))
self.completekey+": complete" is now the same as self.completekey.
Unfortunately, it seems as though the only way around it is to monkey-patch the method cmdloop from the cmd.Cmd class, or roll your own.
The right approach is to use "Tab: menu-complete", but that's overriden by the class as shown in line 115: readline.parse_and_bind(self.completekey+": complete"), it is never activated. (For line 115, and the entire cmd package, see this: https://hg.python.org/cpython/file/2.7/Lib/cmd.py). I've shown an edited version of that function below, and how to use it:
import cmd
# note: taken from Python's library: https://hg.python.org/cpython/file/2.7/Lib/cmd.py
def cmdloop(self, intro=None):
"""Repeatedly issue a prompt, accept input, parse an initial prefix
off the received input, and dispatch to action methods, passing them
the remainder of the line as argument.
"""
self.preloop()
if self.use_rawinput and self.completekey:
try:
import readline
self.old_completer = readline.get_completer()
readline.set_completer(self.complete)
readline.parse_and_bind(self.completekey+": menu-complete") # <---
except ImportError:
pass
try:
if intro is not None:
self.intro = intro
if self.intro:
self.stdout.write(str(self.intro)+"\n")
stop = None
while not stop:
if self.cmdqueue:
line = self.cmdqueue.pop(0)
else:
if self.use_rawinput:
try:
line = raw_input(self.prompt)
except EOFError:
line = 'EOF'
else:
self.stdout.write(self.prompt)
self.stdout.flush()
line = self.stdin.readline()
if not len(line):
line = 'EOF'
else:
line = line.rstrip('\r\n')
line = self.precmd(line)
stop = self.onecmd(line)
stop = self.postcmd(stop, line)
self.postloop()
finally:
if self.use_rawinput and self.completekey:
try:
import readline
readline.set_completer(self.old_completer)
except ImportError:
pass
# monkey-patch - make sure this is done before any sort of inheritance is used!
cmd.Cmd.cmdloop = cmdloop
# inheritance of the class with the active monkey-patched `cmdloop`
class MyCmd(cmd.Cmd):
pass
Once you've monkey-patched the class method, (or implemented your own class), it provides the correct behavior (albeit without highlighting and reverse-tabbing, but these can be implemented with other keys as necessary).