Develop Reference

How to assign a different process for each object's instance ? Python

I am currently working on a Python driver which wraps a C code used to acquire data from a phasemeter.
It is currently working fine with 1 phasemeter however I need to acquire data from 4 phasemeter at the same time. As there is a huge flow a data for each of them I came up with the idea of running them on different process. So far I understand running programs on different processes use different memory parts and will help to avoid mixing data.
So, my idea would be to create an instance of Phasemeter object for each phasemeter and assign a process for each instance.
I already tried to use multiprocessing or Taskset however the first one is used for functions (as far as I understood) and the second one only assign programs and not instances (also as far as I understood).
I am looking for any advice or hints/keywords on my Python code or to find a solution.
Thank you for your time and help.
Python Driver Code :
#%% Module import
import signal
import ctypes
import os
import sys
import json
#%% Constant definition.
# Opening config File
PATH_TO_CONFIG_FILE="config/config.json"
ConfigFile = open(PATH_TO_CONFIG_FILE, "r")
jsonContent = ConfigFile.read()
Config = json.loads(jsonContent)
# Path to configuration File
PATH_TO_CFG_FILE=Config['PATH_TO_CFG_FILE']
# Path to log File
PATH_TO_LOG_FILE=Config['PATH_TO_LOG_FILE']
# Path to Shared Object File
PATH_TO_SO_FILE=Config["PATH_TO_SO_FILE"]
SO_NAMEFILE=Config["SO_NAMEFILE"]
# Wrapper's name
PROGRAM_NAME=str(__file__)
#%% Class definiton
class Phasemeter :
""" This class is a driver for XXXX Phasemeters data acquisiton """
def __init__(self) :
""" This method define path, enables interuptions of C code and defines C code signature
TODO : Paralelization
"""
# Defining path
self.path=os.path.abspath(os.path.join(os.getcwd(),PATH_TO_SO_FILE))
self.libname = SO_NAMEFILE
self.LIBC = ctypes.CDLL(os.path.join(self.path,self.libname))
# Configuring signal to enable interuption of C_Code
signal.signal(signal.SIGINT,signal.SIG_DFL)
# Settings C library's signature datatype
self.LIBC.lisaf_phasemeter_main.argtypes= [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p),]
self.LIBC.lisaf_phasemeter_main.restypes =[ctypes.c_int,]
def start(self,*args,**kwargs):
""" This method starts acquisition for the Phasemeter. It handles options and launches the Phasemeter's C code for acquisiiton.
-------------------------------------------------------------------------------------------------------------------------------------------------------------------
Optional arguments | fsampling : Sampling frequency (default is 0.0) -> my_phasemeter.start(fsampling=#value)
| vcal : Calibration Voltage -> my_phasemeter.start(vcal=#value)
| trun : Specify acquisition time (default is 0.0 and -1 is infinite) -> my_phasemeter.start(trun=#value)
| intclk : Call internal clock (unstable) -> my_phasemeter.start("intclk")
| comment : Add a comment to the data file's header -> my_phasemeter.start(comment="#your_comment")
| erase : Erase previous Data file without asking permission (default is off) -> my_phasemeter.start("erase")
| help : Call c code's help (not very usefull to use the wrapper) -> my_phasemeter.start("help")
-------------------------------------------------------------------------------------------------------------------------------------------------------------------
Method call examples : my_phasemeter.start()
: my_phasemeter.start("erase",fsampling=100,trun=50,comment="Test")
: my_phasemeter.start("help")
: my_phasemeter.start("intclk","erase",comment="Test",fsampling=100,trun=100,vcal=2)
-------------------------------------------------------------------------------------------------------------------------------------------------------------------
"""
# Robustness
for argument in args :
if type(argument) is not str :
sys.tracebacklimit = 0
raise TypeError("Wrong type for argument ",argument, ", only strings are authorized")
for argument in kwargs :
if argument in ["fsampling","vcal"] and int(kwargs[argument]) < 0 :
raise ValueError('Please enter a positive number for keyword ',argument)
if argument == "comment" and type(kwargs[argument]) is not str :
raise TypeError('Please enter a string for comment keyword, for e.g. : comment="Test"')
if argument == "trun" and int(kwargs[argument]) < -1 :
raise ValueError('Please enter a positive number for keyword trun or -1 for infinite acquisition time')
# Changing data structure to fit Marshalling
# Here we want to mimic C code's argv
args_list=[]
args_list.append(PROGRAM_NAME)
args_list.append(PATH_TO_CFG_FILE)
args_list.append(PATH_TO_LOG_FILE)
# Handling optionnal arguments
for argument in args :
if argument == "erase" or "help" or "intclk" :
args_list.append(str("--"+str(argument)))
for argument in kwargs :
if argument == "fsampling" or "vcal" or "comment" or "trun" :
args_list.append(str("--"+str(argument)))
args_list.append(str(kwargs[argument]))
args_list=[*args_list]
args=args_list
# Debug
print("Args : ",*args)
# Marshalling data for C library
self.c_char_pointer_array = ctypes.c_char_p * len(args)
self.encoded_args_list= [str.encode(str(i)) for i in args ]
self.arguments = self.c_char_pointer_array (*self.encoded_args_list)
# Calling C library
self.status = self.LIBC.lisaf_phasemeter_main(len(self.arguments),self.arguments,None)
Edit : I tried to write a little program to create each class's on different CPU or at least different process however I don't understand the result of it.
Python program to create each instance on different process :
import multiprocessing
import os
class Dog() :
def __init__ (self,name) :
self.name = name
Dog.info(self)
def info(self):
self.pid=os.getpid()
self.ppid=os.getppid()
print("Name : ",self.name)
print(" PID : ", self.pid)
print(" PPID : ", self.ppid)
if __name__ == '__main__' :
print("Variable creation")
with multiprocessing.Pool(4) as Process :
Max,Leon,Bob,Leonard = Process.map(Dog,["Max","Leon","Bob","Leonard"])
print("End of variable creation")
print("Displaying variable info")
print()
print('Max info: ')
print(Max.info())
print()
print('Leon info: ')
print(Leon.info())
print()
print('Bob info: ')
print(Bob.info())
print()
print('Leonard info: ')
print(Leonard.info())
Output :
Variable creation
Name : Max
PID : 5717
PPID : 5716
Name : Leon
PID : 5718
PPID : 5716
Name : Leonard
Name : Bob
PID : 5719
PPID : 5716
PID : 5717
PPID : 5716
End of variable creation
Displaying variable info
Max info:
Name : Max
PID : 5716
PPID : 5126
None
Leon info:
Name : Leon
PID : 5716
PPID : 5126
None
Bob info:
Name : Bob
PID : 5716
PPID : 5126
None
Leonard info:
Name : Leonard
PID : 5716
PPID : 5126
None
Why do they have a different process at declaration and when I call it again ?

Antlr 4 in python not working as expected (trying to parse chapter and paragraph of a book)

I want to create a very simple ANTLR4 parser (in Python) without listener and visitor, which takes as input the chapter and paragraph of a book in any order, and returns the high_level (chapter) and low_level (paragraph) of the entry, e.g. if I enter 2 a or a 2 it should print "chapter 2, paragraph a".
Here is my Example.g4
grammar Example;
text
: paragraph ;
paragraph
: high_level (WS low_level)?
| low_level WS high_level
;
low_level
: 'a' | 'b' | 'c' ;
high_level
: '1' | '2' | '3';
WS : [ \t\r\n]+ ;
I do this in my terminal
java -jar ~/antlr-4.8-complete.jar -Dlanguage=Python3 -no-listener -no-visitor Example.g4
which generates two python files, and then I wrote the following python script
from antlr4 import *
from ExampleLexer import ExampleLexer
from ExampleParser import ExampleParser
def main():
while True:
text = InputStream(input(">"))
lexer = ExampleLexer(text)
stream = CommonTokenStream(lexer)
parser = ExampleParser(stream)
tree = parser.text()
query = tree.paragraph()
low_level = query.low_level()
high_level = query.high_level()
print(f"chapter {high_level}, paragraph {low_level}")
if __name__ == '__main__':
main()
However, if I then run it and input 2 a, I get this
chapter [10 8], paragraph [12 8]
Can anyone explain what I'm doing wrong please? I don't understand the numbers in square brackets.

It is just some debugging information displayed by the RuleContext (from which your generated Low_levelContext and High_levelContext classes extend). In your case, the rule's invokingState and parentCtx are displayed.
Have a look at the source:
class RuleContext(RuleNode):
...
def __str__(self):
return self.toString(None, None)
...
def toString(self, ruleNames:list, stop:RuleContext)->str:
with StringIO() as buf:
p = self
buf.write("[")
while p is not None and p is not stop:
if ruleNames is None:
if not p.isEmpty():
buf.write(str(p.invokingState))
else:
ri = p.getRuleIndex()
ruleName = ruleNames[ri] if ri >= 0 and ri < len(ruleNames) else str(ri)
buf.write(ruleName)
if p.parentCtx is not None and (ruleNames is not None or not p.parentCtx.isEmpty()):
buf.write(" ")
p = p.parentCtx
buf.write("]")
return buf.getvalue()
...
https://github.com/antlr/antlr4/blob/master/runtime/Python3/src/antlr4/RuleContext.py
You didn't explain what you wanted to display, but I guess the text the rules matched, in which case you can do this instead:
print(f"chapter {high_level.getText()}, paragraph {low_level.getText()}")

Call python function with arguments and get returned value in autohotkey

I have a python script called "server.py" and inside it I have a function def calcFunction(arg1): ... return output How can I call the function calcFunction with arguments and use the return value in autohotkey? This is what I want to do in autohotkey:
ToSend = someString ; a string
output = Run server.py, calcFunction(ToSend) ; get the returned value from the function with ToSend as argument
Send, output ; use the returned value in autohotkey
I have looked online but nothing seems to fully answer my question. Can it even be done?

In order to send your parameters to Python, you could use arguments from within your Python script. You can do this with the sys library:
import sys
print(sys.argv[0]) # name of file
print(sys.argv[1]) # first argument
print(sys.argv[2]) # second argument...
From within your AutoHotKey script, you can send parameters to the Python script by adding them as arguments right after specifying the file name:
RunWait, server.py "This will be printed as the first argument!" "This is the second!"
Then, to get the output of the function back to AHK, you could use sys again by utilizing it's exit() function:
sys.exit(EXIT_NUMBER)
And back in AHK, you recieve the EXIT_NUMBER inside the variable ErrorLevel.
Put all together, your code should look something like this:
; AHK
RunWait, server.py "%ToSend%"
# Python
sys.exit(calcFunction(sys.argv[1]))
; AHK
MsgBox %ErrorLevel%

using python COM server, ahk can really calls python functions. directly.
you use it like this: MsgBox % pythonComServer.toUppercase("hello world")
simple example: return uppercased string
use the python part from How to program hotstrings in python like in autohotkey and use this for ahk part:
call python function uppercase.ahk
#NoEnv ; Recommended for performance and compatibility with future AutoHotkey releases.
#SingleInstance, force
SendMode Input ; Recommended for new scripts due to its superior speed and reliability.
SetWorkingDir %A_ScriptDir% ; Ensures a consistent starting directory.
SetBatchLines, -1
#KeyHistory 0
ListLines Off
pythonComServer:=ComObjCreate("Python.stringUppercaser")
;or
; pythonComServer:=ComObjCreate("{C70F3BF7-2947-4F87-B31E-9F5B8B13D24F}") ;use your own CLSID
MsgBox % pythonComServer.toUppercase("hello world")
Exitapp
f3::Exitapp
customized version: (math) use SymPy to simplify Expression
read this first to understand: How to program hotstrings in python like in autohotkey
sympy com server.py
from sympy import simplify, Number, N
from sympy.parsing.sympy_parser import standard_transformations, implicit_multiplication_application, convert_xor
from sympy.parsing.sympy_parser import parse_expr
from decimal import Decimal
from winsound import MessageBeep
transformations = standard_transformations + (implicit_multiplication_application, convert_xor)
def removeTrailingZerosFromNum(num):
dec = Decimal(str(num))
tup = dec.as_tuple()
delta = len(tup.digits) + tup.exponent
digits = ''.join(str(d) for d in tup.digits)
if delta <= 0:
zeros = abs(tup.exponent) - len(tup.digits)
val = '0.' + ('0' * zeros) + digits
else:
val = digits[:delta] + ('0' * tup.exponent) + '.' + digits[delta:]
val = val.rstrip('0')
if val[-1] == '.':
val = val[:-1]
if tup.sign:
return '-' + val
return val
def removeTrailingZerosFromExpr(operatorObject):
if operatorObject.args:
return type(operatorObject)(*[removeTrailingZerosFromExpr(i) for i in operatorObject.args])
else:
try:
return Number(removeTrailingZerosFromNum(operatorObject))
except:
return operatorObject
def removeTrailingZerosFromExprOrNumber(operatorObject):
try:
return removeTrailingZerosFromNum(operatorObject)
except:
return removeTrailingZerosFromExpr(operatorObject)
class BasicServer:
# list of all method names exposed to COM
_public_methods_ = ["parExprN"]
#staticmethod
def parExprN(clipBak):
parsed = parse_expr(clipBak, transformations=transformations)
simplified = simplify(N(parsed))
finalStr = str(removeTrailingZerosFromExprOrNumber(simplified))
MessageBeep(-1)
return finalStr.replace("**", "^")
if __name__ == "__main__":
import sys
if len(sys.argv) < 2:
print("Error: need to supply arg (""--register"" or ""--unregister"")")
sys.exit(1)
else:
import win32com.server.register
import win32com.server.exception
# this server's CLSID
# NEVER copy the following ID
# Use "print(pythoncom.CreateGuid())" to make a new one.
myClsid="{4530C817-6C66-46C8-8FB0-E606970A8DF6}"
# this server's (user-friendly) program ID, can be anything you want
myProgID="Python.SimplifyExpr"
import ctypes
def make_sure_is_admin():
try:
if ctypes.windll.shell32.IsUserAnAdmin():
return
except:
pass
exit("YOU MUST RUN THIS AS ADMIN")
if sys.argv[1] == "--register":
make_sure_is_admin()
import pythoncom
import os.path
realPath = os.path.realpath(__file__)
dirName = os.path.dirname(realPath)
nameOfThisFile = os.path.basename(realPath)
nameNoExt = os.path.splitext(nameOfThisFile)[0]
# stuff will be written here
# HKEY_LOCAL_MACHINE\SOFTWARE\Classes\CLSID\${myClsid}
# HKEY_LOCAL_MACHINE\SOFTWARE\Classes\CLSID\{c2467d33-71c5-4057-977c-e847c2286882}
# and here
# HKEY_LOCAL_MACHINE\SOFTWARE\Classes\${myProgID}
# HKEY_LOCAL_MACHINE\SOFTWARE\Classes\Python.SimplifyExpr
win32com.server.register.RegisterServer(
clsid=myClsid,
# I guess this is {fileNameNoExt}.{className}
pythonInstString=nameNoExt + ".BasicServer", #sympy com server.BasicServer
progID=myProgID,
# optional description
desc="(math) SymPy simplify Expression",
#we only want the registry key LocalServer32
#we DO NOT WANT InProcServer32: pythoncom39.dll, NO NO NO
clsctx=pythoncom.CLSCTX_LOCAL_SERVER,
#this is needed if this file isn't in PYTHONPATH: it tells regedit which directory this file is located
#this will write HKEY_LOCAL_MACHINE\SOFTWARE\Classes\CLSID\{4530C817-6C66-46C8-8FB0-E606970A8DF6}\PythonCOMPath : dirName
addnPath=dirName,
)
print("Registered COM server.")
# don't use UseCommandLine(), as it will write InProcServer32: pythoncom39.dll
# win32com.server.register.UseCommandLine(BasicServer)
elif sys.argv[1] == "--unregister":
make_sure_is_admin()
print("Starting to unregister...")
win32com.server.register.UnregisterServer(myClsid, myProgID)
print("Unregistered COM server.")
else:
print("Error: arg not recognized")
to register:
python "sympy com server.py" --register
sympy com client.ahk
#NoEnv ; Recommended for performance and compatibility with future AutoHotkey releases.
#SingleInstance, force
SendMode Input ; Recommended for new scripts due to its superior speed and reliability.
SetWorkingDir %A_ScriptDir% ; Ensures a consistent starting directory.
SetBatchLines, -1
#KeyHistory 0
ListLines Off
sympyComServer:=ComObjCreate("Python.SimplifyExpr")
;or
; pythonComServer:=ComObjCreate("{4530C817-6C66-46C8-8FB0-E606970A8DF6}") ;use your own CLSID
; clipboard:=sympyComServer.parExprN("1+3*7")
clipboard:=sympyComServer.parExprN("1/3 + 1/2")
$#s::
clipboard:=sympyComServer.parExprN(clipboard)
return
f3::Exitapp

Get unix file type with Python os module

I would like to get the unix file type of a file specified by path (find out whether it is a regular file, a named pipe, a block device, ...)
I found in the docs os.stat(path).st_type but in Python 3.6, this seems not to work.
Another approach is to use os.DirEntry objects (e. g. by os.listdir(path)), but there are only methods is_dir(), is_file() and is_symlink().
Any ideas how to do it?

You use the stat module to interpret the result of os.stat(path).st_mode.
>>> import os
>>> import stat
>>> stat.S_ISDIR(os.stat('/dev/null').st_mode)
False
>>> stat.S_ISCHR(os.stat('/dev/null').st_mode)
True
You can make a general function to return the determined type. This works for both Python 2 and 3.
import enum
import os
import stat
class PathType(enum.Enum):
dir = 0 # directory
chr = 1 # character special device file
blk = 2 # block special device file
reg = 3 # regular file
fifo = 4 # FIFO (named pipe)
lnk = 5 # symbolic link
sock = 6 # socket
door = 7 # door (Py 3.4+)
port = 8 # event port (Py 3.4+)
wht = 9 # whiteout (Py 3.4+)
unknown = 10
#classmethod
def get(cls, path):
if not isinstance(path, int):
path = os.stat(path).st_mode
for path_type in cls:
method = getattr(stat, 'S_IS' + path_type.name.upper())
if method and method(path):
return path_type
return cls.unknown
PathType.__new__ = (lambda cls, path: cls.get(path))
>>> PathType('/dev/null')
<PathType.chr: 1>
>>> PathType('/home')
<PathType.dir: 0>

Python 3.6 has pathlib and its Path objects have methods:
is_dir()
is_file()
is_symlink()
is_socket()
is_fifo()
is_block_device()
is_char_device()
pathlib takes a bit to get used to (at least for me having come to Python from C/C++ on Unix), but it is a nice library

20 November 2020
The code supplied in the answer by 'Artyer' does not work in python 2.7.17, but appears to work in python 3.6.9 except for one detail:
The call:
path = os.stat(path).st_mode
should be:
path = os.lstat(path).st_mode
Otherwise you just get the regular file that a soft link points to.
For python 2 (and 3), if you wish to follow this style of coding, the following is better, with some name changes for clarity:
import enum,os,stat
class PathTypes(enum.Enum):
door = 0 # door (Py 3.4+)
port = 1 # event port (Py 3.4+)
wht = 2 # whiteout (Py 3.4+)
dir = 3 # directory
chr = 4 # character special device file
blk = 5 # block special device file
reg = 6 # regular file
fifo = 7 # FIFO (named pipe)
lnk = 8 # symbolic link
sock = 9 # socket
unimplemented = 10
def filetype(path):
mode=os.lstat(path).st_mode # do not follow links
for t in PathTypes:
try: func=getattr(stat, 'S_IS' + t.name.upper())
except AttributeError: continue
if func(mode): return t
return PathTypes["unimplemented"]
Notice the reordering that forces testing in python 2 to consider undefined stat functions and exercise the necessary try...except statement. The members are also renumbered since in python 2, enum.ENUM apparently sorts the members by value, and this is apparently undocumented. Since python 2 is now not supported, bug or not, that is the way it is.
The enum documentation recommends against using value 0 since that is boolean False and all members are supposed to be boolean True. The documentation also recommends using this simpler helper function style.
(https://cpython-test-docs.readthedocs.io/en/latest/library/enum.html)
To avoid some of these ambiguities the following will behave as documented, and also orders processing to return the most likely results quickly:
import enum,os,stat
members= ( \
('reg', 'S_ISREG' ), \
('dir', 'S_ISDIR' ), \
('lnk', 'S_ISLNK' ), \
('fifo', 'S_ISFIFO'), \
('sock', 'S_ISSOCK'), \
('chr', 'S_ISCHR' ), \
('blk', 'S_ISBLK' ), \
('door', 'S_ISDOOR'), \
('port', 'S_ISPORT'), \
('wht', 'S_ISWHT' ), \
('unimplemented', '') \
)
FileTypes=enum.Enum('FileTypes',members)
def filetype(path):
"""Get unix filetype:
reg,dir,lnk,fifo,sock,chr,blk
and for py3.4+:
door,port,wht.
'path' is a full pathname for some file."""
mode=os.lstat(path).st_mode # do not follow links
for t in FileTypes:
try: func=getattr(stat, t.value)
except AttributeError: continue
if func(mode): return t
return FileTypes["unimplemented"]
The enum.ENUM functional API apparently does not have the sorting bug and keeps the members in the order they are presented, as documented.
Given the troublesome nature of enum.ENUM, it is probably better simply to avoid it by using time
tested python primitives:
import os,stat
_stat_funcs=( \\
'S_ISREG','S_ISDIR','S_ISLNK','S_ISFIFO','S_ISSOCK', \\
'S_ISCHR','S_ISBLK','S_ISDOOR','S_ISPORT','S_ISWHT' )
# ^----- python 3.4+ only ----->|
# ^-- SOLARIS only --->|^--BSD->|
_ls_chr=( \\
'-' ,'d' ,'l' ,'p' ,'s' , \\
'c' ,'b' ,'D' ,'P' ,'w' )
# ^----- python 3.4+ only------>|
# ^-- SOLARIS only --->|^--BSD->|
_ftypes=tuple( (c,getattr(stat,f)) \\
for c,f in zip(_ls_chr,_stat_funcs) if f in dir(stat))
def filetype(path):
"""Get unix filetype designator used in 'ls' shell command listings:
reg('-'),dir('d'),lnk('l'),fifo('p'),sock('s'),chr('c'),blk('b')
and for py3.4+:
door('D'),port('P'),wht('w'). (solaris,solaris,BSD only)
'path' is a full pathname for some file. Returns 'u' for an
unknown or unimplemented filetype."""
mode=os.lstat(path).st_mode # do not follow links
for c,func in _ftypes:
if func(mode): return c
return 'u'
This is a lot more efficient than the code 'Artyer' offers, which is important when processing large numbers of files. Usage for both python2 and 3:
>>> filetype('/dev/null')
'c'
>>> filetype('/dev/sda')
'b'
>>> filetype('/home/test')
'd'
>>> filetype('/home/random.txt')
'-'
>>> filetype('/home/test/hlnk') # hard link
'-'
>>> filetype('/home/test/slnk') # soft link
'l'
>>> filetype('/home/test/sckt')
's'
>>> filetype('/home/test/fifo.pipe')
'p'
>>>

Ipython customize prompt to display cell run time

i am wondering how to configure Ipython so that it adds the run time of the last command in milliseconds/seconds to the right command prompt. This could be done in ZSH/Bash shells as illustrated here https://coderwall.com/p/kmchbw
How should I go about doing this?

This is a code snippet that times each statement and prints it right adjusted before the next prompt, and also makes the value accessible by name 'texc'.
# Assumes from __future__ import print_function
from time import time
import blessings # Not a necessary requirement
class ExecTimer(object):
def __init__(self, ip):
self.shell = ip
self.t_pre = time()
self.texc = 0
self.prev_texc = 0
self.term = blessings.Terminal()
def pre_execute(self):
self.t_pre = time()
def post_execute(self):
self.prev_texc = self.texc
self.texc = round(time() - self.t_pre, 4)
print(self.term.bold_blue(
'{} s'.format(self.texc).rjust(self.term.width - 1)
))
# Only add or update user namespace var if it is safe to do so
if 'texc' not in self.shell.user_ns or \
self.shell.user_ns['texc'] == self.prev_texc:
self.shell.push({'texc': self.texc})
else:
pass
def register(self):
self.shell.events.register('pre_execute', self.pre_execute)
self.shell.events.register('post_execute', self.post_execute)
ExecTimer(get_ipython()).register()
To print it above the in-prompt instead, remove the print, and in ipython_config.py set:
c.PromptManager.in_template = '{texc} s\nIn[\\#]: '
or in the same file (startup.py) use
get_ipython().run_line_magic(
'config',
r"PromptManager.in_template = '{texc} s\nIn[\\#]: '"
)

For those who are interested, please refer to this issue opened in Github.
https://github.com/ipython/ipython/issues/5237