Develop Reference

Check which optional parameters are supplied to a function call

My goal is to run through all the *.py files in a directory and look at each call to a specific function test_func. This function has some optional parameters and I need to audit when the function is called with the optional parameters. My thought is to use the ast library (specifically ast.walk()).
I suppose this is a static analysis problem.
# function definition
def test_func(
name: str,
*,
user: Optional['User'] = None,
request: Optional[WebRequest] = None,
**kwargs
) -> bool:
pass
# somewhere in another file ...
test_func('name0')
test_func('name1', request=request)
test_func('name1')
test_func('name2', user=user)
# figure out something like below:
# name0 is never given any optional parameters
# name1 is sometimes given request
# name2 is always given user

Here is a POC :
import typing
from typing import Optional
class User: pass
class WebRequest: pass
# function definition
def test_func(
name: str,
*,
user: Optional['User'] = None,
request: Optional[WebRequest] = None,
**kwargs
) -> bool:
pass
# somewhere in another file ...
test_func('name0')
test_func('name1', request=WebRequest())
test_func('name1')
test_func('name2', user=User())
# figure out something like below:
# name0 is never given any optional parameters
# name1 is sometimes given request
# name2 is always given user
with open(__file__, "rt") as py_file:
py_code = py_file.read()
import collections
each_call_kwargs_names_by_arg0_value: typing.Dict[str, typing.List[typing.Tuple[str, ...]]] = collections.defaultdict(list)
import ast
tree = ast.parse(py_code)
for node in ast.walk(tree):
if isinstance(node, ast.Call):
if hasattr(node.func, "id"):
name = node.func.id
elif hasattr(node.func, "attr"):
name = node.func.attr
elif hasattr(node.func, "value"):
name = node.func.value.id
else:
raise NotImplementedError
print(name)
if name == "test_func":
arg0_value = typing.cast(ast.Str, node.args[0]).s
each_call_kwargs_names_by_arg0_value[arg0_value].append(
tuple(keyword.arg for keyword in node.keywords)
)
for arg0_value, each_call_kwargs_names in each_call_kwargs_names_by_arg0_value.items():
frequency = "NEVER" if all(len(call_args) == 0 for call_args in each_call_kwargs_names) else \
"ALWAYS" if all(len(call_args) != 0 for call_args in each_call_kwargs_names) else \
"SOMETIMES"
print(f"{arg0_value!r} {frequency}: {each_call_kwargs_names}")
# Output :
# 'name0' NEVER: [()]
# 'name1' SOMETIMES: [('request',), ()]
# 'name2' ALWAYS: [('user',)]

You can use a recursive generator function to traverse an ast of your Python code:
import ast
def get_calls(d, f = ['test_func']):
if isinstance(d, ast.Call) and d.func.id in f:
yield None if not d.args else d.args[0].value, [i.arg for i in d.keywords]
for i in getattr(d, '_fields', []):
vals = (m if isinstance((m:=getattr(d, i)), list) else [m])
yield from [j for k in vals for j in get_calls(k, f = f)]
Putting it all together:
import os, collections
d = collections.defaultdict(list)
for f in os.listdir(os.getcwd()):
if f.endswith('.py'):
with open(f) as f:
for a, b in get_calls(ast.parse(f.read())):
d[a].append(b)
r = {a:{'verdict':'never' if not any(b) else 'always' if all(b) else 'sometimes', 'params':[i[0] for i in b if i]}
for a, b in d.items()}
Output:
{'name0': {'verdict': 'never', 'params': []},
'name1': {'verdict': 'sometimes', 'params': ['request']},
'name2': {'verdict': 'always', 'params': ['user']}}

#rdflib (python): how to get a URIRef from a string such as 'ns:xxx'?

I have a RDFlib graph g, whose NameSpaceManager is aware of some namespaces.
How do I get a URIRef from a string such as 'ns:xxx', where ns is the prefix associated to a namespace known by g.namespace_manager? Basically, I'm looking for a method which does the inverse operation of URIRef's n3(g.namespace_manager). I'm pretty confident that there is a way to do it, as a similar function is needed to parse turtle files, or sparql queries, but I can't find it. Otherwise of course, it must not be very difficult to write it.
TIA

from rdflib import Graph, Namespace
from rdflib.namespace import RDF
g = Graph()
NS = Namespace("http://example.com/")
# then, say Xxx is a class and Aaa is an instance of Xxx...
g.add((NS.Aaa, RDF.type, NS.Xxx))
# so use NS.Xxx (or NS["Xxx"]) to get a URIRef of NS.Xxx from Namespace NS
print(type(NS)) # --> <class 'rdflib.term.URIRef'>
print(type(NS.Xxx)) # --> <class 'rdflib.term.URIRef'>
print(NS.Xxx) # --> "http://example.com/Xxx"
If you want to bind a prefix within a graph, you use the rdflib Graph class' bind() method so, for the code above, you would use:
g.bind("ns", NS)
Now the graph, if serialized with a format that knows about prefixes, like Turtle, will use "ns". The above data would be:
#prefix ns: <http://example.com/> .
#prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
ns:Aaa rdf:type ns:Xxx .
So, in Python, if you want to make the URI "http://example.com/Xxx"" from the string "ns:Xxx" you should have everything you need:
the namespace declaration: NS = Namespace("http://example.com/")
the prefix/namespace binding g.bind("ns", NS)
IFF, on the other hand, you didn't declare the namespace yourself but it's in a graph and you only have the short form URI "ns:Xxx", you can do this to list all bound prefixes & namespaces used in the graph:
for n in g.namespace_manager.namespaces():
print(n)
returns, for the data above:
('xml', rdflib.term.URIRef('http://www.w3.org/XML/1998/namespace'))
('rdf', rdflib.term.URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#'))
('rdfs', rdflib.term.URIRef('http://www.w3.org/2000/01/rdf-schema#'))
('xsd', rdflib.term.URIRef('http://www.w3.org/2001/XMLSchema#'))
('eg', rdflib.term.URIRef('http://example.com/'))
So, if you know "eg:Xxx", you can split off the "eg" part and make the URI you want like this:
print(
[str(x[1]) for x in g.namespace_manager.namespaces() if x[0] == s.split(":")[0]]
[0] + s.split(":")[1]
)
prints:
http://example.com/Xxx

from rdflib.namespace import NamespaceManager
import rdflib as rdflib
class MyNamespacesInfo:
def __init__(self, namespace_manager: NamespaceManager):
# as I don't know how to get the namespace from a prefix from the API
# I construct a dict
self.pref2ns = {}
for pref, ns in namespace_manager.namespaces():
self.pref2ns[pref] = ns
def uriref(self, n3uri: str) -> rdflib.URIRef:
# n3uri: either 'ns:xxx', '<http://..../xxx>' or 'http://..../xxx'
if n3uri[0] == '<':
if n3uri[len(n3uri)-1] == '>':
return rdflib.URIRef(n3uri[1:-1])
else:
raise ValueError("Illegal uri: ", n3uri)
else:
return self.prefixed_2_uriref(n3uri, laxist=True)
def prefixed_2_uriref(self, short_uri: str, laxist=True) -> rdflib.URIRef:
# param short_uri eg. 'ns:xxx', where ns declared in namespace_manager
# using laxist = True, you also can pass a long uri
s = short_uri.split(':')
if len(s) < 2:
if laxist:
return rdflib.URIRef(short_uri)
else:
raise ValueError('Not a prefix:localname string: ' + short_uri)
prefix = s[0]
ns = self.pref2ns.get(prefix)
if ns == None:
if laxist:
return rdflib.URIRef(short_uri)
else:
raise ValueError('Unknown prefix: ' + prefix)
else:
x = ns + s[1]
for i in range(2, len(s)):
x = x + (':' + s[i])
return x
# example of use:
g = rdflib.Graph()
g.parse('http://www.semanlink.net/tag/rdf_tools.rdf')
ns_info = MyNamespacesInfo(g.namespace_manager)
# all of following calls print http://www.semanlink.net/tag/rdflib
x = ns_info.uriref('tag:rdflib')
print(x)
x = ns_info.uriref('http://www.semanlink.net/tag/rdflib')
print(x)
x = ns_info.uriref('<http://www.semanlink.net/tag/rdflib>')
print(x)

Run python script to replace betacode with greek letters LaTeX

I want to convert the betacode in an existing .tex-File to normal greek letters.
For example: I want to replace:
\bcode{lo/gos}
with simple:
λόγος
And so on for all other glyphs. Fortunately there seems to be a python-script that is supposed to do just that. But, being completely inexperienced I simply don’t know how to run it.
Here is the code of the python sript:
# beta2unicode.py
#
# Version 2004-11-23
#
# James Tauber
# http://jtauber.com/
#
# You are free to redistribute this, but please inform me of any errors
#
# USAGE:
#
# trie = beta2unicodeTrie()
# beta = "LO/GOS\n";
# unicode, remainder = trie.convert(beta)
#
# - to get final sigma, string must end in \n
# - remainder will contain rest of beta if not all can be converted
class Trie:
def __init__(self):
self.root = [None, {}]
def add(self, key, value):
curr_node = self.root
for ch in key:
curr_node = curr_node[1].setdefault(ch, [None, {}])
curr_node[0] = value
def find(self, key):
curr_node = self.root
for ch in key:
try:
curr_node = curr_node[1][ch]
except KeyError:
return None
return curr_node[0]
def findp(self, key):
curr_node = self.root
remainder = key
for ch in key:
try:
curr_node = curr_node[1][ch]
except KeyError:
return (curr_node[0], remainder)
remainder = remainder[1:]
return (curr_node[0], remainder)
def convert(self, keystring):
valuestring = ""
key = keystring
while key:
value, key = self.findp(key)
if not value:
return (valuestring, key)
valuestring += value
return (valuestring, key)
def beta2unicodeTrie():
t = Trie()
t.add("*A", u"\u0391")
t.add("*B", u"\u0392")
t.add("*G", u"\u0393")
t.add("*D", u"\u0394")
t.add("*E", u"\u0395")
t.add("*Z", u"\u0396")
t.add("*H", u"\u0397")
t.add("*Q", u"\u0398")
t.add("*I", u"\u0399")
t.add("*K", u"\u039A")
t.add("*L", u"\u039B")
t.add("*M", u"\u039C")
t.add("*N", u"\u039D")
t.add("*C", u"\u039E")
t.add("*O", u"\u039F")
t.add("*P", u"\u03A0")
t.add("*R", u"\u03A1")
t.add("*S", u"\u03A3")
t.add("*T", u"\u03A4")
t.add("*U", u"\u03A5")
t.add("*F", u"\u03A6")
t.add("*X", u"\u03A7")
t.add("*Y", u"\u03A8")
t.add("*W", u"\u03A9")
t.add("A", u"\u03B1")
t.add("B", u"\u03B2")
t.add("G", u"\u03B3")
t.add("D", u"\u03B4")
t.add("E", u"\u03B5")
t.add("Z", u"\u03B6")
t.add("H", u"\u03B7")
t.add("Q", u"\u03B8")
t.add("I", u"\u03B9")
t.add("K", u"\u03BA")
t.add("L", u"\u03BB")
t.add("M", u"\u03BC")
t.add("N", u"\u03BD")
t.add("C", u"\u03BE")
t.add("O", u"\u03BF")
t.add("P", u"\u03C0")
t.add("R", u"\u03C1")
t.add("S\n", u"\u03C2")
t.add("S,", u"\u03C2,")
t.add("S.", u"\u03C2.")
t.add("S:", u"\u03C2:")
t.add("S;", u"\u03C2;")
t.add("S]", u"\u03C2]")
t.add("S#", u"\u03C2#")
t.add("S_", u"\u03C2_")
t.add("S", u"\u03C3")
t.add("T", u"\u03C4")
t.add("U", u"\u03C5")
t.add("F", u"\u03C6")
t.add("X", u"\u03C7")
t.add("Y", u"\u03C8")
t.add("W", u"\u03C9")
t.add("I+", U"\u03CA")
t.add("U+", U"\u03CB")
t.add("A)", u"\u1F00")
t.add("A(", u"\u1F01")
t.add("A)\\", u"\u1F02")
t.add("A(\\", u"\u1F03")
t.add("A)/", u"\u1F04")
t.add("A(/", u"\u1F05")
t.add("E)", u"\u1F10")
t.add("E(", u"\u1F11")
t.add("E)\\", u"\u1F12")
t.add("E(\\", u"\u1F13")
t.add("E)/", u"\u1F14")
t.add("E(/", u"\u1F15")
t.add("H)", u"\u1F20")
t.add("H(", u"\u1F21")
t.add("H)\\", u"\u1F22")
t.add("H(\\", u"\u1F23")
t.add("H)/", u"\u1F24")
t.add("H(/", u"\u1F25")
t.add("I)", u"\u1F30")
t.add("I(", u"\u1F31")
t.add("I)\\", u"\u1F32")
t.add("I(\\", u"\u1F33")
t.add("I)/", u"\u1F34")
t.add("I(/", u"\u1F35")
t.add("O)", u"\u1F40")
t.add("O(", u"\u1F41")
t.add("O)\\", u"\u1F42")
t.add("O(\\", u"\u1F43")
t.add("O)/", u"\u1F44")
t.add("O(/", u"\u1F45")
t.add("U)", u"\u1F50")
t.add("U(", u"\u1F51")
t.add("U)\\", u"\u1F52")
t.add("U(\\", u"\u1F53")
t.add("U)/", u"\u1F54")
t.add("U(/", u"\u1F55")
t.add("W)", u"\u1F60")
t.add("W(", u"\u1F61")
t.add("W)\\", u"\u1F62")
t.add("W(\\", u"\u1F63")
t.add("W)/", u"\u1F64")
t.add("W(/", u"\u1F65")
t.add("A)=", u"\u1F06")
t.add("A(=", u"\u1F07")
t.add("H)=", u"\u1F26")
t.add("H(=", u"\u1F27")
t.add("I)=", u"\u1F36")
t.add("I(=", u"\u1F37")
t.add("U)=", u"\u1F56")
t.add("U(=", u"\u1F57")
t.add("W)=", u"\u1F66")
t.add("W(=", u"\u1F67")
t.add("*A)", u"\u1F08")
t.add("*)A", u"\u1F08")
t.add("*A(", u"\u1F09")
t.add("*(A", u"\u1F09")
#
t.add("*(\A", u"\u1F0B")
t.add("*A)/", u"\u1F0C")
t.add("*)/A", u"\u1F0C")
t.add("*A(/", u"\u1F0F")
t.add("*(/A", u"\u1F0F")
t.add("*E)", u"\u1F18")
t.add("*)E", u"\u1F18")
t.add("*E(", u"\u1F19")
t.add("*(E", u"\u1F19")
#
t.add("*(\E", u"\u1F1B")
t.add("*E)/", u"\u1F1C")
t.add("*)/E", u"\u1F1C")
t.add("*E(/", u"\u1F1D")
t.add("*(/E", u"\u1F1D")
t.add("*H)", u"\u1F28")
t.add("*)H", u"\u1F28")
t.add("*H(", u"\u1F29")
t.add("*(H", u"\u1F29")
t.add("*H)\\", u"\u1F2A")
t.add(")\\*H", u"\u1F2A")
t.add("*)\\H", u"\u1F2A")
#
t.add("*H)/", u"\u1F2C")
t.add("*)/H", u"\u1F2C")
#
t.add("*)=H", u"\u1F2E")
t.add("(/*H", u"\u1F2F")
t.add("*(/H", u"\u1F2F")
t.add("*I)", u"\u1F38")
t.add("*)I", u"\u1F38")
t.add("*I(", u"\u1F39")
t.add("*(I", u"\u1F39")
#
#
t.add("*I)/", u"\u1F3C")
t.add("*)/I", u"\u1F3C")
#
#
t.add("*I(/", u"\u1F3F")
t.add("*(/I", u"\u1F3F")
#
t.add("*O)", u"\u1F48")
t.add("*)O", u"\u1F48")
t.add("*O(", u"\u1F49")
t.add("*(O", u"\u1F49")
#
#
t.add("*(\O", u"\u1F4B")
t.add("*O)/", u"\u1F4C")
t.add("*)/O", u"\u1F4C")
t.add("*O(/", u"\u1F4F")
t.add("*(/O", u"\u1F4F")
#
t.add("*U(", u"\u1F59")
t.add("*(U", u"\u1F59")
#
t.add("*(/U", u"\u1F5D")
#
t.add("*(=U", u"\u1F5F")
t.add("*W)", u"\u1F68")
t.add("*W(", u"\u1F69")
t.add("*(W", u"\u1F69")
#
#
t.add("*W)/", u"\u1F6C")
t.add("*)/W", u"\u1F6C")
t.add("*W(/", u"\u1F6F")
t.add("*(/W", u"\u1F6F")
t.add("*A)=", u"\u1F0E")
t.add("*)=A", u"\u1F0E")
t.add("*A(=", u"\u1F0F")
t.add("*W)=", u"\u1F6E")
t.add("*)=W", u"\u1F6E")
t.add("*W(=", u"\u1F6F")
t.add("*(=W", u"\u1F6F")
t.add("A\\", u"\u1F70")
t.add("A/", u"\u1F71")
t.add("E\\", u"\u1F72")
t.add("E/", u"\u1F73")
t.add("H\\", u"\u1F74")
t.add("H/", u"\u1F75")
t.add("I\\", u"\u1F76")
t.add("I/", u"\u1F77")
t.add("O\\", u"\u1F78")
t.add("O/", u"\u1F79")
t.add("U\\", u"\u1F7A")
t.add("U/", u"\u1F7B")
t.add("W\\", u"\u1F7C")
t.add("W/", u"\u1F7D")
t.add("A)/|", u"\u1F84")
t.add("A(/|", u"\u1F85")
t.add("H)|", u"\u1F90")
t.add("H(|", u"\u1F91")
t.add("H)/|", u"\u1F94")
t.add("H)=|", u"\u1F96")
t.add("H(=|", u"\u1F97")
t.add("W)|", u"\u1FA0")
t.add("W(=|", u"\u1FA7")
t.add("A=", u"\u1FB6")
t.add("H=", u"\u1FC6")
t.add("I=", u"\u1FD6")
t.add("U=", u"\u1FE6")
t.add("W=", u"\u1FF6")
t.add("I\\+", u"\u1FD2")
t.add("I/+", u"\u1FD3")
t.add("I+/", u"\u1FD3")
t.add("U\\+", u"\u1FE2")
t.add("U/+", u"\u1FE3")
t.add("A|", u"\u1FB3")
t.add("A/|", u"\u1FB4")
t.add("H|", u"\u1FC3")
t.add("H/|", u"\u1FC4")
t.add("W|", u"\u1FF3")
t.add("W|/", u"\u1FF4")
t.add("W/|", u"\u1FF4")
t.add("A=|", u"\u1FB7")
t.add("H=|", u"\u1FC7")
t.add("W=|", u"\u1FF7")
t.add("R(", u"\u1FE4")
t.add("*R(", u"\u1FEC")
t.add("*(R", u"\u1FEC")
# t.add("~", u"~")
# t.add("-", u"-")
# t.add("(null)", u"(null)")
# t.add("&", "&")
t.add("0", u"0")
t.add("1", u"1")
t.add("2", u"2")
t.add("3", u"3")
t.add("4", u"4")
t.add("5", u"5")
t.add("6", u"6")
t.add("7", u"7")
t.add("8", u"8")
t.add("9", u"9")
t.add("#", u"#")
t.add("$", u"$")
t.add(" ", u" ")
t.add(".", u".")
t.add(",", u",")
t.add("'", u"'")
t.add(":", u":")
t.add(";", u";")
t.add("_", u"_")
t.add("[", u"[")
t.add("]", u"]")
t.add("\n", u"")
return t
t = beta2unicodeTrie()
import sys
for line in file(sys.argv[1]):
a, b = t.convert(line)
if b:
print a.encode("utf-8"), b
raise Exception
print a.encode("utf-8")
And here is a little .tex-file with which it should work.
\documentclass[12pt]{scrbook}
\usepackage[polutonikogreek, ngerman]{babel}
\usepackage[ngerman]{betababel}
\usepackage{fontspec}
%\defaultfontfeatures{Ligatures=TeX}
%\newfontfeature{Microtype}{protrusion=default;expansion=default;}
\begin{document}
\bcode{lo/gos}
\end{document}
In case the script does not work: would it be possible to convert all the strings within the \bcode-Makro with something like regex? For example the "o/" to the ό and so on? What would be the weapon of choice here?

Do I have python installed?
Try python -V at a shell prompt. Your code is python 2 code, so you will a python 2 version.
I need to install Python
Most straight forward way if you don't need a complex environment (and you don't for this problem) is just to go to python.org. Don't forget you need python 2.
Running the program
Generally it will be as simple as:
python beta2unicode.py myfile.tex-file
And to capture the output:
python beta2unicode.py myfile.tex-file > myfile.not-tex-file
Does the script work?
Almost. You will need to replace the code at the end of the script that starts the same way this does, with this:
import sys
t = beta2unicodeTrie()
import re
BCODE = re.compile(r'\\bcode{[^}]*}')
for line in open(sys.argv[1]):
matches = BCODE.search(line)
for match in BCODE.findall(line):
bcode = match[7:-1]
a, b = t.convert(bcode.upper())
if b:
raise IOError("failed conversion '%s' in '%s'" % (b, line))
converted = a.encode("utf-8")
line = line.replace(match, converted)
print(line.rstrip())
Results
\documentclass[12pt]{scrbook}
\usepackage[polutonikogreek, ngerman]{babel}
\usepackage[ngerman]{betababel}
\usepackage{fontspec}
%\defaultfontfeatures{Ligatures=TeX}
%\newfontfeature{Microtype}{protrusion=default;expansion=default;}
\begin{document}
λόγοσ
\end{document}

Weird bug in python project

I'm working on a big project in Python, and I've run into a bizarre error I can't explain. In one of my classes, I have a private method being called during instantiation:
def _convertIndex(self, dimInd, dimName):
'''Private function that converts numbers to numbers and non-integers using
the subclass\'s convertIndex.'''
print dimInd, ' is dimInd'
try:
return int(dimName)
except:
if dimName == '*':
return 0
else:
print self.param.sets, ' is self.param.sets'
print type(self.param.sets), ' is the type of self.param.sets'
print self.param.sets[dimInd], ' is the param at dimind'
return self.param.sets[dimInd].value(dimName)
What it's printing out:
0 is dimInd
[<coremcs.SymbolicSet.SymbolicSet object at 0x10618ad90>] is self.param.sets
<type 'list'> is the type of self.param.sets
<SymbolicSet BAZ=['baz1', 'baz2', 'baz3', 'baz4']> is the param at dimind
======================================================================
ERROR: testParameterSet (GtapTest.TestGtapParameter)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/myuser/Documents/workspace/ilucmc/gtapmcs/test/GtapTest.py", line 116, in testParameterSet
pset = ParameterSet(prmFile, dstFile, GtapParameter)
File "/Users/myuser/Documents/workspace/ilucmc/coremcs/ParameterSet.py", line 103, in __init__
self.distroDict, corrDefs = AbsBaseDistro.readFile(distFile, self.paramDict)
File "/Users/myuser/Documents/workspace/ilucmc/coremcs/Distro.py", line 359, in readFile
distro = cls.parseDistro(param, target, distroType, args)
File "/Users/myuser/Documents/workspace/ilucmc/coremcs/Distro.py", line 301, in parseDistro
return cls(param, target, distro, dim_names, argDict)
File "/Users/myuser/Documents/workspace/ilucmc/coremcs/Distro.py", line 150, in __init__
self.dim_indxs = list(starmap(self._convertIndex, enumerate(dim_names))) # convert to numeric values and save in dim_indxs
File "/Users/myuser/Documents/workspace/ilucmc/coremcs/Distro.py", line 194, in _convertIndex
print self.param.sets[dimInd], ' is the param at dimind'
IndexError: list index out of range
Obviously this isn't the code for the whole class, but it represents something that I don't understand. The error is coming when I index into self.param.sets. Apparently, dimInd is out of range. the problem is, dimInd is 0, and self.param.sets is a list of length 1 (as shown from the print statements), so why can't I index into it?
EDIT: For what it's worth, the __init__ method looks like this:
'''
Stores a definitions of a distribution to be applied to a header variable.
See the file setup/gtap/DistroDoc.txt for the details.
'''
def __init__(self, param, target, distType, dim_names, argDict):
self.name = param.name
self.dim_names = dim_names
self.dim_indxs = []
self.target = target.lower() if target else None
self.distType = distType.lower() if distType else None
self.rv = None
self.argDict = {}
self.modifier = defaultdict(lambda: None)
self.param = param
# Separate args into modifiers and distribution arguments
for k, v in argDict.iteritems():
if k[0] == '_': # modifiers start with underscore
self.modifier[k] = v
else:
self.argDict[k] = v # distribution arguments do not have underscore
if self.target == 'index':
print dim_names
self.dim_indxs = list(starmap(self._convertIndex, enumerate(dim_names))) # convert to numeric values and save in dim_indxs
if distType == 'discrete':
entries = self.modifier['_entries']
if not entries:
raise DistributionSpecError("Not enough arguments given to discrete distribution.")
modDict = {k[1:]: float(v) for k, v in self.modifier.iteritems() if k[1:] in getOptionalArgs(DiscreteDist.__init__)}
self.rv = DiscreteDist(entries, **modDict)
return
sig = DistroGen.signature(distType, self.argDict.keys())
gen = DistroGen.generator(sig)
if gen is None:
raise DistributionSpecError("Unknown distribution signature %s" % str(sig))
self.rv = gen.makeRV(self.argDict) # generate a frozen RV with the specified arguments
self.isFactor = gen.isFactor

pretty print assertEqual() for HTML strings

I want to compare two strings in a python unittest which contain html.
Is there a method which outputs the result in a human friendly (diff like) version?

A simple method is to strip whitespace from the HTML and split it into a list. Python 2.7's unittest (or the backported unittest2) then gives a human-readable diff between the lists.
import re
def split_html(html):
return re.split(r'\s*\n\s*', html.strip())
def test_render_html():
expected = ['<div>', '...', '</div>']
got = split_html(render_html())
self.assertEqual(expected, got)
If I'm writing a test for working code, I usually first set expected = [], insert a self.maxDiff = None before the assert and let the test fail once. The expected list can then be copy-pasted from the test output.
You might need to tweak how whitespace is stripped depending on what your HTML looks like.

I submitted a patch to do this some years back. The patch was rejected but you can still view it on the python bug list.
I doubt you would want to hack your unittest.py to apply the patch (if it even still works after all this time), but here's the function for reducing two strings a manageable size while still keeping at least part of what differs. So long as all you didn't want the complete differences this might be what you want:
def shortdiff(x,y):
'''shortdiff(x,y)
Compare strings x and y and display differences.
If the strings are too long, shorten them to fit
in one line, while still keeping at least some difference.
'''
import difflib
LINELEN = 79
def limit(s):
if len(s) > LINELEN:
return s[:LINELEN-3] + '...'
return s
def firstdiff(s, t):
span = 1000
for pos in range(0, max(len(s), len(t)), span):
if s[pos:pos+span] != t[pos:pos+span]:
for index in range(pos, pos+span):
if s[index:index+1] != t[index:index+1]:
return index
left = LINELEN/4
index = firstdiff(x, y)
if index > left + 7:
x = x[:left] + '...' + x[index-4:index+LINELEN]
y = y[:left] + '...' + y[index-4:index+LINELEN]
else:
x, y = x[:LINELEN+1], y[:LINELEN+1]
left = 0
cruncher = difflib.SequenceMatcher(None)
xtags = ytags = ""
cruncher.set_seqs(x, y)
editchars = { 'replace': ('^', '^'),
'delete': ('-', ''),
'insert': ('', '+'),
'equal': (' ',' ') }
for tag, xi1, xi2, yj1, yj2 in cruncher.get_opcodes():
lx, ly = xi2 - xi1, yj2 - yj1
edits = editchars[tag]
xtags += edits[0] * lx
ytags += edits[1] * ly
# Include ellipsis in edits line.
if left:
xtags = xtags[:left] + '...' + xtags[left+3:]
ytags = ytags[:left] + '...' + ytags[left+3:]
diffs = [ x, xtags, y, ytags ]
if max([len(s) for s in diffs]) < LINELEN:
return '\n'.join(diffs)
diffs = [ limit(s) for s in diffs ]
return '\n'.join(diffs)

Maybe this is a quite 'verbose' solution. You could add a new 'equality function' for your user defined type (e.g: HTMLString) which you have to define first:
class HTMLString(str):
pass
Now you have to define a type equality function:
def assertHTMLStringEqual(first, second):
if first != second:
message = ... # TODO here: format your message, e.g a diff
raise AssertionError(message)
All you have to do is format your message as you like. You can also use a class method in your specific TestCase as a type equality function. This gives you more functionality to format your message, since unittest.TestCase does this a lot.
Now you have to register this equality function in your unittest.TestCase:
...
def __init__(self):
self.addTypeEqualityFunc(HTMLString, assertHTMLStringEqual)
The same for a class method:
...
def __init__(self):
self.addTypeEqualityFunc(HTMLString, 'assertHTMLStringEqual')
And now you can use it in your tests:
def test_something(self):
htmlstring1 = HTMLString(...)
htmlstring2 = HTMLString(...)
self.assertEqual(htmlstring1, htmlstring2)
This should work well with python 2.7.

I (the one asking this question) use BeautfulSoup now:
def assertEqualHTML(string1, string2, file1='', file2=''):
u'''
Compare two unicode strings containing HTML.
A human friendly diff goes to logging.error() if there
are not equal, and an exception gets raised.
'''
from BeautifulSoup import BeautifulSoup as bs
import difflib
def short(mystr):
max=20
if len(mystr)>max:
return mystr[:max]
return mystr
p=[]
for mystr, file in [(string1, file1), (string2, file2)]:
if not isinstance(mystr, unicode):
raise Exception(u'string ist not unicode: %r %s' % (short(mystr), file))
soup=bs(mystr)
pretty=soup.prettify()
p.append(pretty)
if p[0]!=p[1]:
for line in difflib.unified_diff(p[0].splitlines(), p[1].splitlines(), fromfile=file1, tofile=file2):
logging.error(line)
raise Exception('Not equal %s %s' % (file1, file2))