Develop Reference

#rdflib (python): how to get a URIRef from a string such as 'ns:xxx'?

I have a RDFlib graph g, whose NameSpaceManager is aware of some namespaces.
How do I get a URIRef from a string such as 'ns:xxx', where ns is the prefix associated to a namespace known by g.namespace_manager? Basically, I'm looking for a method which does the inverse operation of URIRef's n3(g.namespace_manager). I'm pretty confident that there is a way to do it, as a similar function is needed to parse turtle files, or sparql queries, but I can't find it. Otherwise of course, it must not be very difficult to write it.
TIA

from rdflib import Graph, Namespace
from rdflib.namespace import RDF
g = Graph()
NS = Namespace("http://example.com/")
# then, say Xxx is a class and Aaa is an instance of Xxx...
g.add((NS.Aaa, RDF.type, NS.Xxx))
# so use NS.Xxx (or NS["Xxx"]) to get a URIRef of NS.Xxx from Namespace NS
print(type(NS)) # --> <class 'rdflib.term.URIRef'>
print(type(NS.Xxx)) # --> <class 'rdflib.term.URIRef'>
print(NS.Xxx) # --> "http://example.com/Xxx"
If you want to bind a prefix within a graph, you use the rdflib Graph class' bind() method so, for the code above, you would use:
g.bind("ns", NS)
Now the graph, if serialized with a format that knows about prefixes, like Turtle, will use "ns". The above data would be:
#prefix ns: <http://example.com/> .
#prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
ns:Aaa rdf:type ns:Xxx .
So, in Python, if you want to make the URI "http://example.com/Xxx"" from the string "ns:Xxx" you should have everything you need:
the namespace declaration: NS = Namespace("http://example.com/")
the prefix/namespace binding g.bind("ns", NS)
IFF, on the other hand, you didn't declare the namespace yourself but it's in a graph and you only have the short form URI "ns:Xxx", you can do this to list all bound prefixes & namespaces used in the graph:
for n in g.namespace_manager.namespaces():
print(n)
returns, for the data above:
('xml', rdflib.term.URIRef('http://www.w3.org/XML/1998/namespace'))
('rdf', rdflib.term.URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#'))
('rdfs', rdflib.term.URIRef('http://www.w3.org/2000/01/rdf-schema#'))
('xsd', rdflib.term.URIRef('http://www.w3.org/2001/XMLSchema#'))
('eg', rdflib.term.URIRef('http://example.com/'))
So, if you know "eg:Xxx", you can split off the "eg" part and make the URI you want like this:
print(
[str(x[1]) for x in g.namespace_manager.namespaces() if x[0] == s.split(":")[0]]
[0] + s.split(":")[1]
)
prints:
http://example.com/Xxx

from rdflib.namespace import NamespaceManager
import rdflib as rdflib
class MyNamespacesInfo:
def __init__(self, namespace_manager: NamespaceManager):
# as I don't know how to get the namespace from a prefix from the API
# I construct a dict
self.pref2ns = {}
for pref, ns in namespace_manager.namespaces():
self.pref2ns[pref] = ns
def uriref(self, n3uri: str) -> rdflib.URIRef:
# n3uri: either 'ns:xxx', '<http://..../xxx>' or 'http://..../xxx'
if n3uri[0] == '<':
if n3uri[len(n3uri)-1] == '>':
return rdflib.URIRef(n3uri[1:-1])
else:
raise ValueError("Illegal uri: ", n3uri)
else:
return self.prefixed_2_uriref(n3uri, laxist=True)
def prefixed_2_uriref(self, short_uri: str, laxist=True) -> rdflib.URIRef:
# param short_uri eg. 'ns:xxx', where ns declared in namespace_manager
# using laxist = True, you also can pass a long uri
s = short_uri.split(':')
if len(s) < 2:
if laxist:
return rdflib.URIRef(short_uri)
else:
raise ValueError('Not a prefix:localname string: ' + short_uri)
prefix = s[0]
ns = self.pref2ns.get(prefix)
if ns == None:
if laxist:
return rdflib.URIRef(short_uri)
else:
raise ValueError('Unknown prefix: ' + prefix)
else:
x = ns + s[1]
for i in range(2, len(s)):
x = x + (':' + s[i])
return x
# example of use:
g = rdflib.Graph()
g.parse('http://www.semanlink.net/tag/rdf_tools.rdf')
ns_info = MyNamespacesInfo(g.namespace_manager)
# all of following calls print http://www.semanlink.net/tag/rdflib
x = ns_info.uriref('tag:rdflib')
print(x)
x = ns_info.uriref('http://www.semanlink.net/tag/rdflib')
print(x)
x = ns_info.uriref('<http://www.semanlink.net/tag/rdflib>')
print(x)

Comparing and delete api data

I have a django project that takes in some data from another app of ours. The data looks like this:
{u'   updated':   u'2017-04-03T22:30:53.760278   Z',
   u'added':   u'2017-04-03T22:30:53.760197   Z',
   u'name':u'Jean Lamb,
   1942-   ', u'   authority':{  
      u'id':2,
      u'added_by':2,
      u'name':u'VIAF'
   },
   u'local_identifier':u'85363862',
   u'concept_type':{  
      u'id':5,
      u'identifier':      u'viaf:personal',
      u'name':u'',
      u'description':None
   },
   u'identifier':   u'http://viaf.org/viaf/85363862',
   u'identities':[  
      {  
         u'part_of':{  
            u'id':1,
            u'added_by':2,
            u'name':            u'builtin:Conceptpower'
         },
         u'added':         u'2017-04-03T22:33:20.476637         Z',
         u'name':u'Jean Lamb',
         u'confidence':1.0,
         u'updated':         u'2017-04-03T22:33:20.476699         Z',
         u'concepts':[  
            u'http://viaf.org/viaf/85363862',
            u'http://www.digitalhps.org/concepts/CONpeSHC70qxNC0'
         ],
         u'id':208,
         u'added_by':{  
            u'username':u'erickjones',
            u'email':u'erick.jones#example.com'
         }
      },
      {  
         u'part_of':{  
            u'id':1,
            u'added_by':2,
            u'name':            u'builtin:Conceptpower'
         },
         u'added':         u'2017-04-03T22:35:02.546054         Z',
         u'name':u'Jean Lamb',
         u'confidence':1.0,
         u'updated':         u'2017-04-03T22:35:02.546116         Z',
         u'concepts':[  
            u'http://viaf.org/viaf/85363862',
            u'http://www.digitalhps.org/concepts/CONpeSHC70qxNC0'
         ],
         u'id':209,
         u'added_by':{  
            u'username':u'erickjones',
            u'email':u'erick.jones#example.com'
         }
      },
Right now I have a function that goes through and compares the concepts in in the identities. What I want to do is delete the duplicate concepts. The nesting of the dictionaries and lists are throwing me off. What I have been trying is:
del results[i]["identities"][z]["concepts"]
Any ideas as to why this does not work?
Here is my loop incase anyone is interested:
while (i != di):
test = results[i]["identities"]
if results[i]["identities"]:
z = 0
while (z != len(results[i]["identities"])):
con1 = results[i]["identities"][z]["concepts"]
print "this is con1: %s", con1
if z != len(results[i]["identities"]):
z = z + 1
else:
break
if z != len(results[i]["identities"]):
con2 = results[i]["identities"][z]["concepts"]
print "this is con2: %s", con2
if set(con1) == set(con2):
del results[i]["identities"][z]["concepts"]
else:
break
i = i + 1

In this line,
if set(con1) and set(con2):
Do you intend to check if con1 and con2 are the same set? Shouldn't you use == operator?

Is there a way to programmatically combine Korean unicode into one?

Using a Korean Input Method Editor (IME), it's possible to type 버리 + 어 and it will automatically become 버려.
Is there a way to programmatically do that in Python?
>>> x, y = '버리', '어'
>>> z = '버려'
>>> ord(z[-1])
47140
>>> ord(x[-1]), ord(y)
(47532, 50612)
Is there a way to compute that 47532 + 50612 -> 47140?
Here's some more examples:
가보 + 아 -> 가봐
끝나 + ㄹ -> 끝날

I'm a Korean. First, if you type 버리 + 어, it becomes 버리어 not 버려. 버려 is an abbreviation of 버리어 and it's not automatically generated. Also 가보아 cannot becomes 가봐 automatically during typing by the same reason.
Second, by contrast, 끝나 + ㄹ becomes 끝날 because 나 has no jongseong(종성). Note that one character of Hangul is made of choseong(초성), jungseong(중성), and jongseong. choseong and jongseong are a consonant, jungseong is a vowel. See more at Wikipedia. So only when there's no jongseong during typing (like 끝나), there's a chance that it can have jongseong(ㄹ).
If you want to make 버리 + 어 to 버려, you should implement some Korean language grammar like, especially for this case, abbreviation of jungseong. For example ㅣ + ㅓ = ㅕ, ㅗ + ㅏ = ㅘ as you provided. 한글 맞춤법 chapter 4. section 5 (I can't find English pages right now) defines abbreviation like this. It's possible, but not so easy job especially for non-Koreans.
Next, if what you want is just to make 끝나 + ㄹ to 끝날, it can be a relatively easy job since there're libraries which can handle composition and decomposition of choseong, jungseong, jongseong. In case of Python, I found hgtk. You can try like this (nonpractical code):
# hgtk methods take one character at a time
cjj1 = hgtk.letter.decompose('나') # ('ㄴ', 'ㅏ', '')
cjj2 = hgtk.letter.decompose('ㄹ') # ('ㄹ', '', '')
if cjj1[2]) == '' and cjj2[1]) == '':
cjj = (cjj1[0], cjj1[1], cjj2[0])
cjj2 = None
Still, without proper knowledge of Hangul, it will be very hard to get it done.

You could use your own Translation table.
The drawback is you have to input all pairs manual or you have a file to get it from.
For instance:
# Sample Korean chars to map
k = [[('버리', '어'), ('버려')], [('가보', '아'), ('가봐')], [('끝나', 'ㄹ'), ('끝날')]]
class Korean(object):
def __init__(self):
self.map = {}
for m in k:
key = m[0][0] + m[0][1]
self.map[hash(key)] = m[1]
def __getitem__(self, item):
return self.map[hash(item)]
def translate(self, s):
return [ self.map[hash(token)] for token in s]
if __name__ == '__main__':
k_map = Korean()
k_chars = [ m[0][0] + m[0][1] for m in k]
print('Input: %s' % k_chars)
print('Output: %s' % k_map.translate(k_chars))
one_char_3 = k[0][0][0] + k[0][0][1]
print('%s = %s' % (one_char_3, k_map[ one_char_3 ]) )
Input: ['버리어', '가보아', '끝나ㄹ']
Output: ['버려', '가봐', '끝날']
버리어 = 버려
Tested with Python:3.4.2

Python - calculations with more than one splited serial data

I use Arduino to receive data from sensors (4 types of data : humidity, temperature, photocell and milliseconds)
Datas comes like this : xx xx xx xxxx in the serial buffer. (data space data space etc...)
I split this line in order to isolate each data because I want to make individual calculations for each sensor.
Calculation for each sensor consist on : ((latest_data) - (data_of_previous_line), latest_data) in order to get a tuple for each sensor. I want all the sensors tuples appearing in the same line.
Doing this with 1 sensor and 1 method (calculate()) is working fine but it doesn't work if I add a second sensor in sensors() object !
QUESTION : how to make all this working with at least 2 sensors data ?
(the code below is working perfectly with 1 "splited" sensor data).
Thanks in advance.
class _share:
def __init__(self):
self.last_val = [0 for i in range(2)]
def calculate(self, val):
self.last_data = val
self.last_val = [self.last_data] + self.last_val[:-1]
diff = reduce(operator.__sub__, self.last_val)
print (diff, val)
return (diff, val)
share = _share()
ser = serial.Serial('/dev/ttyS1', 9600, timeout=0.1)
def sensors():
while True:
try:
time.sleep(0.01)
ser.flushInput()
reception = ser.readline()
receptionsplit = reception.split()
sensor_milli = receptionsplit[3]
sensor_pho_1 = receptionsplit[2]
sensor_tem_1 = receptionsplit[1]
sensor_hum_1 = receptionsplit[0]
int_sensor_milli = int(sensor_milli)
int_sensor_pho_1 = int(sensor_pho_1)
int_sensor_tem_1 = int(sensor_tem_1)
int_sensor_hum_1 = int(sensor_hum_1)
a = int_sensor_milli
b = int_sensor_pho_1
c = int_sensor_tem_1
d = int_sensor_hum_1
return str(share.calculate(b))
except:
pass
time.sleep(0.1)
f = open('da.txt', 'ab')
while 1:
arduino_sensor = sensors()
f.write(arduino_sensor)
f.close()
f = open('da.txt', 'ab')

You need to use different share instance for each sensor otherwise the calculations will be wrong. So use share_a, share_b, share_c and share_d for a, b, c and d respectively for example. Now if I understand this correctly, you can return all the sensors at once by changing your return to:
return [ str(share_a.calculate(a)), str(share_b.calculate(b)), str(share_c.calculate(c)), str(share_d.calculate(d)) ]
The above would return a list containing all 4 sensors and then in your main method you can change to:
arduino_sensor = sensors()
sensor_string ="a:%s b:%s c:%s d:%s"%( arduino_sensor[0], arduino_sensor[1], arduino_sensor[2], arduino_sensor[3] )
print sensor_string # single line screen print of all sensor data
f.write( sensor_string )
I hope that is helpful.

pretty print assertEqual() for HTML strings

I want to compare two strings in a python unittest which contain html.
Is there a method which outputs the result in a human friendly (diff like) version?

A simple method is to strip whitespace from the HTML and split it into a list. Python 2.7's unittest (or the backported unittest2) then gives a human-readable diff between the lists.
import re
def split_html(html):
return re.split(r'\s*\n\s*', html.strip())
def test_render_html():
expected = ['<div>', '...', '</div>']
got = split_html(render_html())
self.assertEqual(expected, got)
If I'm writing a test for working code, I usually first set expected = [], insert a self.maxDiff = None before the assert and let the test fail once. The expected list can then be copy-pasted from the test output.
You might need to tweak how whitespace is stripped depending on what your HTML looks like.

I submitted a patch to do this some years back. The patch was rejected but you can still view it on the python bug list.
I doubt you would want to hack your unittest.py to apply the patch (if it even still works after all this time), but here's the function for reducing two strings a manageable size while still keeping at least part of what differs. So long as all you didn't want the complete differences this might be what you want:
def shortdiff(x,y):
'''shortdiff(x,y)
Compare strings x and y and display differences.
If the strings are too long, shorten them to fit
in one line, while still keeping at least some difference.
'''
import difflib
LINELEN = 79
def limit(s):
if len(s) > LINELEN:
return s[:LINELEN-3] + '...'
return s
def firstdiff(s, t):
span = 1000
for pos in range(0, max(len(s), len(t)), span):
if s[pos:pos+span] != t[pos:pos+span]:
for index in range(pos, pos+span):
if s[index:index+1] != t[index:index+1]:
return index
left = LINELEN/4
index = firstdiff(x, y)
if index > left + 7:
x = x[:left] + '...' + x[index-4:index+LINELEN]
y = y[:left] + '...' + y[index-4:index+LINELEN]
else:
x, y = x[:LINELEN+1], y[:LINELEN+1]
left = 0
cruncher = difflib.SequenceMatcher(None)
xtags = ytags = ""
cruncher.set_seqs(x, y)
editchars = { 'replace': ('^', '^'),
'delete': ('-', ''),
'insert': ('', '+'),
'equal': (' ',' ') }
for tag, xi1, xi2, yj1, yj2 in cruncher.get_opcodes():
lx, ly = xi2 - xi1, yj2 - yj1
edits = editchars[tag]
xtags += edits[0] * lx
ytags += edits[1] * ly
# Include ellipsis in edits line.
if left:
xtags = xtags[:left] + '...' + xtags[left+3:]
ytags = ytags[:left] + '...' + ytags[left+3:]
diffs = [ x, xtags, y, ytags ]
if max([len(s) for s in diffs]) < LINELEN:
return '\n'.join(diffs)
diffs = [ limit(s) for s in diffs ]
return '\n'.join(diffs)

Maybe this is a quite 'verbose' solution. You could add a new 'equality function' for your user defined type (e.g: HTMLString) which you have to define first:
class HTMLString(str):
pass
Now you have to define a type equality function:
def assertHTMLStringEqual(first, second):
if first != second:
message = ... # TODO here: format your message, e.g a diff
raise AssertionError(message)
All you have to do is format your message as you like. You can also use a class method in your specific TestCase as a type equality function. This gives you more functionality to format your message, since unittest.TestCase does this a lot.
Now you have to register this equality function in your unittest.TestCase:
...
def __init__(self):
self.addTypeEqualityFunc(HTMLString, assertHTMLStringEqual)
The same for a class method:
...
def __init__(self):
self.addTypeEqualityFunc(HTMLString, 'assertHTMLStringEqual')
And now you can use it in your tests:
def test_something(self):
htmlstring1 = HTMLString(...)
htmlstring2 = HTMLString(...)
self.assertEqual(htmlstring1, htmlstring2)
This should work well with python 2.7.

I (the one asking this question) use BeautfulSoup now:
def assertEqualHTML(string1, string2, file1='', file2=''):
u'''
Compare two unicode strings containing HTML.
A human friendly diff goes to logging.error() if there
are not equal, and an exception gets raised.
'''
from BeautifulSoup import BeautifulSoup as bs
import difflib
def short(mystr):
max=20
if len(mystr)>max:
return mystr[:max]
return mystr
p=[]
for mystr, file in [(string1, file1), (string2, file2)]:
if not isinstance(mystr, unicode):
raise Exception(u'string ist not unicode: %r %s' % (short(mystr), file))
soup=bs(mystr)
pretty=soup.prettify()
p.append(pretty)
if p[0]!=p[1]:
for line in difflib.unified_diff(p[0].splitlines(), p[1].splitlines(), fromfile=file1, tofile=file2):
logging.error(line)
raise Exception('Not equal %s %s' % (file1, file2))