Develop Reference

How to save <class "ctypes.c_char_Array> with pickle

I'm doing research on palmprint recognition. for that I use the edcc library for the introduction of the palmprint. but i have problem in saving the encoding result from palm. I want to save the encoding result into a file, but I get an error like below
Traceback (most recent call last):
File "/home/pi/Coba/PalmDetection/PalmRecognition.py", line 18, in <module>
pickle.dump(one_palmprint_code, config_dictionary_file)
_pickle.PicklingError: Can't pickle <class 'ctypes.c_char_Array_849'>: attribute lookup c_char_Array_849 on ctypes failed
My code like this :
import os
import edcc
import cv2
import pickle
TEST_PALMPRINT_DATA_DIR = "/home/pi/Coba/PalmDetection/Data"
TEST_A_01_PALMPRINT_IMAGE = os.path.join(TEST_PALMPRINT_DATA_DIR, "Palm1.jpeg")
#TEST_A_02_PALMPRINT_IMAGE = os.path.join(TEST_PALMPRINT_DATA_DIR, "a_02.bmp")
TEST_B_01_PALMPRINT_IMAGE = os.path.join(TEST_PALMPRINT_DATA_DIR, "palme.jpeg")
#TEST_B_02_PALMPRINT_IMAGE = os.path.join(TEST_PALMPRINT_DATA_DIR, "b_02.bmp")
if __name__ == "__main__":
config = edcc.EncoderConfig(29, 5, 5, 10)
encoder = edcc.create_encoder(config)
one_palmprint_code = encoder.encode_using_file(TEST_A_01_PALMPRINT_IMAGE)
with open('encode.encode', 'wb') as config_dictionary_file:
pickle.dump(one_palmprint_code, config_dictionary_file)
another_palmprint_code = encoder.encode_using_file(TEST_B_01_PALMPRINT_IMAGE)
similarity_score = one_palmprint_code.compare_to(another_palmprint_code)
print(
"{} <-> {} similarity score:{}".format(
TEST_A_01_PALMPRINT_IMAGE, TEST_B_01_PALMPRINT_IMAGE, similarity_score
)
)
What should i do?

The edcc module must use ctypes internally, but really should hide that fact instead of returning a ctypes-wrapped object. A ctypes.c_char_Array_849 is just a C-compatible wrapper around an array of bytes. You can access the equivalent Python bytes object via the .raw property (what edcc should return instead) and write that to the file:
import ctypes
import pickle
one_palmprint_code = (ctypes.c_char * 849)()
with open('encode.encode', 'wb') as config_dictionary_file:
#pickle.dump(one_palmprint_code, config_dictionary_file) # reproduces error
config_dictionary_file.write(one_palmprint_code.raw)

kubectl cp in kubernetes python client

I have been trying to convert kubectl cp command to it's equivalent kubernetes python client program. I have following code for that:
from kubernetes import client, config
from kubernetes.stream import stream
import tarfile
from tempfile import TemporaryFile
# create an instance of the API class
config.load_kube_config()
api_instance = client.CoreV1Api()
exec_command = ['tar', 'xvf', '-', '-C', '/']
resp = stream(api_instance.connect_get_namespaced_pod_exec, "nginx-deployment-6bb6554bf-9sdtr", 'default',
command=exec_command,
stderr=True, stdin=True,
stdout=True, tty=False,
_preload_content=False)
source_file = '/tmp/abc.txt'
with TemporaryFile() as tar_buffer:
with tarfile.open(fileobj=tar_buffer, mode='w') as tar:
tar.add(source_file)
tar_buffer.seek(0)
commands = []
commands.append(tar_buffer.read())
while resp.is_open():
resp.update(timeout=1)
if resp.peek_stdout():
print("STDOUT: %s" % resp.read_stdout())
if resp.peek_stderr():
print("STDERR: %s" % resp.read_stderr())
if commands:
c = commands.pop(0)
# print("Running command... %s\n" % c)
resp.write_stdin(c)
else:
break
resp.close()
The above code gives me following error:
/home/velotio/venv/bin/python /home/velotio/PycharmProjects/k8sClient/testing.py
Traceback (most recent call last):
File "/home/velotio/PycharmProjects/k8sClient/testing.py", line 38, in <module>
resp.write_stdin(c)
File "/usr/local/lib/python3.6/site-packages/kubernetes/stream/ws_client.py", line 160, in write_stdin
self.write_channel(STDIN_CHANNEL, data)
File "/usr/local/lib/python3.6/site-packages/kubernetes/stream/ws_client.py", line 114, in write_channel
self.sock.send(chr(channel) + data)
TypeError: must be str, not bytes
I am using Python 3.6.3 and on kubernetes 1.13 version.

You have to convert bytes back to string which is what write_stdin method is expecting to get.
for example:
resp.write_stdin(c.decode())
Another example:
# Array with two byte objects
In [1]: a = [b'1234', b'3455']
# Pop one of them
In [2]: c = a.pop(0)
# it is of byte type
In [3]: c
Out[3]: b'1234'
# wrapping in str won't help if you don't provide decoding
In [4]: str(c)
Out[4]: "b'1234'"
# With decoding
In [5]: str(c, 'utf-8')
Out[5]: '1234'
# Or simply use the decode str method
In [6]: c.decode()
Out[6]: '1234'
More on byte to string conversion here:
Convert bytes to a string?

How do I read this stringified javascript variable into Python?

I'm trying to read _pageData from https://www.simpliowebstudio.com/wp-content/uploads/2014/07/aWfyh1 into Python 2.7.11 so that I can process it, using this code:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Testing _pageData processing. """
import urllib2
import re
import ast
import json
import yaml
BASE_URL = 'https://www.simpliowebstudio.com/wp-content/uploads/2014/07/aWfyh1'
def main():
""" Do the business. """
response = urllib2.urlopen(BASE_URL, None)
results = re.findall('var _pageData = \\"(.*?)\\";</script>', response.read())
first_result = results[0]
# These all fail
data = ast.literal_eval(first_result)
# data = yaml.load(first_result)
# data = json.loads(first_result)
if __name__ == '__main__':
main()
but get the following error:
Traceback (most recent call last):
File "./temp.py", line 24, in <module>
main()
File "./temp.py", line 19, in main
data = ast.literal_eval(first_result)
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ast.py", line 49, in literal_eval
node_or_string = parse(node_or_string, mode='eval')
File "/usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ast.py", line 37, in parse
return compile(source, filename, mode, PyCF_ONLY_AST)
File "<unknown>", line 1
[[1,true,true,true,true,true,true,true,true,,\"at\",\"\",\"\",1450364255674,\"\",\"en_US\",false,[]\n,\"https://www.google.com/maps/d/viewer?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/embed?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/edit?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/thumbnail?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",,,true,\"https://www.google.com/maps/d/print?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/pdf?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/viewer?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",false,false,\"/maps/d\",\"maps/sharing\",\"//www.google.com/intl/en_US/help/terms_maps.html\",true,\"https://docs.google.com/picker\",[]\n,false,true,[[[\"//www.gstatic.com/mapspro/images/google-my-maps-logo-regular-001.png\",143,25]\n,[\"//www.gstatic.com/mapspro/images/google-my-maps-logo-regular-2x-001.png\",286,50]\n]\n,[[\"//www.gstatic.com/mapspro/images/google-my-maps-logo-small-001.png\",113,20]\n,[\"//www.gstatic.com/mapspro/images/google-my-maps-logo-small-2x-001.png\",226,40]\n]\n]\n,1,\"https://www.gstatic.com/mapspro/_/js/k\\u003dmapspro.gmeviewer.en_US.8b9lQX3ifcs.O/m\\u003dgmeviewer_base/rt\\u003dj/d\\u003d0/rs\\u003dABjfnFWonctWGGtD63MaO3UZxCxF6UPKJQ\",true,true,false,true,\"US\",false,true,true,5,false]\n,[\"mf.map\",\"zBghbRiSwHlg.k2ATNtn6BCk0\",\"Hollywood, FL\",\"\",[-80.16005,26.01043,-80.16005,26.01043]\n,[-80.16005,26.01043,-80.16005,26.01043]\n,[[,\"zBghbRiSwHlg.kq4rrF9BNRIg\",\"Untitled layer\",\"\",[[[\"https://mt.googleapis.com/vt/icon/name\\u003dicons/onion/22-blue-dot.png\\u0026scale\\u003d1.0\"]\n,[]\n,1,1,[[,[26.01043,-80.16005]\n]\n,\"MDZBMzJCQjRBOTAwMDAwMQ~CjISKmdlby1tYXBzcHJvLm1hcHNob3AtbGF5ZXItNDUyOWUwMTc0YzhkNmI2ZBgAKAAwABIZACBawIJBU4Fe8v7vNSoAg0dtnhhVotEBLg\",\"vdb:\",\"zBghbRiSwHlg.kq4rrF9BNRIg\",[26.01043,-80.16005]\n,[0,-32]\n,\"06A32BB4A9000001\"]\n,[[\"Hollywood, FL\"]\n]\n,[]\n]\n]\n,,1.0,true,true,,,,[[\"zBghbRiSwHlg.kq4rrF9BNRIg\",1,,,,\"https://mapsengine.google.com/map/kml?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\\u0026lid\\u003dzBghbRiSwHlg.kq4rrF9BNRIg\",,,,,0,2,true,[[[\"06A32BB4A9000001\",[[[26.01043,-80.16005]\n]\n]\n,[]\n,[]\n,0,[[\"name\",[\"Hollywood, FL\"]\n,1]\n,,[]\n,[]\n]\n,,0]\n]\n,[[[\"https://mt.googleapis.com/vt/icon/name\\u003dicons/onion/22-blue-dot.png\\u0026filter\\u003dff\\u0026scale\\u003d1.0\",[16,32]\n,1.0]\n,[[\"0000FF\",0.45098039215686275]\n,5000]\n,[[\"0000FF\",0.45098039215686275]\n,[\"000000\",0.25098039215686274]\n,3000]\n]\n]\n]\n]\n]\n,[]\n,,,,,1]\n]\n,[2]\n,,,\"mapspro\",\"zBghbRiSwHlg.k2ATNtn6BCk0\",,true,false,false,\"\",2,false,\"https://mapsengine.google.com/map/kml?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",3807]\n]\n
^
SyntaxError: invalid syntax
var _pageData is in this format:
"[[1,true,true,true,true,true,true,true,true,,\"at\",\"\",\"\",1450364255674,\"\",\"en_US\",false,[]\n,\"https://www.google.com/maps/d/viewer?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/embed?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/edit?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/thumbnail?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",,,true,\"https://www.google.com/maps/d/print?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/pdf?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",\"https://www.google.com/maps/d/viewer?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",false,false,\"/maps/d\",\"maps/sharing\",\"//www.google.com/intl/en_US/help/terms_maps.html\",true,\"https://docs.google.com/picker\",[]\n,false,true,[[[\"//www.gstatic.com/mapspro/images/google-my-maps-logo-regular-001.png\",143,25]\n,[\"//www.gstatic.com/mapspro/images/google-my-maps-logo-regular-2x-001.png\",286,50]\n]\n,[[\"//www.gstatic.com/mapspro/images/google-my-maps-logo-small-001.png\",113,20]\n,[\"//www.gstatic.com/mapspro/images/google-my-maps-logo-small-2x-001.png\",226,40]\n]\n]\n,1,\"https://www.gstatic.com/mapspro/_/js/k\\u003dmapspro.gmeviewer.en_US.8b9lQX3ifcs.O/m\\u003dgmeviewer_base/rt\\u003dj/d\\u003d0/rs\\u003dABjfnFWonctWGGtD63MaO3UZxCxF6UPKJQ\",true,true,false,true,\"US\",false,true,true,5,false]\n,[\"mf.map\",\"zBghbRiSwHlg.k2ATNtn6BCk0\",\"Hollywood, FL\",\"\",[-80.16005,26.01043,-80.16005,26.01043]\n,[-80.16005,26.01043,-80.16005,26.01043]\n,[[,\"zBghbRiSwHlg.kq4rrF9BNRIg\",\"Untitled layer\",\"\",[[[\"https://mt.googleapis.com/vt/icon/name\\u003dicons/onion/22-blue-dot.png\\u0026scale\\u003d1.0\"]\n,[]\n,1,1,[[,[26.01043,-80.16005]\n]\n,\"MDZBMzJCQjRBOTAwMDAwMQ~CjISKmdlby1tYXBzcHJvLm1hcHNob3AtbGF5ZXItNDUyOWUwMTc0YzhkNmI2ZBgAKAAwABIZACBawIJBU4Fe8v7vNSoAg0dtnhhVotEBLg\",\"vdb:\",\"zBghbRiSwHlg.kq4rrF9BNRIg\",[26.01043,-80.16005]\n,[0,-32]\n,\"06A32BB4A9000001\"]\n,[[\"Hollywood, FL\"]\n]\n,[]\n]\n]\n,,1.0,true,true,,,,[[\"zBghbRiSwHlg.kq4rrF9BNRIg\",1,,,,\"https://mapsengine.google.com/map/kml?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\\u0026lid\\u003dzBghbRiSwHlg.kq4rrF9BNRIg\",,,,,0,2,true,[[[\"06A32BB4A9000001\",[[[26.01043,-80.16005]\n]\n]\n,[]\n,[]\n,0,[[\"name\",[\"Hollywood, FL\"]\n,1]\n,,[]\n,[]\n]\n,,0]\n]\n,[[[\"https://mt.googleapis.com/vt/icon/name\\u003dicons/onion/22-blue-dot.png\\u0026filter\\u003dff\\u0026scale\\u003d1.0\",[16,32]\n,1.0]\n,[[\"0000FF\",0.45098039215686275]\n,5000]\n,[[\"0000FF\",0.45098039215686275]\n,[\"000000\",0.25098039215686274]\n,3000]\n]\n]\n]\n]\n]\n,[]\n,,,,,1]\n]\n,[2]\n,,,\"mapspro\",\"zBghbRiSwHlg.k2ATNtn6BCk0\",,true,false,false,\"\",2,false,\"https://mapsengine.google.com/map/kml?mid\\u003dzBghbRiSwHlg.k2ATNtn6BCk0\",3807]\n]\n"
I've tried replacing the \" and \n and decoding the \uXXXX before using, without success. I've also tried replacing ,, with ,"", and ,'', without success.
Thank you.

It seems like there are three kinds of syntactic errors in your string:
, followed by ,
[ followed by ,
, followed by ]
Assuming that those are supposed to be null elements (or ''?), you can just replace those in the original string -- exactly like you did for the ,, case, but you missed the others. Also, you have to do the ,, replacement twice, otherwise you will miss cases such as ,,,,. Then, you can load the JSON string with json.loads.
>>> s = "your messed up json string"
>>> s = re.sub(r",\s*,", ", null,", s)
>>> s = re.sub(r",\s*,", ", null,", s)
>>> s = re.sub(r"\[\s*,", "[ null,", s)
>>> s = re.sub(r",\s*\]", ", null]", s)
>>> json.loads(s)

I started off using ast.literal.eval(...) because I was under the (mistaken?) impression that javascript arrays and Python lists were mutually compatible, so all I had to do was destringify _pageData.
However, I hadn't noticed that Python doesn't like ,, true, false or [,. Fixing them does the trick (thank you #Two-Bit Alchemist and #tobias_k)
So, the following appears to work:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Testing _pageData processing. """
import urllib2
import re
import ast
import json
import yaml
BASE_URL = 'https://www.simpliowebstudio.com/wp-content/uploads/2014/07/aWfyh1'
def main():
""" Do the business. """
response = urllib2.urlopen(BASE_URL, None)
results = re.findall('var _pageData = \\"(.*?)\\";</script>', response.read())
first_result = results[0]
first_result = first_result.replace(',,,,,,', ',None,None,None,None,None,')
first_result = first_result.replace(',,,,,', ',None,None,None,None,')
first_result = first_result.replace(',,,,', ',None,None,None,')
first_result = first_result.replace(',,,', ',None,None,')
first_result = first_result.replace(',,', ',None,')
first_result = first_result.replace('[,', '[None,')
first_result = first_result.replace('\\"', '\'')
first_result = first_result.replace('\\n', '')
first_result = first_result.replace('true', 'True')
first_result = first_result.replace('false', 'False')
data = ast.literal_eval(first_result)
for entry in data:
print entry
if __name__ == '__main__':
main()

python error : 'module' object is not callable "math.ceil"

Ive the following function which is do POST request to provider , I need to add new param to post request to incress the timeout ( which is by default is 5 mints i want to incress it to 1 hour , i did changes but i keep getting errors
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib64/python2.6/threading.py", line 532, in __bootstrap_inner
self.run()
File "/opt/lvptest/lvp_upload.py", line 226, in run
op = uploadMedia(mediaName, "PyUploader", env)
File "/opt/lvptest/lvp_upload.py", line 121, in uploadMedia
expires = math.ceil(time() + 3000) ["expires"]
TypeError: 'module' object is not callable
Here is my function
def uploadMedia(filepath, description, env):
global verbose
global config
orgId = config[env]["org_id"]
accessKey = config[env]["access_key"]
secret = config[env]["secret"]
expires = math.ceil(time() + 3000) ["expires"]
filename = os.path.basename(filepath)
baseUrl = "http://api.videoplatform.limelight.com/rest/organizations/%s/media" %(orgId)
signedUrl = lvp_auth_util.authenticate_request("POST", baseUrl, accessKey, secret, expires)
c = pycurl.Curl()
c.setopt(c.POST, 1)
c.setopt(c.HEADER, 0)
c.setopt(c.HTTPPOST, [('title', filename), ("description", description), (("media_file", (c.FORM_FILE, filepath)))])
if verbose:
c.setopt(c.VERBOSE, 1)
bodyOutput = StringIO()
headersOutput = StringIO()
c.setopt(c.WRITEFUNCTION, bodyOutput.write)
c.setopt(c.URL, signedUrl)
c.setopt(c.HEADERFUNCTION, headersOutput.write)
try:
c.perform()
c.close()
Any tips if im mistaken adding param "expires" ?
here is example how is my POST request looks like
POST /rest/organizations/9fafklsdf/media?access_key=sfdfsdfsdfsdfsdf89234 &expires=1400406364&signature=Mc9Qsd4sdgdfg0iEOFUaRC4iiAJBtP%2BMCot0sFKM8A$

Two errors:
You should do from time import time instead of just time. Because the time module has a time function inside it.
math.ceil returns a float and you are trying to use it as a dict after:
expires = math.ceil(time() + 3000) ["expires"]
This doesn't make sense. math.ceil(time() + 3000) will be equal to something like 1400406364 and you can't retrieve a data from it.
Removing the ["expires"] should solve the problem.

The time module is not callable, you need to call time method from it:
>>> import time
>>> import math
>>> math.ceil(time())
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: 'module' object is not callable
>>> math.ceil(time.time())
1400657920.0
Then you need to get rid of ["expires"] after it, since it will return a float number not a dictionary.
I don't know why you are using cURL here, with requests your code is a lot simpler:
import time
import math
import urllib
import requests
url = 'http://api.videoplatform.limelight.com/rest/organizations/{}/media'
filename = 'foo/bar/zoo.txt'
params = {}
params['access_key'] = 'dfdfdeef'
params['expires'] = math.ceil(time.time()+3000)
url = '{}?{}'.format(url.format(org_id), urllib.urlquote(params))
payload = {}
payload['title'] = os.path.basename(filename)
payload['description'] = 'description'
file_data = {'media_file': open(filename, 'rb')}
result = requests.post(url, data=payload, files=file_data)
result.raise_for_status() # This will raise an exception if
# there is a problem with the request

Python Invalid Snytax

Below is the code I have been working on.
The very last line write_csv('twitter_gmail.csv', messages, append=True) throws a
[ec2-user#ip-172-31-46-164 ~]$ ./twitter_test16.sh
Traceback (most recent call last):
File "./twitter_test16.sh", line 53, in
write_csv('twitter_gmail.csv', messages, append=True)
NameError: name 'messages' is not defined
I have messages defined so I dont understand why it would do that.
import csv
import json
import oauth2 as oauth
import urllib
import sys
import requests
import time
CONSUMER_KEY = "
CONSUMER_SECRET = "
ACCESS_KEY = "
ACCESS_SECRET = "
class TwitterSearch:
def __init__(self, ckey=CONSUMER_KEY, csecret=CONSUMER_SECRET,
akey=ACCESS_KEY, asecret=ACCESS_SECRET,
query='https://api.twitter.com/1.1/search/tweets.{mode}?{query}'
):
consumer = oauth.Consumer(key=ckey, secret=csecret)
access_token = oauth.Token(key=akey, secret=asecret)
self.client = oauth.Client(consumer, access_token)
self.query = query
def search(self, q, mode='json', **queryargs):
queryargs['q'] = q
query = urllib.urlencode(queryargs)
return self.client.request(self.query.format(query=query, mode=mode))
def write_csv(fname, rows, header=None, append=False, **kwargs):
filemode = 'ab' if append else 'wb'
with open(fname, filemode) as outf:
out_csv = csv.writer(outf, **kwargs)
if header:
out_csv.writerow(header)
out_csv.writerows(rows)
def main():
ts = TwitterSearch()
response, data = ts.search('#gmail.com', result_type='recent')
js = json.loads(data)
messages = ([msg['created_at'], msg['txt'], msg['user']['id']] \
for msg in js.get('statuses', []))
write_csv('twitter_gmail.csv', messages, append=True)

The previous line is missing a parenthesis.
messages = ([msg['created_at'], msg['txt'], msg['user']['id']] for msg in js.get('statuses', [])
Should be:
messages = ([msg['created_at'], msg['txt'], msg['user']['id']] for msg in js.get('statuses', []))
I'm surprised that it works when you change to print? Are you also changing the comprehension when you do that?
You asked why the line number of the error was after the bad syntax?
Try putting this in line one of a file and running it, and note the line of the SyntaxError.
a = (]
Then try this and check out the line number:
a = (
b = "some stuff"
Finally, try this:
a = (
b = "some stuff"
Think about when you would know that the programmer had made a python-illegal typo if you were reading the code and carrying it out via pen and paper.
Basically, a SyntaxError is raised as soon as it can be unambiguously determined that invalid syntax was used, which is often immediately after a statement where a mistake was made, not immediately at.
You'll frequently get line numbers on SyntaxErrors that are a line (or several lines if there's empty lines or a corner case) below the actual typo.