I am having problems with pycurl in conjunction with Twitter's Streaming API filter stream.
What is happening when I run the code below it seems to barf on the perform call. I know this because I placed print statements before and after the perform call. I am using Python 2.6.1 and I am on a Mac if that matters.
#!/usr/bin/python
print "Content-type: text/html"
print
import pycurl, json, urllib
STREAM_URL = "http://stream.twitter.com/1/statuses/filter.json?follow=1&count=100"
USER = "user"
PASS = "password"
print "<html><head></head><body>"
class Client:
def __init__(self):
self.buffer = ""
self.conn = pycurl.Curl()
self.conn.setopt(pycurl.POST,1)
self.conn.setopt(pycurl.USERPWD, "%s:%s" % (USER,PASS))
self.conn.setopt(pycurl.URL, STREAM_URL)
self.conn.setopt(pycurl.WRITEFUNCTION, self.on_receive)
try:
self.conn.perform()
self.conn.close()
except BaseException:
traceback.print_exc()
def on_receive(self,data):
self.buffer += data
if data.endswith("\r\n") and self.buffer.strip():
content = json.loads(self.buffer)
self.buffer = ""
print content
if "text" in content:
print u"{0[user][name]}: {0[text]}".format(content)
client = Client()
print "</body></html>"
First, try turning on verbosity to help debug:
self.conn.setopt(pycurl.VERBOSE ,1)
It looks like you aren't setting the basic auth mode:
self.conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
Also according to the documentation, you need to provide a POST of the parameters to the API, not pass them in as a GET parameter:
data = dict( track='stack overflow' )
self.conn.setopt(pycurl.POSTFIELDS,urlencode(data))
You are trying to use basic authentication.
Basic Authentication sends user
credentials in the header of the HTTP
request. This makes it easy to use,
but insecure. OAuth is the Twitter
preferred method of authentication
moving forward - come August 2010,
we'll be turning off Basic Auth from
the API. --Authentication, Twitter
Related
Hi folks I am new to gRPCs. I am currently building a client that can send requests to the server. I have hit a bit of a roadblock and cant figure out how to continue. I need to code into python the following gRPCurl call:
grpcurl -H "Authorization: Bearer aAbcdEefgHh123poIjuYHtgbNmCxxx" -H "x-team-mesh: TEST" -d '{"context":{"project_id":"this-is-my-project-id"}}' api.app.co.com:443 team.app.infrastructure.v3.App.envs_pb2_grpc.ListEnvs ../protobuf/app-api/team/app/infrastructure/v3/envs.proto
I have looked at examples such as these but they are not quite the same structure as my code. I cannot figure out how to pass in the header and data body when instantiating a channel on the client side.
How and where do I place the -H (header) and -d (body?) parts of the gRPCurl call?
import grpc
import team.app.infrastructure.v3.branches_pb2 as pb2
import team.app.infrastructure.v3.branches_pb2_grpc as pb2_grpc
import google.api
import sys
from okta.token_auth import OktaToken
class AppAuth(grpc.AuthMetadataPlugin):
def __init__(self, key):
self._key = key
def __call__(self, context, callback):
callback((('rpc-auth-header', self._key),), None)
class AppClient(object):
def __init__(self):
self.server = 'api.app.co.com:443'
with open('auth/certs/DigiCertClientRSA4096RootG5.crt.pem', 'rb') as fh:
rootCert = fh.read()
new_token = OktaToken()
metadata = []
metadata.append(('context', 'project_id', 'this-is-my-project-id'))
metadata.append(('x-team-mesh', 'TEST'))
metadata.append(('authorization', 'Bearer ' + new_token.okta_token_request()))
self.channel = grpc.secure_channel('{}'.format(self.server),
grpc.composite_channel_credentials(
grpc.ssl_channel_credentials(),
grpc.ssl_channel_credentials(rootCert),
grpc.metadata_call_credentials(
BitcAuth(new_token.okta_token_request())
)
)
)
try:
grpc.channel_ready_future(self.channel).result(timeout=3)
except grpc.FutureTimeoutError:
sys.exit('Error connecting to server')
else:
self.stub = pb2_grpc.EnvsStub(self.channel)
def get_envs(self):
request = pb2.ListEnvsRequest()
response = self.stub.ListEnvs(request)
print(f'{response}')
return response
if __name__ == '__main__':
client = AppClient()
result = client.get_envs()
I suspect that this line may be where to make the necessary changes based on what I have read so far but all my attempts to add metadata and payload(?) have been unsuccessful: response = self.stub.ListEnvs(request).
Since the curl call works fine, and my python code does not Im pretty sure I am not passing on the right pieces in the gRPC API call:
custom error:
debug_error_string = "UNKNOWN:Error received from peer ipv4:127.0.0.1:443 {grpc_message:"RBAC: access denied", grpc_status:7, created_time:"2022-09-12T17:59:22.654641-07:00"}"
Help is greatly appreciated
I am trying to write a script to call the Watson Speech-to-Text (STT) API to continually transcribe speech being recorded through a microphone word-for-word in real-time. I read that this should be possible using the Websockets version of the API.
I have a Python script that should be able to do this on Linux (assuming the dependencies are installed), however, it does not work on Mac OS X.
from ws4py.client.threadedclient import WebSocketClient
import base64, json, ssl, subprocess, threading, time
class SpeechToTextClient(WebSocketClient):
def __init__(self):
ws_url = "wss://stream.watsonplatform.net/speech-to-text/api/v1/recognize"
username = "your username"
password = "your password"
auth_string = "%s:%s" % (username, password)
base64string = base64.encodestring(auth_string).replace("\n", "")
self.listening = False
try:
WebSocketClient.__init__(self, ws_url,
headers=[("Authorization", "Basic %s" % base64string)])
self.connect()
except: print "Failed to open WebSocket."
def opened(self):
self.send('{"action": "start", "content-type": "audio/l16;rate=16000"}')
self.stream_audio_thread = threading.Thread(target=self.stream_audio)
self.stream_audio_thread.start()
def received_message(self, message):
message = json.loads(str(message))
if "state" in message:
if message["state"] == "listening":
self.listening = True
print "Message received: " + str(message)
def stream_audio(self):
while not self.listening:
time.sleep(0.1)
reccmd = ["arecord", "-f", "S16_LE", "-r", "16000", "-t", "raw"]
p = subprocess.Popen(reccmd, stdout=subprocess.PIPE)
while self.listening:
data = p.stdout.read(1024)
try: self.send(bytearray(data), binary=True)
except ssl.SSLError: pass
p.kill()
def close(self):
self.listening = False
self.stream_audio_thread.join()
WebSocketClient.close(self)
try:
stt_client = SpeechToTextClient()
raw_input()
finally:
stt_client.close()
Ideally, I wouldn't even be doing this in Python, but R, which is my native language which I will have to transfer the results back to to be processed anyway.
Could anyone provide me with a solution for how I can get a streamed transcription?
Not sure if this answer is exactly what you want, but sounds like an issue with the parameter continuous.
As you can see, has the lib Python SDK inside Watson-developer-cloud.
You can install with: pip install watson-developer-cloud
import json
from os.path import join, dirname
from watson_developer_cloud import SpeechToTextV1
speech_to_text = SpeechToTextV1(
username='YOUR SERVICE USERNAME',
password='YOUR SERVICE PASSWORD',
x_watson_learning_opt_out=False
)
print(json.dumps(speech_to_text.models(), indent=2))
print(json.dumps(speech_to_text.get_model('en-US_BroadbandModel'), indent=2))
with open(join(dirname(__file__), '../resources/speech.wav'),
'rb') as audio_file:
data = json.dumps(speech_to_text.recognize(audio_file, content_type='audio/wav', timestamps=False, word_confidence=False, continuous=True), indent=2)
print(data)
Obs.: The service returns an array of results with one per utterance.
In the line #L44, has the params that you can use, so, for continuous transcription, you need to use the parameter continuous and set to true like the example above.
See the Official Documentation talking about Websockets for keeping the connection alive. (Maybe this is what you need).
For some good examples of how to do this using R, check out these great blog posts by Ryan Anderson.
Voice Controlled Music Machine
Python as a tool to help with Continuous Audio - This shows how you can use R for your main logic, and use Python to deal with handling just the audio.
Ryan has done a lot of work with R and the Watson API's - he shares a lot of his knowledge on his blog.
I'm trying to make a universal script in Python that can be used by anybody to import/export all sorts of information from/to Work Etc CRM platform. It has all the documentation here: http://admin.worketc.com/xml.
However, I am now a bit stuck. Authentication works, I can call different API methods, but only the ones without parameters. I am new to Python and that's why I can't figure out how to pass the parameters onto that specific method in the API. Specifically I need to export all time sheets. I'm trying to call this method specifically: http://admin.worketc.com/xml?op=GetDraftTimesheets. For obvious reasons I cannot disclose the login information so it might be a bit hard to test for you.
The code itself:
import xml.etree.ElementTree as ET
import urllib2
import sys
email = 'email#domain.co.uk'
password = 'pass'
#service = 'GetEmployee?EntityID=1658'
#service = 'GetEntryID?EntryID=23354'
#service = ['GetAllCurrenciesWebSafe']
#service = ['GetEntryID', 'EntryID=23354']
service = ['GetDraftTimesheets','2005-08-15T15:52:01+00:00','2014-08-15T15:52:01+00:00' ]
class workEtcUniversal():
sessionkey = None
def __init__(self,url):
if not "http://" in url and not "https://" in url:
url = "http://%s" % url
self.base_url = url
else:
self.base_url = url
def authenticate(self, user, password):
try:
loginurl = self.base_url + email + '&pass=' + password
req = urllib2.Request(loginurl)
response = urllib2.urlopen(req)
the_page = response.read()
root = ET.fromstring(the_page)
sessionkey = root[1].text
print 'Authentication successful!'
try:
f = self.service(sessionkey, service)
except RuntimeError:
print 'Did not perform function!'
except RuntimeError:
print 'Error logging in or calling the service method!'
def service(self, sessionkey, service):
try:
if len(service)<2:
retrieveurl = 'https://domain.worketc.com/xml/' + service[0] + '?VeetroSession=' + sessionkey
else:
retrieveurl = 'https://domain.worketc.com/xml/' + service[0,1,2] + '?VeetroSession=' + sessionkey
except TypeError as err:
print 'Type Error, which means arguments are wrong (or wrong implementation)'
print 'Quitting..'
sys.exit()
try:
responsefile = urllib2.urlopen(retrieveurl)
except urllib2.HTTPError as err:
if err.code == 500:
print 'Internal Server Error: Permission Denied or Object (Service) Does Not Exist'
print 'Quitting..'
sys.exit()
elif err.code == 404:
print 'Wrong URL!'
print 'Quitting..'
sys.exit()
else:
raise
try:
f = open("ExportFolder/worketcdata.xml",'wb')
for line in responsefile:
f.write(line)
f.close()
print 'File has been saved into: ExportFolder'
except (RuntimeError,UnboundLocalError):
print 'Could not write into the file'
client = workEtcUniversal('https://domain.worketc.com/xml/AuthenticateWebSafe?email=')
client.authenticate(email, password)
Writing a code Consuming API requires resolving few questions:
what methods on API are available (get their list with names)
how does a request to such method looks like (find out url, HTTP method to use, requirements to body if used, what headers are expected)
how to build up all the parts to make the request
What methods are available
http://admin.worketc.com/xml lists many of them
How does a request looks like
GetDraftTimesheet is described here http://admin.worketc.com/xml?op=GetDraftTimesheets
and it expects you to create following HTTP request:
POST /xml HTTP/1.1
Host: admin.worketc.com
Content-Type: text/xml; charset=utf-8
Content-Length: length
SOAPAction: "http://schema.veetro.com/GetDraftTimesheets"
<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
<soap:Body>
<GetDraftTimesheets xmlns="http://schema.veetro.com">
<arg>
<FromUtc>dateTime</FromUtc>
<ToUtc>dateTime</ToUtc>
</arg>
</GetDraftTimesheets>
</soap:Body>
</soap:Envelope>
Building up the request
The biggest task it to build properly shaped XML document as shown above and having elements FromUtc and ToUtc filled with proper values. I guess, the values shall be in format of ISO datetime, this you shall find yourself.
You shall be able building such an XML by some Python library, I would use lxml.
Note, that the XML document is using namespaces, you have to handle them properly.
Making POST request with all the headers shall be easy. The library you use to make HTTP requests shall fill in properly Content-Length value, but this is mostly done automatically.
Veerto providing many alternative methods
E.g. for "http://admin.worketc.com/xml?op=FindArticlesWebSafe" there is set of different methods for the same service:
SOAP 1.1
SOAP 1.2
HTTP GET
HTTP POST
Depending on your preferences, pick the one which fits your needs.
The simplest is mostly HTTP GET.
For HTTP requests, I would recommend using requests, which are really easy to use, if you get through tutorial, you will understand what I mean.
I'm a bit new to twisted servers and servers in general. I need to have an asynchronous server running on my local retrieving data from twitter tweets (writing in a local file the data) and at the same time, I want to server some simple content such as "Hello world" at "local_ip:8880"
The problem is that the stream api creates a http connection and I dont know how to do the two tasks at the same time. My code look like this:
"imports and connection to twitter api keys"
def pp(o):
return json.dumps(o, indent=3)
class listener(StreamListener):
def on_data(self,data):
#Convert to JSON the input string
data_json = json.loads(data)
#print pp(data_json['user'])
#We get the html from embedly
tweet_id = data_json['id_str']
tweet_user = data_json['user']['screen_name']
tweet_url = "https://twitter.com/"+tweet_user+"/status/"+tweet_id
html="<a class=\"embedly-card\" href=\""+tweet_url+"\">Prueba</a>\n"
print "Datos recibidos en el listener."
#Write the results in the file
f = open('myfile.html','a')
f.write(html) # python will convert \n to os.linesep
f.close()
return True
def on_error(self,status):
print status
class Root(resource.Resource,StreamListener):
isLeaf = True
def __init__(self):
print "Server iniciado"
auth = OAuthHandler(CONSUMER_KEY,CONSUMER_SECRET)
auth.set_access_token(OAUTH_TOKEN,OAUTH_TOKEN_SECRET)
twitterStream = Stream(auth, listener())
twitterStream.filter(track=["query"])
def render_GET(self, request):
print "Recibida peticion"
return '<html><body>Hello world<body></html>'
root = Root()
site = server.Site(root)
reactor.listenTCP(8880, site)
reactor.run()
When I run this, i got stuck in the console, it receives the stream data, but when I access my "local_ip:8880" it cant load the "Hello world".
Is it possible to make this? Thanks in advance for attention and sorry for my English.
If you use blocking APIs - for example, a function that does network I/O using blocking sockets - then you will prevent your Twisted application from performing operations concurrently. Only one blocking operation can be in progress at a time.
Switch to a non-blocking Twitter API - for example, https://github.com/fiorix/twisted-twitter-stream
I am struggling getting a Rest API Post to work with a vendor api and hope someone can give me a pointer.
The intent is to feed a cli command to the post body and pass to a device which returns the output.
The call looks like this : ( this works for all other calls but this is different because of posting to body)
def __init__(self,host,username,password,sid,method,http_meth):
self.host=host
self.username= username
self.password= password
self.sid=sid
self.method=method
self.http_meth=http_meth
def __str__(self):
self.url = 'http://' + self.host + '/rest/'
self.authparams = urllib.urlencode({ "session_id":self.sid,"method": self.method,"username": self.username,
"password": self.password,
})
call = urllib2.urlopen(self.url.__str__(), self.authparams).read()
return (call)
No matter how I have tried this I cant get it to work properly. Here is an excerpt from the API docs that explains how to use this method:
To process these APIs, place your CLI commands in the HTTP post buffer, and then place the
method name, session ID, and other parameters in the URL.
Can anyone give me an idea of how to properly do this. I am not a developer and am trying to learn this correctly. For example if I wanted to send the command "help" in the post body?
Thanks for any guidance
Ok this was ridiculously simple and I was over-thinking this. I find that I can sometimes look at a much higher level than a problem really is and waist time. Anyway this is how it should work:
def cli(self,method):
self.url = ("http://" + str(self.host) + "/rest//?method=" +str(method)+ "&username=" +str(self.username)+ "&password=" +str(self.password)+ "&enable_password=test&session_id="+str(self.session_id))
data="show ver"
try:
req = urllib2.Request(self.url)
response = urllib2.urlopen(req,data)
result = response.read()
print result
except urllib2.URLError, e:
print e.reason
The cli commands are just placed in the buffer and not encoded....