Related
I have about 100 spiders on a server. Every morning all spiders start scraping and writing all of the logs in their logs. Sometimes a couple of them gives me an error. When a spider gives me an error I have to go to the server and read from log file but I want to read the logs from the mail.
I already set dynamic mail sender as follow:
class FirstBotSpiderMiddleware:
def __init__(self, stats):
self.stats = stats
#classmethod
def from_crawler(cls, crawler):
s = cls(crawler.stats)
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
return s
def process_spider_input(self, response, spider):
return None
def process_spider_output(self, response, result, spider):
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
pass
def process_start_requests(self, start_requests, spider):
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
def spider_closed(self, spider,reason):
error_count = self.stats.get_value('log_count/ERROR')
counts = self.stats.get_value('item_scraped_count')
count_403 = self.stats.get_value('downloader/response_status_count/403')
count_404 = self.stats.get_value('downloader/response_status_count/404')
robots_404 = self.stats.get_value('robotstxt/response_status_count/404')
robots_403 = self.stats.get_value('robotstxt/response_status_count/403')
duplicate_count = self.stats.get_value('item_dropped_count')
#I want to read all logs here
content = "some stat string"
self.mailSender(spider.name,content,logs)
def mailSender(self,spider,content,logs):
send_mail(
"Scrapy "+spider+" done",
content,
djsettings.EMAIL_HOST_USER,
['xxx#xxx.com'],
)
I couldn't figure out how to read the error log at spider_closed on middleware dynamically. Do you have any suggestions?
I have implemented a similar method in my web scraping module.
Below is the implementation you can look at and take reference from.
import gzip
import datetime
from scrapy import signals
from scrapy.mail import MailSender
from scrapy.exceptions import NotConfigured
from scrapy.utils.serialize import ScrapyJSONEncoder
from collections import defaultdict
try:
from cStringIO import cStringIO as StringIO
except ImportError:
from io import StringIO
def format_size(size):
for x in ['bytes', 'KB', 'MB', 'GB']:
if size < 1024.0:
return "%3.1f %s" % (size, x)
size /= 1024.0
class GzipCompressor(gzip.GzipFile):
extension = '.gz'
mimetype = 'application/gzip'
def __init__(self):
super(GzipCompressor, self).__init__(
fileobj=PlainCompressor(), mode='w')
self.read = self.fileobj.read
class PlainCompressor(StringIO):
extension = ''
mimetype = 'text/plain'
def read(self, *args, **kwargs):
self.seek(0)
return StringIO.read(self, *args, **kwargs)
#property
def size(self):
return len(self.getvalue())
class StatusMailer(object):
def __init__(self, recipients, mail, compressor, crawler):
self.recipients = recipients
self.mail = mail
self.encoder = ScrapyJSONEncoder()
self.files = defaultdict(compressor)
self.num_items = 0
self.num_errors = 0
self.start_time = datetime.datetime.now()
#classmethod
def from_crawler(cls, crawler):
recipients = crawler.settings.getlist('STATUSMAILER_RECIPIENTS')
compression = crawler.settings.get('STATUSMAILER_COMPRESSION')
if not compression:
compressor = PlainCompressor
elif compression.lower().startswith('gz'):
compressor = GzipCompressor
else:
raise NotConfigured
if not recipients:
raise NotConfigured
mail = MailSender.from_settings(crawler.settings)
instance = cls(recipients, mail, compressor, crawler)
crawler.signals.connect(instance.item_scraped,
signal=signals.item_scraped)
crawler.signals.connect(instance.spider_error,
signal=signals.spider_error)
crawler.signals.connect(instance.spider_closed,
signal=signals.spider_closed)
return instance
def item_scraped(self, item, response, spider):
self.num_items += 1
self.files[spider.name + '.log'].write(str(self.num_items)+ " " + str(response.url) + '\n')
self.files[spider.name +
'-items.json'].write(self.encoder.encode(item))
def spider_error(self, failure, response, spider):
self.files[spider.name + '.log'].write(failure.getTraceback())
self.num_errors += 1
def spider_closed(self, spider, reason):
files = []
for name, compressed in self.files.items():
files.append((name + compressed.extension,
compressed.mimetype, compressed))
try:
size = self.files[spider.name + '-items.json'].size
except KeyError:
size = 0
body = '''Crawl statistics:
- Spider name: {0}
- Spider started at: {1}
- Spider finished at: {2}
- Number of items scraped: {3}
- Number of errors: {4}
- Size of scraped items: {5}'''.format(
spider.name,
self.start_time,
datetime.datetime.now(),
self.num_items,
self.num_errors,
format_size(size)
)
return self.mail.send(
to=self.recipients,
subject='Crawler for %s: %s' % (spider.name, reason),
body=body,
attachs=files
)
I tried the GitHub code to make a USB connection with an iOS device and Python on a PC.
The above code was created in Python2, so I ran 2to3 -w usbmux.py to convert it to Python3 code.
The Python3 code is as follows.
import socket, struct, select, sys
try:
import plistlib
haveplist = True
except:
haveplist = False
class MuxError(Exception):
pass
class MuxVersionError(MuxError):
pass
class SafeStreamSocket:
def __init__(self, address, family):
self.sock = socket.socket(family, socket.SOCK_STREAM)
self.sock.connect(address)
def send(self, msg):
totalsent = 0
while totalsent < len(msg):
sent = self.sock.send(msg[totalsent:])
if sent == 0:
raise MuxError("socket connection broken")
totalsent = totalsent + sent
def recv(self, size):
msg = ''
while len(msg) < size:
chunk = self.sock.recv(size-len(msg))
if chunk == '':
raise MuxError("socket connection broken")
msg = msg + chunk
return msg
class MuxDevice(object):
def __init__(self, devid, usbprod, serial, location):
self.devid = devid
self.usbprod = usbprod
self.serial = serial
self.location = location
def __str__(self):
return "<MuxDevice: ID %d ProdID 0x%04x Serial '%s' Location 0x%x>"%(self.devid, self.usbprod, self.serial, self.location)
class BinaryProtocol(object):
TYPE_RESULT = 1
TYPE_CONNECT = 2
TYPE_LISTEN = 3
TYPE_DEVICE_ADD = 4
TYPE_DEVICE_REMOVE = 5
VERSION = 0
def __init__(self, socket):
self.socket = socket
self.connected = False
def _pack(self, req, payload):
if req == self.TYPE_CONNECT:
return struct.pack("IH", payload['DeviceID'], payload['PortNumber']) + "\x00\x00"
elif req == self.TYPE_LISTEN:
return ""
else:
raise ValueError("Invalid outgoing request type %d"%req)
def _unpack(self, resp, payload):
if resp == self.TYPE_RESULT:
return {'Number':struct.unpack("I", payload)[0]}
elif resp == self.TYPE_DEVICE_ADD:
devid, usbpid, serial, pad, location = struct.unpack("IH256sHI", payload)
serial = serial.split("\0")[0]
return {'DeviceID': devid, 'Properties': {'LocationID': location, 'SerialNumber': serial, 'ProductID': usbpid}}
elif resp == self.TYPE_DEVICE_REMOVE:
devid = struct.unpack("I", payload)[0]
return {'DeviceID': devid}
else:
raise MuxError("Invalid incoming request type %d"%req)
def sendpacket(self, req, tag, payload={}):
payload = self._pack(req, payload)
if self.connected:
raise MuxError("Mux is connected, cannot issue control packets")
length = 16 + len(payload)
data = struct.pack("IIII", length, self.VERSION, req, tag) + payload
self.socket.send(data)
def getpacket(self):
if self.connected:
raise MuxError("Mux is connected, cannot issue control packets")
dlen = self.socket.recv(4)
dlen = struct.unpack("I", dlen)[0]
body = self.socket.recv(dlen - 4)
version, resp, tag = struct.unpack("III",body[:0xc])
if version != self.VERSION:
raise MuxVersionError("Version mismatch: expected %d, got %d"%(self.VERSION,version))
payload = self._unpack(resp, body[0xc:])
return (resp, tag, payload)
class PlistProtocol(BinaryProtocol):
TYPE_RESULT = "Result"
TYPE_CONNECT = "Connect"
TYPE_LISTEN = "Listen"
TYPE_DEVICE_ADD = "Attached"
TYPE_DEVICE_REMOVE = "Detached" #???
TYPE_PLIST = 8
VERSION = 1
def __init__(self, socket):
if not haveplist:
raise Exception("You need the plistlib module")
BinaryProtocol.__init__(self, socket)
def _pack(self, req, payload):
return payload
def _unpack(self, resp, payload):
return payload
def sendpacket(self, req, tag, payload={}):
payload['ClientVersionString'] = 'usbmux.py by marcan'
if isinstance(req, int):
req = [self.TYPE_CONNECT, self.TYPE_LISTEN][req-2]
payload['MessageType'] = req
payload['ProgName'] = 'tcprelay'
BinaryProtocol.sendpacket(self, self.TYPE_PLIST, tag, plistlib.writePlistToString(payload))
def getpacket(self):
resp, tag, payload = BinaryProtocol.getpacket(self)
if resp != self.TYPE_PLIST:
raise MuxError("Received non-plist type %d"%resp)
payload = plistlib.readPlistFromString(payload)
return payload['MessageType'], tag, payload
class MuxConnection(object):
def __init__(self, socketpath, protoclass):
self.socketpath = socketpath
if sys.platform in ['win32', 'cygwin']:
family = socket.AF_INET
address = ('127.0.0.1', 27015)
else:
family = socket.AF_UNIX
address = self.socketpath
self.socket = SafeStreamSocket(address, family)
self.proto = protoclass(self.socket)
self.pkttag = 1
self.devices = []
def _getreply(self):
while True:
resp, tag, data = self.proto.getpacket()
if resp == self.proto.TYPE_RESULT:
return tag, data
else:
raise MuxError("Invalid packet type received: %d"%resp)
def _processpacket(self):
resp, tag, data = self.proto.getpacket()
if resp == self.proto.TYPE_DEVICE_ADD:
self.devices.append(MuxDevice(data['DeviceID'], data['Properties']['ProductID'], data['Properties']['SerialNumber'], data['Properties']['LocationID']))
elif resp == self.proto.TYPE_DEVICE_REMOVE:
for dev in self.devices:
if dev.devid == data['DeviceID']:
self.devices.remove(dev)
elif resp == self.proto.TYPE_RESULT:
raise MuxError("Unexpected result: %d"%resp)
else:
raise MuxError("Invalid packet type received: %d"%resp)
def _exchange(self, req, payload={}):
mytag = self.pkttag
self.pkttag += 1
self.proto.sendpacket(req, mytag, payload)
recvtag, data = self._getreply()
if recvtag != mytag:
raise MuxError("Reply tag mismatch: expected %d, got %d"%(mytag, recvtag))
return data['Number']
def listen(self):
ret = self._exchange(self.proto.TYPE_LISTEN)
if ret != 0:
raise MuxError("Listen failed: error %d"%ret)
def process(self, timeout=None):
if self.proto.connected:
raise MuxError("Socket is connected, cannot process listener events")
rlo, wlo, xlo = select.select([self.socket.sock], [], [self.socket.sock], timeout)
if xlo:
self.socket.sock.close()
raise MuxError("Exception in listener socket")
if rlo:
self._processpacket()
def connect(self, device, port):
ret = self._exchange(self.proto.TYPE_CONNECT, {'DeviceID':device.devid, 'PortNumber':((port<<8) & 0xFF00) | (port>>8)})
if ret != 0:
raise MuxError("Connect failed: error %d"%ret)
self.proto.connected = True
return self.socket.sock
def close(self):
self.socket.sock.close()
class USBMux(object):
def __init__(self, socketpath=None):
if socketpath is None:
if sys.platform == 'darwin':
socketpath = "/var/run/usbmuxd"
else:
socketpath = "/var/run/usbmuxd"
self.socketpath = socketpath
self.listener = MuxConnection(socketpath, BinaryProtocol)
try:
self.listener.listen()
self.version = 0
self.protoclass = BinaryProtocol
except MuxVersionError:
self.listener = MuxConnection(socketpath, PlistProtocol)
self.listener.listen()
self.protoclass = PlistProtocol
self.version = 1
self.devices = self.listener.devices
def process(self, timeout=None):
self.listener.process(timeout)
def connect(self, device, port):
connector = MuxConnection(self.socketpath, self.protoclass)
return connector.connect(device, port)
if __name__ == "__main__":
mux = USBMux()
print("Waiting for devices...")
if not mux.devices:
mux.process(0.1)
while True:
print("Devices:")
for dev in mux.devices:
print(dev)
mux.process()
I ran the above code in Python 3.7 and got the error.
The details of the error are as follows.
File "C:\Users\taichi\Documents\usbmux.py", line 81, in sendpacket
data = struct.pack("IIII", length, self.VERSION, req, tag) + payload
TypeError: can't concat str to bytes
How can I rewrite this code to get rid of the error?
struct.pack returns bytes in python 3, not a string as it did in python 2. Have a look at this answer to see one way to emulate the python 2 behavior.
Edit: actually you may need to do something slightly different, since it looks like socket.send wants bytes as input. So you'd need to convert the combined payload to bytes.
You could try changing the following code that seems to be populating the payload to return bytes in both cases. Notice the "b" added in the return values to specify bytes instead of str. You are hitting the TYPE_LISTEN case but should test the other case as well, as it looks like it had the same bug trying to concat str and bytes:
def _pack(self, req, payload):
if req == self.TYPE_CONNECT:
return struct.pack("IH", payload['DeviceID'], payload['PortNumber']) + b"\x00\x00"
elif req == self.TYPE_LISTEN:
return b""
else:
raise ValueError("Invalid outgoing request type %d"%req)
You'll likely need to edit _pack in PlistProtocol as well.
The line:
data = struct.pack("IIII", length, self.VERSION, req, tag) + payload
is causing the trouble. According to the struct.pack() docs, the method returns bytes. You need to convert your payload also into bytes so that the two can be concatenated, making the error go away.
Not sure what your payload is, but encoding the payload using something like 'utf-8' might do the trick. I believe a simple payload.encode() will work, since 'utf-8' is the default encoding. So, try:
data = struct.pack("IIII", length, self.VERSION, req, tag) + payload.encode()
# add this ^^^^^^^^^
This is second stage question of problem. I originaly asked here
, I forked pyrebase to implement onDisconnect feature, can you tell me how can I implement that on this module?
I know, it should be very easy change on this file but I stucked coz I am newbie at python. Should I put a new method in to Database class? or new class needed?
import requests
from requests import Session
from requests.exceptions import HTTPError
try:
from urllib.parse import urlencode, quote
except:
from urllib import urlencode, quote
import json
import math
from random import uniform
import time
from collections import OrderedDict
from sseclient import SSEClient
import threading
import socket
from oauth2client.service_account import ServiceAccountCredentials
from gcloud import storage
from requests.packages.urllib3.contrib.appengine import is_appengine_sandbox
from requests_toolbelt.adapters import appengine
import python_jwt as jwt
from Crypto.PublicKey import RSA
import datetime
def initialize_app(config):
return Firebase(config)
class Firebase:
""" Firebase Interface """
def __init__(self, config):
self.api_key = config["apiKey"]
self.auth_domain = config["authDomain"]
self.database_url = config["databaseURL"]
self.storage_bucket = config["storageBucket"]
self.credentials = None
self.requests = requests.Session()
if config.get("serviceAccount"):
scopes = [
'https://www.googleapis.com/auth/firebase.database',
'https://www.googleapis.com/auth/userinfo.email',
"https://www.googleapis.com/auth/cloud-platform"
]
service_account_type = type(config["serviceAccount"])
if service_account_type is str:
self.credentials = ServiceAccountCredentials.from_json_keyfile_name(config["serviceAccount"], scopes)
if service_account_type is dict:
self.credentials = ServiceAccountCredentials.from_json_keyfile_dict(config["serviceAccount"], scopes)
if is_appengine_sandbox():
# Fix error in standard GAE environment
# is releated to https://github.com/kennethreitz/requests/issues/3187
# ProtocolError('Connection aborted.', error(13, 'Permission denied'))
adapter = appengine.AppEngineAdapter(max_retries=3)
else:
adapter = requests.adapters.HTTPAdapter(max_retries=3)
for scheme in ('http://', 'https://'):
self.requests.mount(scheme, adapter)
def auth(self):
return Auth(self.api_key, self.requests, self.credentials)
def database(self):
return Database(self.credentials, self.api_key, self.database_url, self.requests)
def storage(self):
return Storage(self.credentials, self.storage_bucket, self.requests)
class Auth:
""" Authentication Service """
def __init__(self, api_key, requests, credentials):
self.api_key = api_key
self.current_user = None
self.requests = requests
self.credentials = credentials
def sign_in_with_email_and_password(self, email, password):
request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyPassword?key={0}".format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"email": email, "password": password, "returnSecureToken": True})
request_object = requests.post(request_ref, headers=headers, data=data)
raise_detailed_error(request_object)
self.current_user = request_object.json()
return request_object.json()
def create_custom_token(self, uid, additional_claims=None):
service_account_email = self.credentials.service_account_email
private_key = RSA.importKey(self.credentials._private_key_pkcs8_pem)
payload = {
"iss": service_account_email,
"sub": service_account_email,
"aud": "https://identitytoolkit.googleapis.com/google.identity.identitytoolkit.v1.IdentityToolkit",
"uid": uid
}
if additional_claims:
payload["claims"] = additional_claims
exp = datetime.timedelta(minutes=60)
return jwt.generate_jwt(payload, private_key, "RS256", exp)
def sign_in_with_custom_token(self, token):
request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyCustomToken?key={0}".format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"returnSecureToken": True, "token": token})
request_object = requests.post(request_ref, headers=headers, data=data)
raise_detailed_error(request_object)
return request_object.json()
def refresh(self, refresh_token):
request_ref = "https://securetoken.googleapis.com/v1/token?key={0}".format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"grantType": "refresh_token", "refreshToken": refresh_token})
request_object = requests.post(request_ref, headers=headers, data=data)
raise_detailed_error(request_object)
request_object_json = request_object.json()
# handle weirdly formatted response
user = {
"userId": request_object_json["user_id"],
"idToken": request_object_json["id_token"],
"refreshToken": request_object_json["refresh_token"]
}
return user
def get_account_info(self, id_token):
request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/getAccountInfo?key={0}".format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"idToken": id_token})
request_object = requests.post(request_ref, headers=headers, data=data)
raise_detailed_error(request_object)
return request_object.json()
def send_email_verification(self, id_token):
request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/getOobConfirmationCode?key={0}".format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"requestType": "VERIFY_EMAIL", "idToken": id_token})
request_object = requests.post(request_ref, headers=headers, data=data)
raise_detailed_error(request_object)
return request_object.json()
def send_password_reset_email(self, email):
request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/getOobConfirmationCode?key={0}".format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"requestType": "PASSWORD_RESET", "email": email})
request_object = requests.post(request_ref, headers=headers, data=data)
raise_detailed_error(request_object)
return request_object.json()
def verify_password_reset_code(self, reset_code, new_password):
request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/resetPassword?key={0}".format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"oobCode": reset_code, "newPassword": new_password})
request_object = requests.post(request_ref, headers=headers, data=data)
raise_detailed_error(request_object)
return request_object.json()
def create_user_with_email_and_password(self, email, password):
request_ref = "https://www.googleapis.com/identitytoolkit/v3/relyingparty/signupNewUser?key={0}".format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8" }
data = json.dumps({"email": email, "password": password, "returnSecureToken": True})
request_object = requests.post(request_ref, headers=headers, data=data)
raise_detailed_error(request_object)
return request_object.json()
class Database:
""" Database Service """
def __init__(self, credentials, api_key, database_url, requests):
if not database_url.endswith('/'):
url = ''.join([database_url, '/'])
else:
url = database_url
self.credentials = credentials
self.api_key = api_key
self.database_url = url
self.requests = requests
self.path = ""
self.build_query = {}
self.last_push_time = 0
self.last_rand_chars = []
def order_by_key(self):
self.build_query["orderBy"] = "$key"
return self
def order_by_value(self):
self.build_query["orderBy"] = "$value"
return self
def order_by_child(self, order):
self.build_query["orderBy"] = order
return self
def start_at(self, start):
self.build_query["startAt"] = start
return self
def end_at(self, end):
self.build_query["endAt"] = end
return self
def equal_to(self, equal):
self.build_query["equalTo"] = equal
return self
def limit_to_first(self, limit_first):
self.build_query["limitToFirst"] = limit_first
return self
def limit_to_last(self, limit_last):
self.build_query["limitToLast"] = limit_last
return self
def shallow(self):
self.build_query["shallow"] = True
return self
def child(self, *args):
new_path = "/".join([str(arg) for arg in args])
if self.path:
self.path += "/{}".format(new_path)
else:
if new_path.startswith("/"):
new_path = new_path[1:]
self.path = new_path
return self
def build_request_url(self, token):
parameters = {}
if token:
parameters['auth'] = token
for param in list(self.build_query):
if type(self.build_query[param]) is str:
parameters[param] = quote('"' + self.build_query[param] + '"')
elif type(self.build_query[param]) is bool:
parameters[param] = "true" if self.build_query[param] else "false"
else:
parameters[param] = self.build_query[param]
# reset path and build_query for next query
request_ref = '{0}{1}.json?{2}'.format(self.database_url, self.path, urlencode(parameters))
self.path = ""
self.build_query = {}
return request_ref
def build_headers(self, token=None):
headers = {"content-type": "application/json; charset=UTF-8"}
if not token and self.credentials:
access_token = self.credentials.get_access_token().access_token
headers['Authorization'] = 'Bearer ' + access_token
return headers
def get(self, token=None, json_kwargs={}):
build_query = self.build_query
query_key = self.path.split("/")[-1]
request_ref = self.build_request_url(token)
# headers
headers = self.build_headers(token)
# do request
request_object = self.requests.get(request_ref, headers=headers)
raise_detailed_error(request_object)
request_dict = request_object.json(**json_kwargs)
# if primitive or simple query return
if isinstance(request_dict, list):
return PyreResponse(convert_list_to_pyre(request_dict), query_key)
if not isinstance(request_dict, dict):
return PyreResponse(request_dict, query_key)
if not build_query:
return PyreResponse(convert_to_pyre(request_dict.items()), query_key)
# return keys if shallow
if build_query.get("shallow"):
return PyreResponse(request_dict.keys(), query_key)
# otherwise sort
sorted_response = None
if build_query.get("orderBy"):
if build_query["orderBy"] == "$key":
sorted_response = sorted(request_dict.items(), key=lambda item: item[0])
elif build_query["orderBy"] == "$value":
sorted_response = sorted(request_dict.items(), key=lambda item: item[1])
else:
sorted_response = sorted(request_dict.items(), key=lambda item: item[1][build_query["orderBy"]])
return PyreResponse(convert_to_pyre(sorted_response), query_key)
def push(self, data, token=None, json_kwargs={}):
request_ref = self.check_token(self.database_url, self.path, token)
self.path = ""
headers = self.build_headers(token)
request_object = self.requests.post(request_ref, headers=headers, data=json.dumps(data, **json_kwargs).encode("utf-8"))
raise_detailed_error(request_object)
return request_object.json()
def set(self, data, token=None, json_kwargs={}):
request_ref = self.check_token(self.database_url, self.path, token)
self.path = ""
headers = self.build_headers(token)
request_object = self.requests.put(request_ref, headers=headers, data=json.dumps(data, **json_kwargs).encode("utf-8"))
raise_detailed_error(request_object)
return request_object.json()
def update(self, data, token=None, json_kwargs={}):
request_ref = self.check_token(self.database_url, self.path, token)
self.path = ""
headers = self.build_headers(token)
request_object = self.requests.patch(request_ref, headers=headers, data=json.dumps(data, **json_kwargs).encode("utf-8"))
raise_detailed_error(request_object)
return request_object.json()
def remove(self, token=None):
request_ref = self.check_token(self.database_url, self.path, token)
self.path = ""
headers = self.build_headers(token)
request_object = self.requests.delete(request_ref, headers=headers)
raise_detailed_error(request_object)
return request_object.json()
def stream(self, stream_handler, token=None, stream_id=None):
request_ref = self.build_request_url(token)
return Stream(request_ref, stream_handler, self.build_headers, stream_id)
def check_token(self, database_url, path, token):
if token:
return '{0}{1}.json?auth={2}'.format(database_url, path, token)
else:
return '{0}{1}.json'.format(database_url, path)
def generate_key(self):
push_chars = '-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'
now = int(time.time() * 1000)
duplicate_time = now == self.last_push_time
self.last_push_time = now
time_stamp_chars = [0] * 8
for i in reversed(range(0, 8)):
time_stamp_chars[i] = push_chars[now % 64]
now = int(math.floor(now / 64))
new_id = "".join(time_stamp_chars)
if not duplicate_time:
for i in range(0, 12):
self.last_rand_chars.append(int(math.floor(uniform(0, 1) * 64)))
else:
for i in range(0, 11):
if self.last_rand_chars[i] == 63:
self.last_rand_chars[i] = 0
self.last_rand_chars[i] += 1
for i in range(0, 12):
new_id += push_chars[self.last_rand_chars[i]]
return new_id
def sort(self, origin, by_key):
# unpack pyre objects
pyres = origin.each()
new_list = []
for pyre in pyres:
new_list.append(pyre.item)
# sort
data = sorted(dict(new_list).items(), key=lambda item: item[1][by_key])
return PyreResponse(convert_to_pyre(data), origin.key())
class Storage:
""" Storage Service """
def __init__(self, credentials, storage_bucket, requests):
self.storage_bucket = "https://firebasestorage.googleapis.com/v0/b/" + storage_bucket
self.credentials = credentials
self.requests = requests
self.path = ""
if credentials:
client = storage.Client(credentials=credentials, project=storage_bucket)
self.bucket = client.get_bucket(storage_bucket)
def child(self, *args):
new_path = "/".join(args)
if self.path:
self.path += "/{}".format(new_path)
else:
if new_path.startswith("/"):
new_path = new_path[1:]
self.path = new_path
return self
def put(self, file, token=None):
# reset path
path = self.path
self.path = None
if isinstance(file, str):
file_object = open(file, 'rb')
else:
file_object = file
request_ref = self.storage_bucket + "/o?name={0}".format(path)
if token:
headers = {"Authorization": "Firebase " + token}
request_object = self.requests.post(request_ref, headers=headers, data=file_object)
raise_detailed_error(request_object)
return request_object.json()
elif self.credentials:
blob = self.bucket.blob(path)
if isinstance(file, str):
return blob.upload_from_filename(filename=file)
else:
return blob.upload_from_file(file_obj=file)
else:
request_object = self.requests.post(request_ref, data=file_object)
raise_detailed_error(request_object)
return request_object.json()
def delete(self, name):
self.bucket.delete_blob(name)
def download(self, filename, token=None):
# remove leading backlash
path = self.path
url = self.get_url(token)
self.path = None
if path.startswith('/'):
path = path[1:]
if self.credentials:
blob = self.bucket.get_blob(path)
blob.download_to_filename(filename)
else:
r = requests.get(url, stream=True)
if r.status_code == 200:
with open(filename, 'wb') as f:
for chunk in r:
f.write(chunk)
def get_url(self, token):
path = self.path
self.path = None
if path.startswith('/'):
path = path[1:]
if token:
return "{0}/o/{1}?alt=media&token={2}".format(self.storage_bucket, quote(path, safe=''), token)
return "{0}/o/{1}?alt=media".format(self.storage_bucket, quote(path, safe=''))
def list_files(self):
return self.bucket.list_blobs()
def raise_detailed_error(request_object):
try:
request_object.raise_for_status()
except HTTPError as e:
# raise detailed error message
# TODO: Check if we get a { "error" : "Permission denied." } and handle automatically
raise HTTPError(e, request_object.text)
def convert_to_pyre(items):
pyre_list = []
for item in items:
pyre_list.append(Pyre(item))
return pyre_list
def convert_list_to_pyre(items):
pyre_list = []
for item in items:
pyre_list.append(Pyre([items.index(item), item]))
return pyre_list
class PyreResponse:
def __init__(self, pyres, query_key):
self.pyres = pyres
self.query_key = query_key
def val(self):
if isinstance(self.pyres, list):
# unpack pyres into OrderedDict
pyre_list = []
# if firebase response was a list
if isinstance(self.pyres[0].key(), int):
for pyre in self.pyres:
pyre_list.append(pyre.val())
return pyre_list
# if firebase response was a dict with keys
for pyre in self.pyres:
pyre_list.append((pyre.key(), pyre.val()))
return OrderedDict(pyre_list)
else:
# return primitive or simple query results
return self.pyres
def key(self):
return self.query_key
def each(self):
if isinstance(self.pyres, list):
return self.pyres
class Pyre:
def __init__(self, item):
self.item = item
def val(self):
return self.item[1]
def key(self):
return self.item[0]
class KeepAuthSession(Session):
"""
A session that doesn't drop Authentication on redirects between domains.
"""
def rebuild_auth(self, prepared_request, response):
pass
class ClosableSSEClient(SSEClient):
def __init__(self, *args, **kwargs):
self.should_connect = True
super(ClosableSSEClient, self).__init__(*args, **kwargs)
def _connect(self):
if self.should_connect:
super(ClosableSSEClient, self)._connect()
else:
raise StopIteration()
def close(self):
self.should_connect = False
self.retry = 0
self.resp.raw._fp.fp.raw._sock.shutdown(socket.SHUT_RDWR)
self.resp.raw._fp.fp.raw._sock.close()
class Stream:
def __init__(self, url, stream_handler, build_headers, stream_id):
self.build_headers = build_headers
self.url = url
self.stream_handler = stream_handler
self.stream_id = stream_id
self.sse = None
self.thread = None
self.start()
def make_session(self):
"""
Return a custom session object to be passed to the ClosableSSEClient.
"""
session = KeepAuthSession()
return session
def start(self):
self.thread = threading.Thread(target=self.start_stream)
self.thread.start()
return self
def start_stream(self):
self.sse = ClosableSSEClient(self.url, session=self.make_session(), build_headers=self.build_headers)
for msg in self.sse:
if msg:
msg_data = json.loads(msg.data)
msg_data["event"] = msg.event
if self.stream_id:
msg_data["stream_id"] = self.stream_id
self.stream_handler(msg_data)
def close(self):
while not self.sse and not hasattr(self.sse, 'resp'):
time.sleep(0.001)
self.sse.running = False
self.sse.close()
self.thread.join()
return self
Thanks.
I'm using the pyvona package (from July 3rd 2016). I have all dependencies installed. It works correctly when I call it to speak for the first time. But if I run the command again, it gives me the local unbound error:
>>> import pyvona
>>> v = pyvona.create_voice('<key>', '<secret>')
>>> v.speak('hello')
>>> v.speak('hello')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Python34\lib\site-packages\pyvona.py", line 159, in speak
channel.play(sound)
UnboundLocalError: local variable 'channel' referenced before assignment
>>>
Here's the pyvona.py script:
#!/usr/bin/env python
# encoding: utf-8
"""Pyvona : an IVONA python library
Author: Zachary Bears
Contact Email: bears.zachary#gmail.com
Note: Full operation of this library requires the requests and pygame libraries
"""
import datetime
import hashlib
import hmac
import json
import tempfile
import contextlib
import os
class PyvonaException(Exception):
pass
try:
import pygame
except ImportError:
pygame_available = False
else:
pygame_available = True
try:
import requests
requests.packages.urllib3.disable_warnings()
except ImportError:
msg = 'The requests library is essential for Pyvona operation. '
msg += 'Without it, Pyvona will not function correctly.'
raise PyvonaException(msg)
_amazon_date_format = '%Y%m%dT%H%M%SZ'
_date_format = '%Y%m%d'
def create_voice(access_key, secret_key):
"""Creates and returns a voice object to interact with
"""
return Voice(access_key, secret_key)
class Voice(object):
"""An object that contains all the required methods for interacting
with the IVONA text-to-speech system
"""
voice_name = None
language = None
gender = None
speech_rate = None
sentence_break = None
paragraph_break = None
_codec = "ogg"
region_options = {
'us-east': 'us-east-1',
'us-west': 'us-west-2',
'eu-west': 'eu-west-1',
}
access_key = None
secret_key = None
algorithm = 'AWS4-HMAC-SHA256'
signed_headers = 'content-type;host;x-amz-content-sha256;x-amz-date'
_region = None
_host = None
#property
def region(self):
return self._region
#region.setter
def region(self, region_name):
self._region = self.region_options.get(region_name, 'us-east-1')
self._host = 'tts.{}.ivonacloud.com'.format(self._region)
#property
def codec(self):
return self._codec
#codec.setter
def codec(self, codec):
if codec not in ["mp3", "ogg"]:
raise PyvonaException(
"Invalid codec specified. Please choose 'mp3' or 'ogg'")
self._codec = codec
#contextlib.contextmanager
def use_ogg_codec(self):
current_codec = self.codec
self.codec = "ogg"
try:
yield
finally:
self.codec = current_codec
def fetch_voice_ogg(self, text_to_speak, filename):
"""Fetch an ogg file for given text and save it to the given file name
"""
with self.use_ogg_codec():
self.fetch_voice(text_to_speak, filename)
def fetch_voice(self, text_to_speak, filename):
"""Fetch a voice file for given text and save it to the given file name
"""
file_extension = ".{codec}".format(codec=self.codec)
filename += file_extension if not filename.endswith(
file_extension) else ""
with open(filename, 'wb') as f:
self.fetch_voice_fp(text_to_speak, f)
def fetch_voice_fp(self, text_to_speak, fp):
"""Fetch a voice file for given text and save it to the given file pointer
"""
r = self._send_amazon_auth_packet_v4(
'POST', 'tts', 'application/json', '/CreateSpeech', '',
self._generate_payload(text_to_speak), self._region, self._host)
if r.content.startswith(b'{'):
raise PyvonaException('Error fetching voice: {}'.format(r.content))
else:
fp.write(r.content)
def speak(self, text_to_speak, use_cache=False):
"""Speak a given text
"""
if not pygame_available:
raise PyvonaException(
"Pygame not installed. Please install to use speech.")
if not pygame.mixer.get_init():
pygame.mixer.init()
channel = pygame.mixer.Channel(5)
if use_cache is False:
with tempfile.SpooledTemporaryFile() as f:
with self.use_ogg_codec():
self.fetch_voice_fp(text_to_speak, f)
f.seek(0)
sound = pygame.mixer.Sound(f)
else:
cache_f = hashlib.md5(text_to_speak).hexdigest() + '.ogg'
speech_cache_dir = os.getcwd() + '/speech_cache/'
if not os.path.isdir(speech_cache_dir):
os.makedirs(speech_cache_dir)
if not os.path.isfile(speech_cache_dir + cache_f):
with self.use_ogg_codec():
self.fetch_voice(text_to_speak, 'speech_cache/' + cache_f)
f = speech_cache_dir + cache_f
sound = pygame.mixer.Sound(f)
channel.play(sound)
while channel.get_busy():
pass
def list_voices(self):
"""Returns all the possible voices
"""
r = self._send_amazon_auth_packet_v4(
'POST', 'tts', 'application/json', '/ListVoices', '', '',
self._region, self._host)
return r.json()
def _generate_payload(self, text_to_speak):
return json.dumps({
'Input': {
"Type":"application/ssml+xml",
'Data': text_to_speak
},
'OutputFormat': {
'Codec': self.codec.upper()
},
'Parameters': {
'Rate': self.speech_rate,
'SentenceBreak': self.sentence_break,
'ParagraphBreak': self.paragraph_break
},
'Voice': {
'Name': self.voice_name,
'Language': self.language,
'Gender': self.gender
}
})
def _send_amazon_auth_packet_v4(self, method, service, content_type,
canonical_uri, canonical_querystring,
request_parameters, region, host):
"""Send a packet to a given amazon server using Amazon's signature Version 4,
Returns the resulting response object
"""
# Create date for headers and the credential string
t = datetime.datetime.utcnow()
amazon_date = t.strftime(_amazon_date_format)
date_stamp = t.strftime(_date_format)
# Step 1: Create canonical request
payload_hash = self._sha_hash(request_parameters)
canonical_headers = 'content-type:{}\n'.format(content_type)
canonical_headers += 'host:{}\n'.format(host)
canonical_headers += 'x-amz-content-sha256:{}\n'.format(payload_hash)
canonical_headers += 'x-amz-date:{}\n'.format(amazon_date)
canonical_request = '\n'.join([
method, canonical_uri, canonical_querystring, canonical_headers,
self.signed_headers, payload_hash])
# Step 2: Create the string to sign
credential_scope = '{}/{}/{}/aws4_request'.format(
date_stamp, region, service)
string_to_sign = '\n'.join([
self.algorithm, amazon_date, credential_scope,
self._sha_hash(canonical_request)])
# Step 3: Calculate the signature
signing_key = self._get_signature_key(
self.secret_key, date_stamp, region, service)
signature = hmac.new(
signing_key, string_to_sign.encode('utf-8'),
hashlib.sha256).hexdigest()
# Step 4: Create the signed packet
endpoint = 'https://{}{}'.format(host, canonical_uri)
authorization_header = '{} Credential={}/{}, ' +\
'SignedHeaders={}, Signature={}'
authorization_header = authorization_header.format(
self.algorithm, self.access_key, credential_scope,
self.signed_headers, signature)
headers = {
'Host': host,
'Content-type': content_type,
'X-Amz-Date': amazon_date,
'Authorization': authorization_header,
'x-amz-content-sha256': payload_hash,
'Content-Length': len(request_parameters)
}
# Send the packet and return the response
return requests.post(endpoint, data=request_parameters,
headers=headers)
def _sha_hash(self, to_hash):
return hashlib.sha256(to_hash.encode('utf-8')).hexdigest()
def _sign(self, key, msg):
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
def _get_signature_key(self, key, date_stamp, region_name, service_name):
k_date = self._sign(('AWS4{}'.format(key)).encode('utf-8'), date_stamp)
k_region = self._sign(k_date, region_name)
k_service = self._sign(k_region, service_name)
k_signing = self._sign(k_service, 'aws4_request')
return k_signing
def __init__(self, access_key, secret_key):
"""Set initial voice object parameters
"""
self.region = 'us-east'
self.voice_name = 'Brian'
self.access_key = access_key
self.secret_key = secret_key
self.speech_rate = 'medium'
self.sentence_break = 400
self.paragraph_break = 650
Any help is highly appreciated.
Was able to find a workaround by adding:
pygame.mixer.init()
channel = pygame.mixer.Channel(5)
immediately after defining the speak function:
def speak(self, text_to_speak, use_cache=False):
def speak(self, text_to_speak, use_cache=False):
"""Speak a given text
"""
pygame.mixer.init()
channel = pygame.mixer.Channel(5)
Usually UnboundLocalError relates to Scopes and Namespaces in Python Scopes and NameSpaces ,
but in Your case:
In function speak() You create channel in code
if not pygame.mixer.get_init():
pygame.mixer.init()
channel = pygame.mixer.Channel(5)
In case this code does not execute channel does not bound to any object.
For example, You can check this situation by this code sample:
def test_if():
if True:
# Bound
channel_1 = "Channel_1"
if False:
# Not bound
channel_2 = "Channel_2"
print(channel_1)
print(channel_2)
test_if()
I'm putting together a python Tinder bot that auto swipes right, and I also want it to send messages. Everything works properly except that when it matches, it should send a message, but doesn't. I've tried every possible combination of code I can think of, but to no avail. Any help with figuring this out would be greatly appreciate by me and all my future Tinder matches (if I even get any).
The code calls a funtcion called talk(user_id) that should perform a POST curl request.
Code:
1,1 Top# encoding: utf8
import argparse
from datetime import datetime
import json
from random import randint
import requests
import sys
from time import sleep
headers = {
'app_version': '519',
'platform': 'ios',
}
fb_id = 'XXXXXXXXXXXXX'
fb_auth_token = 'XXXXXXXXXXXX'
class User(object):
def __init__(self, data_dict):
self.d = data_dict
#property
def user_id(self):
return self.d['_id']
#property
def name(self):
return self.d['name']
#property
def ago(self):
raw = self.d.get('ping_time')
if raw:
d = datetime.strptime(raw, '%Y-%m-%dT%H:%M:%S.%fZ')
secs_ago = int(datetime.now().strftime("%s")) - int(d.strftime("%s"))
if secs_ago > 86400:
return u'{days} days ago'.format(days=secs_ago / 86400)
elif secs_ago < 3600:
return u'{mins} mins ago'.format(mins=secs_ago / 60)
else:
return u'{hours} hours ago'.format(hours=secs_ago / 3600)
return '[unknown]'
#property
def bio(self):
try:
x = self.d['bio'].encode('ascii', 'ignore').replace('\n', '')[:50].strip()
except (UnicodeError, UnicodeEncodeError, UnicodeDecodeError):
return '[garbled]'
else:
return x
#property
def age(self):
raw = self.d.get('birth_date')
if raw:
d = datetime.strptime(raw, '%Y-%m-%dT%H:%M:%S.%fZ')
return datetime.now().year - int(d.strftime('%Y'))
return 0
def __unicode__(self):
return u'{name} ({age}), {distance}km, {ago}'.format(
name=self.d['name'],
age=self.age,
distance=self.d['distance_mi'],
ago=self.ago
)
def auth_token(fb_auth_token, fb_user_id):
h = headers
h.update({'content-type': 'application/json'})
req = requests.post(
'https://api.gotinder.com/auth',
headers=h,
data=json.dumps({'facebook_token': fb_auth_token, 'facebook_id': fb_user_id})
)
try:
return req.json()['token']
except:
return None
def recommendations(auth_token):
h = headers
h.update({'X-Auth-Token': auth_token})
r = requests.get('https://api.gotinder.com/user/recs', headers=h)
if r.status_code == 401 or r.status_code == 504:
raise Exception('Invalid code')
print r.content
if not 'results' in r.json():
print r.json()
for result in r.json()['results']:
yield User(result)
def like(user_id):
try:
u = 'https://api.gotinder.com/like/%s' % user_id
d = requests.get(u, headers=headers, timeout=0.7).json()
except KeyError:
raise
else:
return d['match']
def nope(user_id):
try:
u = 'https://api.gotinder.com/pass/%s' % user_id
requests.get(u, headers=headers, timeout=0.7).json()
except KeyError:
raise
def like_or_nope():
return 'nope' if randint(1, 100) == 31 else 'like'
def talk(user_id):
try:
u = 'https://api.gotinder.com/user/matches/%s' % user_id
requests.post(
u,
data=json.dumps=({'message': 'Hey! How are you?'})
)
except KeyError:
raise
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Tinder automated bot')
parser.add_argument('-l', '--log', type=str, default='activity.log', help='Log file destination')
args = parser.parse_args()
print 'Tinder bot'
print '----------'
matches = 0
liked = 0
nopes = 0
while True:
token = auth_token(fb_auth_token, fb_id)
if not token:
print 'could not get token'
sys.exit(0)
for user in recommendations(token):
if not user:
break
print unicode(user)
if user.name == 'Tinder Team':
print('Out of swipes, pausing one hour...')
sleep(3601)
else:
try:
action = like_or_nope()
if action == 'like':
print ' -> Like'
match = like(user.user_id)
if match:
print ' -> Match!'
conversation = talk(user.user_id)
if conversation:
print ' -> Message Sent!'
with open('./matched.txt', 'a') as m:
m.write(user.user_id + u'\n')
with open('./liked.txt', 'a') as f:
f.write(user.user_id + u'\n')
else:
print ' -> random nope :('
nope(user.user_id)
except:
print 'networking error %s' % user.user_id
s = float(randint(10000, 20000) / 1000)
sleep(s)`
Check the below code if this helps. I used pynder package for dissecting the APIs, but I guess the logic remains the same.
session._post('/user/matches/' + match['id'], {"message": "Hey! How are you?"})
Where the post function works like below :-
def _post(self, url, data={}):
return self._request("post", url, data=data)
Try this method data={message} directly
It works for me, perfectly fine.