emulate file-like behavior in python - python

I'm writing a script, where I have to dump some columns of tables from an SQL database into a file, and transfer it via FTP.
Because dumps can get really big, my Idea was to write a FakeFile which querys row by row from a cursor in its readline method and pass it to ftplib.FTP.storlines.
This is what I have so far:
import ftplib
import MySQLdb
def MySQLFakeFile(object):
'''
Simulates a read-only file, which dumps rows on demand.
Use this, to pass it to the FTP protocol to make the dump more efficient,
without the need to dump it somewhere and copy it over the net afterwords
'''
def __init__(self, cursor, delimeter, table_name, query):
self.cursor = cursor
self.delimeter = delimeter
self.table_name = table_name
#query is something along select ... FROM %s
self.cursor.execute(query, table_name)
self._has_written_index = False
#file attrs
self.closed = False
self.name = table_name + ".csv"
self.encoding = "utf-8"
self.mode = "r"
def close(self):
self.cursor.close()
self.closed = True
def flush(self):
'''No-OP'''
pass
def read(self, size):
pass
def readline(self, size):
if not self._has_written_index:
ret = []
for desc in self.cursor.description:
ret.append(desc[0])
self._has_written_index = True
else:
ret = self.cursor.fetchone()
if not ret:
return None
s = ""
for col in ret:
s += str(col) + self.delimeter
return s + "\n"
def readlines(self, size):
ret = []
line = self.readline()
while line:
ret.append(line)
line = self.readline()
def write(self, string):
raise Exception("cannot write to a MySQLFakeFile")
def writelines(self, lines)
raise Exception("cannot write to a MySQLFakeFile")
db = MySQLdb("host", "user", "pass", "db")
ftp = ftplib.FTP("host", "user", "pass")
fakeFile = MySQLFakeFile(db.cursor(), ";", "tableName", "SELECT * FROM %s")
ftp.storlines("STOR tableName.csv", fakeFile)
gives me
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python2.7/ftplib.py", line 496, in storlines
if len(buf) > self.maxline:
TypeError: object of type 'NoneType' has no len()
What am I doing wrong, and whats the NoneType here?

Your readline returns None instead of empty string "" when reached end-of-rows.
Your readlines returns nothing.
def readlines(self, size):
ret = []
while True:
line = self.readline()
if not line:
break
ret.append(line)
return ret

Related

python AttributeError: 'mHID' object has no attribute 'dict'

I am trying to make a makeblock mblock implementation with python and i found this api on github but it uses python 2.7 and i am using python 3.8 so i am trying to configure the code so it can run on python 3.8 but i ran into an error which is
init mBot
bot
<lib.mBot.mSerial object at 0x02CE9898>
start with serial
<lib.mBot.mHID object at 0x02E2B310>
self.device = mHID()
'mHID' object has no attribute 'dict'
Exception in thread Thread-2:
self.start()
Traceback (most recent call last):
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 154, in __onRead
start with HID
--------------------
n = self.device.inWaiting()
<lib.mBot.mBot object at 0x002FF640>
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 96, in inWaiting
--------------------
Error in sys.excepthook:
buf = self.dict.device.read(64)
Traceback (most recent call last):
AttributeError: 'mHID' object has no attribute 'dict'
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 134, in excepthook
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 932, in _bootstrap_inner
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.device.close()
self.run()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 870, in run
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
Original exception was:
Traceback (most recent call last):
File "c:/Users/abdsak11/OneDrive - Lärande/Dokument/GitHub/python-for-mbot/test.py", line 15, in <module>
self._target(*self._args, **self._kwargs)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 163, in __onRead
bot.doMove(50,50)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 179, in doMove
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x5]+self.short2bytes(-leftSpeed)+self.short2bytes(rightSpeed)))
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 278, in short2bytes
self.device.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
return [ord(val[0]),ord(val[1])]
TypeError: ord() expected string of length 1, but int found
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
'mHID' object has no attribute 'dict'
Exception in thread Thread-1:
Traceback (most recent call last):
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 154, in __onRead
n = self.device.inWaiting()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 96, in inWaiting
buf = self.dict.device.read(64)
AttributeError: 'mHID' object has no attribute 'dict'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 932, in _bootstrap_inner
self.run()
File "C:\Users\abdsak11\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 163, in __onRead
self.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 142, in close
self.device.close()
File "c:\Users\abdsak11\OneDrive - Lärande\Dokument\GitHub\python-for-mbot\lib\mBot.py", line 106, in close
self.dict.device.close()
AttributeError: 'mHID' object has no attribute 'dict'
So what i am trying to do is to control and upload code to my mbot via usb COM3 serial and thats the error that i am getting i tried everthing but its not working and i cant seem to figure out the cause of the error.
mbot.py
this is the main api that i found
import serial
import sys,time
import signal
from time import ctime,sleep
import glob,struct
from multiprocessing import Process,Manager,Array
import threading
import hid
class mSerial():
ser = None
def __init__(self):
print (self)
def start(self, port):
self.ser = serial.Serial(port,115200)
def device(self):
return self.ser
def serialPorts(self):
if sys.platform.startswith('win'):
ports = ['COM%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
ports = glob.glob('/dev/tty[A-Za-z]*')
elif sys.platform.startswith('darwin'):
ports = glob.glob('/dev/tty.*')
else:
raise EnvironmentError('Unsupported platform')
result = []
for port in ports:
s = serial.Serial()
s.port = port
s.close()
result.append(port)
return result
def writePackage(self,package):
self.ser.write(package)
sleep(0.01)
def read(self):
return self.ser.read()
def isOpen(self):
return self.ser.isOpen()
def inWaiting(self):
return self.ser.inWaiting()
def close(self):
self.ser.close()
class mHID():
def __init__(self):
print (self)
def start(self):
print ("starting start \n\n")
self.manager = Manager()
print("manager pass \n\n")
self.dict = self.manager.dict()
print("dict pass \n\n")
self.dict.device = hid.device()
print("dict device pass \n\n")
self.dict.device.open(0x0416, 0xffff)
print("dict device open pass \n\n")
self.dict.device.hid_set_nonblocking(self.device,1)
print ("start")
self.buffer = []
self.bufferIndex = 0
def enumerate(self):
print ("enumerate")
for dev in self.dict.device.enumerate():
print ('------------------------------------------------------------')
print (dev.description())
def writePackage(self,package):
buf = []
buf += [0, len(package)]
for i in range(len(package)):
buf += [package[i]]
n = self.dict.device.write(buf)
sleep(0.01)
def read(self):
c = self.buffer[0]
self.buffer = self.buffer[1:]
return chr(c)
def isOpen(self):
return True
def inWaiting(self):
buf = self.dict.device.read(64)
l = 0
if len(buf)>0:
l = buf[0]
if l>0:
for i in range(0,l):
self.buffer += [buf[i+1]]
return len(self.buffer)
def close(self):
self.dict.device.close()
class mBot():
def __init__(self):
print ("init mBot")
signal.signal(signal.SIGINT, self.exit)
self.manager = Manager()
self.__selectors = self.manager.dict()
self.buffer = []
self.bufferIndex = 0
self.isParseStart = False
self.exiting = False
self.isParseStartIndex = 0
def startWithSerial(self, port):
self.device = mSerial()
self.device.start(port)
self.start()
def startWithHID(self):
self.device = mHID()
print ("self.device = mHID()\n \n")
self.device.start()
print ("self.device.start()\n \n")
self.start()
print ("self.start()\n \n")
def excepthook(self, exctype, value, traceback):
self.close()
def start(self):
sys.excepthook = self.excepthook
th = threading.Thread(target=self.__onRead,args=(self.onParse,))
th.start()
def close(self):
self.device.close()
def exit(self, signal, frame):
self.exiting = True
sys.exit(0)
def __onRead(self,callback):
while 1:
if(self.exiting==True):
break
try:
if self.device.isOpen()==True:
n = self.device.inWaiting()
for i in range(n):
r = ord(self.device.read())
callback(r)
sleep(0.01)
else:
sleep(0.5)
except Exception as ex:
print (str(ex))
self.close()
sleep(1)
def __writePackage(self,pack):
self.device.writePackage(pack)
def doRGBLed(self,port,slot,index,red,green,blue):
self.__writePackage(bytearray([0xff,0x55,0x9,0x0,0x2,0x8,port,slot,index,red,green,blue]))
def doRGBLedOnBoard(self,index,red,green,blue):
self.doRGBLed(0x7,0x2,index,red,green,blue)
def doMotor(self,port,speed):
self.__writePackage(bytearray([0xff,0x55,0x6,0x0,0x2,0xa,port]+self.short2bytes(speed)))
def doMove(self,leftSpeed,rightSpeed):
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x5]+self.short2bytes(-leftSpeed)+self.short2bytes(rightSpeed)))
def doServo(self,port,slot,angle):
self.__writePackage(bytearray([0xff,0x55,0x6,0x0,0x2,0xb,port,slot,angle]))
def doBuzzer(self,buzzer,time=0):
self.__writePackage(bytearray([0xff,0x55,0x7,0x0,0x2,0x22]+self.short2bytes(buzzer)+self.short2bytes(time)))
def doSevSegDisplay(self,port,display):
self.__writePackage(bytearray([0xff,0x55,0x8,0x0,0x2,0x9,port]+self.float2bytes(display)))
def doIROnBoard(self,message):
self.__writePackage(bytearray([0xff,0x55,len(message)+3,0x0,0x2,0xd,message]))
def requestLightOnBoard(self,extID,callback):
self.requestLight(extID,8,callback)
def requestLight(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x3,port]))
def requestButtonOnBoard(self,extID,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x1f,0x7]))
def requestIROnBoard(self,extID,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x3,extID,0x1,0xd]))
def requestUltrasonicSensor(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x1,port]))
def requestLineFollower(self,extID,port,callback):
self.__doCallback(extID,callback)
self.__writePackage(bytearray([0xff,0x55,0x4,extID,0x1,0x11,port]))
def onParse(self, byte):
position = 0
value = 0
self.buffer+=[byte]
bufferLength = len(self.buffer)
if bufferLength >= 2:
if (self.buffer[bufferLength-1]==0x55 and self.buffer[bufferLength-2]==0xff):
self.isParseStart = True
self.isParseStartIndex = bufferLength-2
if (self.buffer[bufferLength-1]==0xa and self.buffer[bufferLength-2]==0xd and self.isParseStart==True):
self.isParseStart = False
position = self.isParseStartIndex+2
extID = self.buffer[position]
position+=1
type = self.buffer[position]
position+=1
# 1 byte 2 float 3 short 4 len+string 5 double
if type == 1:
value = self.buffer[position]
if type == 2:
value = self.readFloat(position)
if(value<-255 or value>1023):
value = 0
if type == 3:
value = self.readShort(position)
if type == 4:
value = self.readString(position)
if type == 5:
value = self.readDouble(position)
if(type<=5):
self.responseValue(extID,value)
self.buffer = []
def readFloat(self, position):
v = [self.buffer[position], self.buffer[position+1],self.buffer[position+2],self.buffer[position+3]]
return struct.unpack('<f', struct.pack('4B', *v))[0]
def readShort(self, position):
v = [self.buffer[position], self.buffer[position+1]]
return struct.unpack('<h', struct.pack('2B', *v))[0]
def readString(self, position):
l = self.buffer[position]
position+=1
s = ""
for i in Range(l):
s += self.buffer[position+i].charAt(0)
return s
def readDouble(self, position):
v = [self.buffer[position], self.buffer[position+1],self.buffer[position+2],self.buffer[position+3]]
return struct.unpack('<f', struct.pack('4B', *v))[0]
def responseValue(self, extID, value):
self.__selectors["callback_"+str(extID)](value)
def __doCallback(self, extID, callback):
self.__selectors["callback_"+str(extID)] = callback
def float2bytes(self,fval):
val = struct.pack("f",fval)
return [ord(val[0]),ord(val[1]),ord(val[2]),ord(val[3])]
def short2bytes(self,sval):
val = struct.pack("h",sval)
return [ord(val[0]),ord(val[1])]
test.py
this is the test file that i am working with
from lib.mBot import *
if __name__ == "__main__":
bot = mBot()
print ("bot \n\n")
bot.startWithSerial("COM3")
print ("start with serial\n\n")
bot.startWithHID()
print ("start with HID \n\n")
print(f'--------------------\n')
print (bot)
print(f'--------------------\n')
bot.doMove(50,50)
I believe your trouble is here:
class mHID():
def __init__(self):
print (self)
def start(self):
# ...
self.dict = self.manager.dict()
# ...
def close(self):
self.dict.device.close()
You should always define your instance attributes in __init__(), at least with an initial value of None, even if you plan to change them later.
Additionally, whenever you have a situation where "A must be called before B", you should write the code to properly handle when B is called without A. In this case, what should close() do if start() was never called? For simplicity, I'm just having it do nothing, but you may consider raising an exception.
class mHID():
def __init__(self):
print (self)
self.dct = None
def start(self):
# ...
self.dct = self.manager.dict()
# ...
def close(self):
if self.dct:
self.dct.device.close()
You'll notice I also renamed self.dict to self.dct. Although the former name may technically be allowable because of the self. namespace, dict is actually a built-in class. Under various circumstances, you could wind up dealing with shadowing. In the very least, it's confusing to always have to remind oneself that "this isn't the built-in dict, but an instance variable instead.
(Possible aside: is there a particular reason why you don't want to call start() from the initializer?)
WARNING: This is not a simple case of "copy and paste the answer". I've illustrated a couple of design principles here that you should incorporate into the rest of your code.

Import data into mysql from JSON file using Python Code

How to import data into mysql from the given JSON file in Python? I'm getting error as
Traceback (most recent call last): File "C:/Users/risha/PycharmProjects/Test/JSON_TEST.py", line 24, in
con = pymysql.connect(host = '127.0.0.1',user = 'root',passwd = '',db = 'test',port ='3306') File
"C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql__init__.py",
line 90, in Connect
return Connection(*args, **kwargs) File "C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql\connections.py",
line 704, in init
self.connect() File "C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql\connections.py",
line 964, in connect
self.host_info = "socket %s:%d" % (self.host, self.port) TypeError: %d format: a number is required, not str
import pymysql
import os
import json
# read JSON file which is in the next parent folder
file = 'C://Users//risha//PycharmProjects//Test//video.json'
json_data= open(file).read()
json_obj = json.loads(json_data)
# do validation and checks before insert
def validate_string(val):
if val != None:
if type(val) is int:
#for x in val:
# print(x)
return str(val).encode('utf-8')
else:
return val
# connect to MySQL
con = pymysql.connect(host = '127.0.0.1',user = 'root',passwd = '',db = 'test',port ='3306')
cursor = con.cursor()
# parse json data to SQL insert
for i, item in enumerate(json_obj):
person = validate_string(item.get("person", None))
year = validate_string(item.get("year", None))
company = validate_string(item.get("company", None))
cursor.execute("INSERT INTO testp (person, year, company) VALUES (%s, %s, %s)", (person, year, company))
con.commit()
con.close()

Tweepy not working

So I was just trying to run a tweepy script to collect tweets.
I've setup the database but now I'm running into an error.
Starting...
Started user: user1
Exception in thread Thread-1:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "build/bdist.macosx-10.6-intel/egg/tweepy/streaming.py", line 414, in filter
self.body['follow'] = u','.join(follow).encode(encoding)
TypeError: sequence item 0: expected string or Unicode, int found
EDIT: The script is actually using: from urllib import urlencode_noplus but the _noplus is not in the urllib which is why I simply deleted it from the code. Although I suspect this for causing the error..
import tweepy
import threading
import logging
from tweepy.models import Status
from tweepy.utils import import_simplejson
from urllib import urlencode
import json
import re
json = import_simplejson()
class Stream:
def __init__(self, consumer_key, consumer_secret,
key, secret, name):
self.auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
self.auth.set_access_token(key, secret)
self.tweetsBuffer = TweetsBuffer()
self.name = name
self.logger = logging.getLogger('TwitterCollector')
#check credentials
if not tweepy.API(self.auth).verify_credentials():
print "Invalid credentials for user: ",self.name,".\nExiting..."
logging.error("Invalid credentials for user: "+self.name+".\nExiting...")
exit(0)
def run(self, users_list = None):
sl = StreamListener()
sl.init(self.tweetsBuffer)
try:
streamer = tweepy.Stream(auth=self.auth,
listener=sl,
timeout=3000000000,
include_entities=1,
include_rts=1)
#load friends
filter = []
if users_list is None:
filter = tweepy.API(self.auth).friends_ids()
else:
for subList in users_list:
for user in subList['users']:
filter.append(user.id)
#remove duplicates
filter = list(set(filter))
sThread = threading.Thread(target=streamer.filter, args=(filter,))
sThread.start()
return sThread
except Exception, e:
print e
def getTweetsBuffer(self):
return self.tweetsBuffer
def getUserList(self, lists):
if lists is None:
return None
api = tweepy.API(self.auth)
users_list = []
for list in lists:
users = []
members = tweepy.Cursor(
api.list_members,
list['owner'],
list['slug']
).items()
for member in members:
users.append(member)
users_list.append(
{
'owner' : list['owner'],
'slug' : list['slug'],
'users' : users
})
return users_list
class StreamListener(tweepy.StreamListener):
def init(self, tweetsBuffer):
#set buffer
self.tweetsBuffer = tweetsBuffer
def parse_status(self, status, retweet = False):
tweet = {
'tweet_id':status.id,
'tweet_text':status.text,
'created_at':status.created_at,
'geo_lat':status.coordinates['coordinates'][0]
if not status.coordinates is None
else 0,
'geo_long': status.coordinates['coordinates'][1]
if not status.coordinates is None
else 0,
'user_id':status.user.id,
'tweet_url':"http://twitter.com/"+status.user.id_str+"/status/"+status.id_str,
'retweet_count':status.retweet_count,
'original_tweet_id':status.retweeted_status.id
if not retweet and (status.retweet_count > 0)
else 0,
'urls': status.entities['urls'],
'hashtags':status.entities['hashtags'],
'mentions': status.entities['user_mentions']
}
#parse user object
user = {
'user_id':status.user.id,
'screen_name': status.user.screen_name,
'name': status.user.name,
'followers_count': status.user.followers_count,
'friends_count': status.user.friends_count,
'description': status.user.description
if not status.user.description is None
else "N/A",
'image_url': status.user.profile_image_url,
'location': status.user.location
if not status.user.location is None
else "N/A",
'created_at': status.user.created_at
}
return {'tweet':tweet, 'user':user}
def on_data(self, data):
if 'in_reply_to_status_id' in data:
status = Status.parse(self.api, json.loads(data))
if self.on_status(status, data) is False:
return False
elif 'delete' in data:
delete = json.loads(data)['delete']['status']
if self.on_delete(delete['id'], delete['user_id']) is False:
return False
elif 'limit' in data:
if self.on_limit(json.loads(data)['limit']['track']) is False:
return False
def on_status(self, status, rawJsonData):
try:
#parse tweet
tweet = self.parse_status(status)
tweet['raw_json'] = rawJsonData
self.tweetsBuffer.insert(tweet)
#parse retweet
if tweet['tweet']['retweet_count'] > 0:
retweet = self.parse_status(status.retweeted_status, True)
retweet['raw_json'] = None
self.tweetsBuffer.insert(retweet)
except Exception:
# Catch any unicode errors while printing to console
# and just ignore them to avoid breaking application.
pass
class TweetsBuffer():
tweetsBuffer = []
def __init__(self):
self.lock = threading.Lock()
def insert(self, tweet):
self.lock.acquire()
self.tweetsBuffer.append(tweet)
self.lock.release()
def pop(self):
self.lock.acquire()
tweet = self.tweetsBuffer.pop() if len(self.tweetsBuffer) > 0 else None
self.lock.release()
return tweet

TypeError: cannot concatenate 'str'

I got this error TypeError: cannot concatenate 'str' and 'builtin_function_or_method' objects when i tried to change directory in the ftp.
ftp.py
from ftplib import FTP
ftp = FTP()
class FTPClient():
connection_id = None
login_ok = False
message_array = []
def __init__(self):
pass
def log_message(self, message, clear=True):
if clear:
self.message_array = message
def get_message(self):
return self.message_array
def connect(self, server, ftp_user, ftp_password, is_passive = False):
self.connection_id = ftp.connect(server)
login_result = ftp.login(user=ftp_user, passwd=ftp_password)
ftp.set_pasv(is_passive)
if not self.connection_id or not login_result:
self.log_message("FTP Connection has Failed")
self.log_message("Attempted to connect to {0} for {1}".format(server, ftp_user))
return False
else:
self.log_message("Connected to {0} for {1}".format(server, ftp_user))
self.login_ok = True
return True
def make_dir(self, directory):
if ftp.mkd(directory):
self.log_message("Directory {0} created successfully".format(directory))
return True
else:
self.log_message("Failed creating directory")
return False
def change_directory(self, directory):
if ftp.cwd(directory):
self.log_message("Current Directory is now {0}".format(ftp.pwd))
else:
self.log_message("Can't change Directory")
def upload_file(self, file_from, file_to):
if file_from.endswith(('.csv', '.txt')):
with open(file_from, 'r') as f:
self.connection_id.storelines("Uploaded from {0} to {1}".format(file_from, file_to), f)
else:
with open(file_from, 'rb') as f:
self.connection_id.storebinary('Uploaded from {0} to {1}'.format(file_from, file_to), f)
ftp_obj = FTPClient()
FTP_HOST = "yyy"
FTP_USER = "xxx"
FTP_PASS = "zzz"
ftp_obj.connect(FTP_HOST, FTP_USER, FTP_PASS)
print(ftp_obj.get_message())
FILE_FROM = "config.txt"
FILE_TO = ftp_obj.change_directory(dir)
ftp_obj.upload_file(FILE_FROM, FILE_TO)
print(ftp_obj.get_message())
Full traceback:
Connected to yyy for xxx
Traceback (most recent call last):
File "C:/Users/Ajay/PycharmProjects/database/test.py", line 67, in <module>
FILE_TO = ftp_obj.change_directory(dir)
File "C:/Users/Ajay/PycharmProjects/database/test.py", line 44, in change_directory
if ftp.cwd(directory):
File "C:\Python27\lib\ftplib.py", line 552, in cwd
cmd = 'CWD ' + dirname
TypeError: cannot concatenate 'str' and 'builtin_function_or_method' objects
Whats happening there ? and why this error ?
Error is here:
FILE_TO = ftp_obj.change_directory(dir)
you have not declared dir var, builtin function dir passed instead.

Python - how to upload large file

I write script, which uploads large file to third party website. I use poster module
Here code
class upload_in_chunks(object):
def __init__(self, filename, chunksize):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
def __iter__(self):
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
sys.stderr.write("\n")
break
self.readsofar += len(data)
percent = self.readsofar * 1e2 / self.totalsize
sys.stderr.write("\r{percent:3.0f}%".format(percent=percent))
yield data
def __len__(self):
return self.totalsize
class IterableToFileAdapter(object):
def __init__(self, iterable):
self.iterator = iter(iterable)
self.length = len(iterable)
def read(self, size=-1): # TBD: add buffer for `len(data) > size` case
return next(self.iterator, b'')
def __len__(self):
return self.length
def upload_file(s, data, files, url):
class FuncThread(threading.Thread):
def __init__(self, s, data, files, url):
threading.Thread.__init__(self)
self.result = False
self.error = None
self.s = s
self.data = data
self.files = files
self.url = url
def run(self):
try:
register_openers()
items = []
for name, value in self.data.items():
items.append(MultipartParam(name, value))
#add file
for name, value in self.files.items():
items.append(MultipartParam.from_file(name, value))
datagen, h = multipart_encode(items)
#h.update(headers)
# Create the Request object
#self.result = self.s.post(self.url, data = datagen.read(), headers = h)
for item, value in headers.iteritems():
h[item] = value
#for item, value in headers.iteritems():
#h[item] = value
h["Cookie"] = ""
for item, value in requests.utils.dict_from_cookiejar(self.s.cookies).iteritems():
h["Cookie"] += item + "=" + value + "; "
print h["Cookie"]
#h[item] = value
print h
request = urllib2.Request(self.url, datagen, h)
#Actually do the request, and get the response
self.result = urllib2.urlopen(request).read()
except Exception as error:
self.error = error
self.result = False
def _stop(self):
if self.isAlive():
threading.Thread._Thread__stop(self)
try:
it = FuncThread(s, data, files, url)
it.start()
it.join(upload_timeout)
if it.isAlive():
it._stop()
return (False, "[-] Time expired.")
else:
return (it.result, it.error)
except Exception as error:
return (False, error)
And in my upload.py I use
try:
payload = {'Filename': f_path,
'tags': 'tag1',
'keywords': 'keyword1',
'cookie': cookie,
'title': 'Large file',
'categories': '["35","13","37"]',
'privacy': 'community',
'source': 5,
'production': "professional"}
(r, error) = upload_file(self.s, payload, {'Filedata': f_path}, url)
I have dedicated server with 100m\bit internet channel.
But my script upload files using only 1-2 m\bit
The website recipient has not any limitations and it doesn't throttle me.
How can I force my code to use full 100 m\bit?

Categories

Resources