Develop Reference

Send Table contents from CSV file in Outlook Mail Body using Python

i programmed a code to send a outlook mail , which should contain the contents of CSV file as it's body , Mail part is working fine . But the table appears to be distorted .
[![MailBody][1]][1]
So here is there anyway to arrange this . and make it pretty .
and here is my code :
def sendMailt():
print("*** SENDING MAIL ****")
email_user = 'ABC#domain.com'
email_send = 'DCF#domain.com'
subject = ''
msg = MIMEMultipart()
msg['From'] = email_user
msg['To'] = email_send
msg['Subject'] = subject
body = 'Hi Team , Please Open This attachment for Folderstat Report,'
msg.attach(MIMEText(body, 'plain'))
text = """
Hello, Friend.
Here is your data:
{table}
Regards,
Me"""
with open(filtered_CSV) as input_file:
reader = csv.reader(input_file)
data = list(reader)
text = text.format(table=tabulate(data, headers=['Unnamed: 0','id','path','name','extension','size','FolderSize in GB','LastAccessTime','LastModifiedTime','LastCreationTime','folder','Total Number of files','Absolute File Count','depth','parent','uid','Oldest File Timestamp','Recent File Timestamp','Folder Modified Timestamp','Status','md5]'] ,tablefmt='orgtbl'))
server = smtplib.SMTP('domain')
sender = 'ABC#domain.com'
reciever = ['DCF#domain.com']
server.sendmail(sender, reciever, text)
server.quit()
print("Mail SEND")
and also when i receive mail , am not able to see subject or receivers ID
[1]: https://i.stack.imgur.com/y5Gxy.png

if you Use html in tablefmt you can get the data in decent format
from tabulate import tabulate
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
text = """
Hello, Friend.
Here is your data:
{table}
Regards,
Me"""
with open(filtered_CSV) as input_file:
reader = csv.reader(input_file)
data = list(reader)
html = text.format(table=tabulate(data, headers="firstrow", tablefmt="html")
# for html design ( you can add whatever style you want)
html = html.replace('<table>', """<table border=1 style="
border-color:grey;
font-family: Helvetica, Arial;
font-size: 15px;
padding: 5px;">""")
server = smtplib.SMTP('domain')
msg = MIMEMultipart("alternative", None, [MIMEText(html, 'html')])
msg['From'] = from address
msg['To'] = to address
msg['Subject'] = "your subject"
server.sendmail(msg['From'], msg['To'], msg.as_string())
server.quit()
print("Mail SEND")

Scraping data from website using Beautiful Soup and Pandas

I have a python script that use the BeautifulSoup and Pandas packages in order to scrape data from a list of urls and convert the data into a dataframe then save it as excel file and send it by email as attachment.
The problem is that when the script run and finish the scraping of the first item it crash and return the error below:
ValueError: 15 columns passed, passed data had 14 columns
I think this means that there is a missing html tag right??
The list includes 3 urls.
code:
import time
from datetime import date
import smtplib
import requests
import pandas as pd
from bs4 import BeautifulSoup
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
from email.utils import formatdate
def scrape_website():
url_list = ["https://www.bayt.com/en/international/jobs/executive-chef-jobs/",
"https://www.bayt.com/en/international/jobs/head-chef-jobs/",
"https://www.bayt.com/en/international/jobs/executive-sous-chef-jobs/"]
for url in url_list:
soup = BeautifulSoup(requests.get(url).content, "lxml")
links = []
for a in soup.select("h2.m0.t-regular a"):
if a['href'] not in links:
links.append("https://www.bayt.com" + a['href'])
joineddd = []
for link in links:
s = BeautifulSoup(requests.get(link).content, "lxml")
alldd = [dd.text for dd in s.select(
"div[class='card-content is-spaced'] dd")]
alldd.insert(0, link)
joineddd.append(alldd)
print("Web Crawling is Done for {}".format(url))
convert_to_dataFrame(joineddd)
send_email()
def remove_unwanted_cols(dataset, cols):
for col in cols:
del dataset[col]
return dataset
def convert_to_dataFrame(joineddd):
df = pd.DataFrame(joineddd, columns=[
"link", "location", "Company_Industry", "Company_Type",
"Job_Role", "Employment_Type", "Monthly_Salary_Range",
"Number_of_Vacancies", "Career_Level",
"Years_of_Experience", "Residence_Location",
"Gender","Nationality","Degree","Age"])
df = remove_unwanted_cols(df, ["Company_Industry","Company_Type","Job_Role","Number_of_Vacancies"])
df_to_excel = df.to_excel(r"F:\\AIenv\web_scrapping\\jobDesc.xlsx", index = False, header=True)
send_email()
def send_email():
'''send email '''
today = date.today()
file = 'F:\\AIenv\web_scrapping\\jobDesc.xlsx'
username='XXXXXXXXXXX'
password='XXXXXXXXXXXXX'
send_from = 'XXXXXXXXXXXXX'
send_to = 'XXXXXXXXXXXXXX'
Cc = 'recipient'
msg = MIMEMultipart()
msg['From'] = send_from
msg['To'] = send_to
msg['Cc'] = Cc
msg['Date'] = formatdate(localtime=True)
msg['Subject'] = 'Hello, This is a test mail {}'.format(today)
server = smtplib.SMTP('smtp.gmail.com')
port = '587'
fp = open(file, 'rb')
part = MIMEBase('application', 'vnd.ms-excel')
part.set_payload(fp.read())
fp.close()
encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment', filename='jobs Description--{}'.format(today))
msg.attach(part)
smtp = smtplib.SMTP('smtp.gmail.com')
smtp.ehlo()
smtp.starttls()
smtp.login(username, password)
smtp.sendmail(send_from, send_to.split(',') + msg['Cc'].split(','), msg.as_string())
smtp.quit()
print('Mail Sent')
if __name__ == "__main__":
scrape_website()

update func scrape_website(), save alldd as dictionary.
for link in links:
s = BeautifulSoup(requests.get(link).content, "lxml")
### update Start ###
alldd = dict()
alldd['link'] = link
dd_div = [i for i in s.select("div[class='card-content is-spaced'] div")
if ('<dd>' in str(i) ) and ( "<dt>" in str(i))]
for div in dd_div:
k = div.select_one('dt').get_text(';', True)
v = div.select_one('dd').get_text(';', True)
alldd[k] = v
### update End ###
joineddd.append(alldd)
# result
df = pd.DataFrame(joineddd)
alladd sample:
{
'link': 'https://www.bayt.com/en/qatar/jobs/executive-chef-4298309/',
'Job Location': 'Doha, Qatar',
'Company Industry': 'Real Estate; Hospitality & Accomodation; Catering, Food Service, & Restaurant',
'Company Type': 'Employer (Private Sector)',
'Job Role': 'Hospitality and Tourism',
'Employment Type': 'Unspecified',
'Monthly Salary Range': 'Unspecified',
'Number of Vacancies': 'Unspecified',
'Career Level': 'Mid Career',
'Years of Experience': 'Min: 7',
'Residence Location': 'Qatar',
'Degree': "Bachelor's degree / higher diploma"
}

ValueError: 15 columns passed, passed data had 14 columns
What I read here means that you designated the dataframe to have 15 columns, but the data that you feed it only has 14 features. You need to check your original file to make sure it actually has the data you expect, or adjust your expected columns and their names to match the file.

Let's clean up some of this code.
You don't need to write a function to remove columns, there's already a method to do that with .drop(). So delete the function remove_unwanted_cols(dataset, cols) and simply change the line:
df = remove_unwanted_cols(df, ["Company_Industry","Company_Type","Job_Role","Number_of_Vacancies"])
to
df = df.drop(["Company_Industry","Company_Type","Job_Role","Number_of_Vacancies"], axis=1)
Do you mean to have it send email twice? You have it do it in the scrape_website() finction and the convert_to_dataFrame() function.
If pulling data to construct a dataframe, I usually try to avoid lists, for the exact reason you get the error of some sites have x columns, but the next scrape has and extra one (or not a match in len). Dictionaries are a better way to handle that with the key being the column name, and value the data. So you'll have a list of dictionaries. Each item in the list is a row, and each dictionary corresponds to the value for a column. Then you can get rid of the convert_to_dataFrame() function as pandas can do that for you, but we'll leave that in there and you can keep it, or remove it, if you like.
If you are using r'' for your strings, you don't need to character escape the \. Either do: r"F:\AIenv\web_scrapping\jobDesc.xlsx", or "F:\\AIenv\web_scrapping\\jobDesc.xlsx"
Code:
import time
from datetime import date
import smtplib
import requests
import pandas as pd
from bs4 import BeautifulSoup
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
from email.utils import formatdate
def scrape_website(url):
soup = BeautifulSoup(requests.get(url).content, "lxml")
subject = url.split('/')
subject = [x for x in subject if x != ''][-1]
links = []
for a in soup.select("h2.m0.t-regular a"):
if a['href'] not in links:
links.append("https://www.bayt.com" + a['href'])
joineddd = []
for link in links:
row = {}
s = BeautifulSoup(requests.get(link).content, "lxml")
job_description = s.find('h2', text='Job Description').find_next('dl')
data_titles = job_description.find_all('dt')
for data_title in data_titles:
dt = '_'.join(data_title.text.split())
dd = data_title.find_next('dd').text.strip()
row.update({dt: dd})
if s.find('h2', text='Preferred Candidate'):
preferred_candidate = s.find('h2', text='Preferred Candidate').find_next('dl')
data_titles = preferred_candidate.find_all('dt')
for data_title in data_titles:
dt = '_'.join(data_title.text.split())
dd = data_title.find_next('dd').text.strip()
row.update({dt: dd})
joineddd.append(row)
print("Web Crawling is Done for {}".format(url))
convert_to_dataFrame(joineddd, subject)
#send_email(subject) #<-- did you want to send here?
def convert_to_dataFrame(joineddd, subject):
df = pd.DataFrame(joineddd)
df = df.drop(["Company_Industry","Company_Type","Job_Role","Number_of_Vacancies"], axis=1)
df_to_excel = df.to_excel(r"F:\\AIenv\web_scrapping\\jobDesc.xlsx", index = False, header=True)
send_email(subject) #<--or do you want to send here??
def send_email(subject):
'''send email '''
today = date.today()
file = 'F:\\AIenv\web_scrapping\\jobDesc.xlsx'
username='XXXXXXXXXXX'
password='XXXXXXXXXXXXX'
send_from = 'XXXXXXXXXXXXX'
send_to = 'XXXXXXXXXXXXXX'
Cc = 'recipient'
msg = MIMEMultipart()
msg['From'] = send_from
msg['To'] = send_to
msg['Cc'] = Cc
msg['Date'] = formatdate(localtime=True)
msg['Subject'] = 'Hello, This is a test mail {} - {}'.format(today,subject)
server = smtplib.SMTP('smtp.gmail.com')
port = '587'
fp = open(file, 'rb')
part = MIMEBase('application', 'vnd.ms-excel')
part.set_payload(fp.read())
fp.close()
encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment', filename='jobs Description--{}'.format(today))
msg.attach(part)
smtp = smtplib.SMTP('smtp.gmail.com')
smtp.ehlo()
smtp.starttls()
smtp.login(username, password)
smtp.sendmail(send_from, send_to.split(',') + msg['Cc'].split(','), msg.as_string())
smtp.quit()
print('Mail Sent')
url_list = ["https://www.bayt.com/en/international/jobs/executive-chef-jobs/",
"https://www.bayt.com/en/international/jobs/head-chef-jobs/",
"https://www.bayt.com/en/international/jobs/executive-sous-chef-jobs/"]
if __name__ == "__main__":
for url in url_list:
scrape_website(url)

Sending two Pandas dataframes side-by-side using HTML in email

I'm trying to send some summary on my shares portfolio creation via email. I'm using Python + Pandas for the calculations and email.mime module to send html via email.
I am using Pandas to_html method and email.mime module to include the html in the email:
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
import numpy as np
import pandas as pd
import pickle
from utils import mail
def send_fancy_mail(subject, text_message, html_message):
msg = MIMEMultipart('alternative')
msg['Subject'] = subject
msg['From'] = 'mymail#domain.com'
msg['To'] = settings.MAIL_RECIPIENTS
# Record the MIME types of both parts - text/plain and text/html.
part1 = MIMEText(text_message, 'plain')
part2 = MIMEText(html_message, 'html')
msg.attach(part1)
msg.attach(part2)
# Send the message via our own SMTP server.
s = smtplib.SMTP_SSL(settings.SMTP_HOST)
s.login(settings.SMTP_USER, settings.SMTP_PASSWORD)
s.send_message(msg)
s.quit()
fake_or_load = 'fake'
if fake_or_load == 'fake':
n_signals = 20
symbols = np.round(1000*np.random.rand(n_signals)).astype(int)
direction = ["Buy" for i in range(n_signals//2)]
direction.extend(["Sell" for i in range(n_signals//2)])
quantity = np.round(10000*np.random.rand(n_signals)).astype(int)
portfolio = pd.DataFrame({'symbols': symbols, 'direction': direction, 'quantity': quantity})
elif fake_or_load == 'load':
with open('c:\\\\temp\\signals_list', 'rb') as fp:
signals = pickle.load(fp)
portfolio = pd.DataFrame(signals)
portfolio.rename(index=str, inplace=True, columns={0: "symbol", 1: "direction", 2: "quantity"})
shares_to_buy = portfolio[portfolio['direction'] == 'Buy'].copy()
shares_to_buy.sort_values(by='quantity', inplace=True, ascending=False)
shares_to_sell = portfolio[portfolio['direction'] == 'Sell'].copy()
shares_to_sell.sort_values(by='quantity', inplace=True, ascending=False)
# The basic way to convert portfolio to html:
html_to_buy = shares_to_buy.to_html(index=False, header=True, col_space=20, justify='center')
html_to_sell = shares_to_sell.to_html(index=False, header=True, col_space=20, justify='center')
mail_body = "Test Message"
css = """
.div {
flex-direction: row;
}
"""
html_body = """
<html>
<head>
<style>{}</style>
</head>
<body>
<div>{}</div>
<div>{}</div>
</body>
</html>
""".format(css, html_to_buy, html_to_sell)
mail.send_fancy_mail("Test Mail", mail_body, html_body)
I would like to get two columns side-by-side in the email body, preferably able to regulate the column width as well. Is it possible to make the tables responsive?

If you want to create two columns, replace the two divs with the below table. Div's dont have the same support as tables on all email clients.
<table style="width:100%">
<tr>
<td>column 1</td>
<td>column 2</td>
</tr>
</table>

Python IMAPLIB HTML body parsing

so basically i have this script that runs continuously and when a new email arrives in the inbox with specific text in the subject, it grabs information from the email. I have only managed to get it to pull the subject from the email but I cant get it do get the body of the email no matter what I try, I believe the email body is in HTML so i attempted to use BeautifulSoup to parse the body but that doesnt work at all. Please help!!! :( Here is what i have so far:
import email
import imaplib
from bs4 import BeautifulSoup
import time
import sys
username = 'xxx.xxx#xxx.xx'
password = 'xxxxxx'
mail = imaplib.IMAP4_SSL('imap-mail.outlook.com')
(retcode, capabilities) = mail.login(username, password)
mail.list()
n=0
while True:
mail.select('inbox')
(retcode, messages) = mail.search(None, 'UNSEEN', '(SUBJECT "xxxxxxx-
")', '(FROM "xx.xx#xxxx.xx")')
if retcode == 'OK':
for num in messages[0].split():
n=n+1
print('Processing Email ' + str(n))
typ, data = mail.fetch(num, '(RFC822)')
for response_part in data:
if isinstance(response_part, tuple):
original = email.message_from_bytes(response_part[1])
print("Subject: " + original['Subject'])
typ, data = mail.store(num,'+FLAGS','\\Seen')
time.sleep(120)

Comment: The "body" returned by imap.fetch are usually bytes, not a string, which throws an exception
Change to:
msg = email.message_from_bytes(body)
Question: I cant get it do get the body of the email
For example:
import email, imaplib
username = 'xxx.xxx#xxx.xx'
password = 'xxxxxx'
imap = imaplib.IMAP4_SSL('imap-mail.outlook.com')
imap.login(username, password)
imap.select("inbox")
resp, items = imap.search(None, "(UNSEEN)")
for n, num in enumerate(items[0].split(), 1):
resp, data = imap.fetch(num, '(RFC822)')
body = data[0][1]
msg = email.message_from_string(body)
content = msg.get_payload(decode=True)
print("Message content[{}]:{}".format(n, content))

Include Excel table in the body of an email in Python

Using Python, I am trying to send an email with an Excel table inside the body of an email. I would like to maintain all the conditional formatting from the Excel file. I can send the Excel file as an attachment easy, but I would like to also put the table inside the body of the email. I will convert it to an HTML table if I need to, but I need to know how to include the HTML table into body of the email then. Below attaches the file as an email, but I haven't been able to figure out how to put the table inside of the email. How can I do this?
msg = MIMEMultipart()
msg['Subject'] = 'Subject goes here'
msg.attach(MIMEText('Text goes here'))
part = MIMEBase('application', "octet-stream")
f = 'file_name.xlsx'
part.set_payload(open(f, "rb").read())
encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment; filename="%s"' % f)
msg.attach(part)
Thanks for your help!

The easiest way to do is use pandas. Something like:
import pandas as pd
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
def generate_html():
read_file = pd.read_csv("example.csv")
html_file = read_file.to_html()
sendEmail(html_file)
def sendEmail(html_file):
msg = MIMEMultipart('alternative')
msg['Subject'] = "Hello"
msg['From'] = EmailFrom
msg['To'] = EmailTo
part = MIMEText(html_file, 'html')
msg.attach(part)
s = smtplib.SMTP("smtp.gmail.com")
s.sendmail(EmailFrom, EmailTo, msg.as_string())
s.quit()

You might want to look into openpyxl https://openpyxl.readthedocs.io/en/default/
Something like this might solve your problem:
import openpyxl
from openpyxl import load_workbook
workbook = load_workbook(f)
worksheet = workbook.get_active_sheet()
html_data = """
<html>
<head>
<title>
XLSX to HTML demo
<title>
<head>
<body>
<h3>
XLSX to HTML demo
<h3>
<table>
"""
ws_range = worksheet.range('A1:H13')
for row in ws_range:
html_data += "<tr>
for cell in row:
if cell.value is None:
html_data += "<td> + ' ' + "<td>
else:
html_data += "<td> + str(cell.value) + "<td>
html_data += "<tr>
html_data += "</table></body></html>
msg.attach(MIMEText(html_data))
with open(f, "rb") as fil:
part = MIMEApplication(
fil.read(),
Name=basename(f)
)
part['Content-Disposition'] = 'attachment; filename="{0}"'.format(basename(f))
msg.attach(part)
Inspired by https://jugad2.blogspot.ch/2013/11/publish-microsoft-excel-xlsx-data-to.html?m=1