We Keep Coding

Tweepy on_data receiving None

I am following guide on how stream data to Kafka with Python
After making the modifications suggested by #MarkTolonen I am now getting the following error:
AttributeError: 'NoneType' object has no attribute 'encode'
The full code is as follows:
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from kafka import KafkaClient
#from kafka import SimpleProducer
access_token = ""
access_token_secret = ""
consumer_key = ""
consumer_secret = ""
#Configure Kafka
kafkaBrokers = '127.0.1.1:9092'
#producer = KafkaProducer(bootstrap_servers=kafkaBrokers,key_serializer=lambda k: k.encode('ascii','ignore'),value_serializer=lambda x: dumps(x).encode('utf-8'))
class StdOutListener(StreamListener):
def on_data(self, data):
producer.send("trump", data.encode('utf-8'))
print (data)
return True
def on_error(self, status):
print (status)
kafka = 'localhost:9092'
#kafka = KafkaClient("localhost:9092")
#producer = SimpleProducer(kafka)
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track="trump")

You need to refer the documentation why the data could ever be None, but you can work around that like so
def on_data(self, data):
if data:
producer.send("trump", data.encode('utf-8'))
print (data)
return True
return False

How to get Tweets of a Keyword

I'm trying to get tweets from a certain keyword 'comfama'. but I can't seem to get any results. Is something wrong with my code? I'm tried with 'donald trump' and this keyword shows results but with 'comfama' nothing happens.
import tweepy
import pandas
import json # The API returns JSON formatted text
TRACKING_KEYWORDS = ['comfama']
OUTPUT_FILE = "comfama_tweets.txt"
TWEETS_TO_CAPTURE = 10
access_token = "xxx"
access_token_secret = "xxx"
consumer_key = "xxx"
consumer_secret = "xxx"
# Pass OAuth details to tweepy's OAuth handler
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
class MyStreamListener(tweepy.StreamListener):
"""
Twitter listener, collects streaming tweets and output to a file
"""
def __init__(self, api=None):
super(MyStreamListener, self).__init__()
self.num_tweets = 0
self.file = open(OUTPUT_FILE, "w")
def on_status(self, status):
tweet = status._json
self.file.write( json.dumps(tweet) + '\n' )
self.num_tweets += 1
# Stops streaming when it reaches the limit
if self.num_tweets <= TWEETS_TO_CAPTURE:
if self.num_tweets % 100 == 0: # just to see some progress...
print('Numer of tweets captured so far: {}'.format(self.num_tweets))
return True
else:
return False
self.file.close()
def on_error(self, status):
print(status)
# Initialize Stream listener
l = MyStreamListener()
# Create you Stream object with authentication
stream = tweepy.Stream(auth, l)
# Filter Twitter Streams to capture data by the keywords:
stream.filter(track=[TRACKING_KEYWORDS])

Tweepy Streaming Direct Messages

I've been using Tweepy with Python 2.7 to stream tweets and everything has been working fine, except the on_direct_message() method isn't being called when I send the account a direct message. I've updated my permissions and even tried using the on_data() method, but it can't seem to detect direct messages being sent to the account:
import tweepy
CONSUMER_KEY = ''
CONSUMER_SECRET = ''
ACCESS_KEY = ''
ACCESS_SECRET = ''
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
api = tweepy.API(auth, wait_on_rate_limit=True)
followed_accounts = ['account', 'account']
followed_ids = []
for account in followed_accounts:
followed_ids.append(str(api.get_user(screen_name=account).id))
class StdOutListener(tweepy.StreamListener):
def on_direct_message(self, status):
author = status.author.screen_name
api.send_direct_message(screen_name=author, text='response')
return True
def on_status(self, status):
author = status.author.screen_name
statusID = status.id
print status.text + "\n"
api.update_status('response')
api.send_direct_message(screen_name='my username', text='Just sent a Tweet')
return True
def on_data(self, status):
print 'Entered on_data()'
print status
return True
def on_error(self, status_code):
print "Error Code: " + str(status_code)
if status_code == 420:
return False
else:
return True
def on_timeout(self):
print('Timeout...')
return True
if __name__ == '__main__':
listener = StdOutListener()
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
stream = tweepy.Stream(auth, listener)
stream.filter(follow=followed_ids)
Sending the account a direct message gives no errors, and the account receives the message properly on Twitter.

save twitter user information into a file using StreamListener

Guys i wanna save twitter user info like name, statuses, tweet in my file (either json,txt,csv or any other json or text are prefered). I tried this code and some other similar but none of them work. Guys have a look at below code and suggest me what changes should i made??
import time
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import os
import json
ckey = '**********'
consumer_secret = '**********'
access_token_key = '**********'
access_token_secret = '**********'
start_time = time.time() #grabs the system time
keyword_list = ['twitter'] #track list
#Listener Class Override
class listener(StreamListener):
def __init__(self, start_time, time_limit=60):
self.time = start_time
self.limit = time_limit
def on_data(self, data):
while (time.time() - self.time) < self.limit:
try:
all_data = json.loads["text"]
username = all_data["user"]["name"]
tweets = all_date["user"]["statuses"]
saveFile = open('raw_tweets29.json', 'a')
saveFile.write(username)
saveFile.write('\n')
saveFile.close()
return True
except BaseException, e:
print 'failed ondata,', str(e)
time.sleep(5)
pass
exit()
def on_error(self, status):
print statuses
auth = OAuthHandler(ckey, consumer_secret) #OAuth object
auth.set_access_token(access_token_key, access_token_secret)
twitterStream = Stream(auth, listener(start_time, time_limit=20))
twitterStream.filter(track=['twitter'])
when i run below code this give me error -
failed ondata, 'function' object has no attribute '__getitem__'
I would greatly appreciate any help you can give me in working this problem

I am doing some mistake, now i figure it out there is no need of temp variable 'text' what i need to do is load actual data.
there is one more thing require is encoding.
thanks everyone for your time.
import time
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import os,sys
import json
ckey = '***'
consumer_secret = '***'
access_token_key = '***'
access_token_secret = '***'
start_time = time.time()
class listener(StreamListener):
def __init__(self, start_time, time_limit=300):
self.time = start_time
self.limit = time_limit
def on_data(self, data):
while (time.time() - self.time) < self.limit:
try:
tweet = json.loads(data)
user_name = tweet['user']['name']
tweet_count = tweet['user']['statuses_count']
text = tweet['text']
saveFile = open('user_tweets29.json', 'a')
saveFile.write(text.encode('utf8'))
saveFile.write('\n')
saveFile.close()
return True
except BaseException, e:
print 'failed ondata,', str(e)
time.sleep(5)
pass
exit()
def on_error(self, status):
print statuses
auth = OAuthHandler(ckey, consumer_secret)
auth.set_access_token(access_token_key, access_token_secret)
twitterStream = Stream(auth, listener(start_time, time_limit=60))
twitterStream.filter(track=['twitter'])

Retrieving Twitter data

I am using this code below to retrieve twitter hashtag data using tweepy, but this code only retrieve the tweet message and the time created, but I need to retrieve the metadata for that hashtag, any help!!
import tweepy
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import json
import datetime
#setting up the keys
consumer_key = '-------'
consumer_secret = '----------'
access_token = '--------'
access_secret = '-----------'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
def date_range(start,end):
current = start
while (end - current).days >= 0:
yield current
current = current + datetime.timedelta(seconds=1)
class TweetListener(StreamListener):
def on_status(self, status):
#api = tweepy.API(auth_handler=auth)
#status.created_at += timedelta(hours=900)
startDate = datetime.datetime(2014, 03, 11)
stopDate = datetime.datetime(2014, 03, 13)
for date in date_range(startDate,stopDate):
status.created_at = date
print "tweet " + str(status.created_at) +"\n"
print status.text + "\n"
stream = Stream(auth, TweetListener(), secure=True, )
t = u"#سوريا"
stream.filter(track=[t])

this might help, a full specification of the status object.