I'm trying to make a Python server that I can call from other applications to request Twitter data. I usually work with Python as a scripting language, so if there are any red flags anyone sees in my code, I'm all ears!
This is basically what I have so far, which works well when I ping the server, it gets 10 tweets from my timeline and sends them back to my other applications. My main issue is that I'd like to combine streaming and searching. That way I can have the stream open for a specific hash tag that I'd like to have sent to my other applications in real-time, but then I'd periodically search for other things that don't need to be coming down to me in real-time.
I've had success using both separately, but not sure where to start if I wanted to implement both, which in this case I'd like to bring the stream functionality into this.
I'm using Python Twitter Tools 1.10.2 - http://mike.verdone.ca/twitter/
and Python 3.3
Code below, thanks!
EDIT:I was able to get a step further by adding the twitter streaming connection after the if data == "SEARCH_NOW" if statement. But this brings up the original issue I was having. Once the twitter stream is open, the code seems to just wait there. If i put it before timeline lookup, then I can never call the timeline lookup. Updated code to reflect.
EDIT 2: Putting the search request inside of the twitter stream loop gets a little closer. I can now have the stream open and every time I get a tweet that matches the search term, then I can also do a request. But still not independently...
File: network_settings.py
#!/usr/bin/env python
#network settings
import socket
#set server variables
TCP_IP = '127.0.0.1'
TCP_PORT = 7001
BUFFER_SIZE = 20
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP, TCP_PORT))
s.listen(1)
conn, addr = s.accept()
#print connection address when someone connects
print ('Connection address:', addr)
File: twitter_settings.py
from twitter import *
import re
OAUTH_TOKEN = ''
OAUTH_SECRET = ''
CONSUMER_KEY = ''
CONSUMER_SECRET = ''
t = Twitter(auth=OAuth(OAUTH_TOKEN, OAUTH_SECRET, CONSUMER_KEY, CONSUMER_SECRET))
auth = OAuth(OAUTH_TOKEN, OAUTH_SECRET, CONSUMER_KEY, CONSUMER_SECRET)
stream = TwitterStream(auth = auth, secure = True)
File: python_server.py
#python server
import json
from network_settings import *
from twitter_settings import *
search_term = 'test'
while 1:
tweet_iter = stream.statuses.filter(track = search_term)
for tweet in tweet_iter:
# check whether this is a valid tweet
if tweet.get('text'):
userName = tweet["user"]["screen_name"]
userTweet = tweet["text"]
# now print our tweet
print ('user: ', userName)
print ('tweet: ', userTweet)
#send data back
delivery1 = json.dumps({'type':'showdown','userName':userName,'userTweet':userTweet})
conn.send(delivery1.encode('utf-8'))
data = conn.recv(BUFFER_SIZE)
data = data.decode('utf-8')
if data == "SEARCH_NOW":
print ('request newest IDS tweets')
x = t.statuses.home_timeline(count=10)
for i in range(10):
try:
#print(x[i])
userName = x[i]['entities']['user_mentions'][0]['screen_name']
userTweet = x[i]['text']
print('username: ', userName)
print('tweet: ', userTweet)
delivery = json.dumps({'type':'display','userName':userName,'userTweet':userTweet})
conn.send(delivery.encode('utf-8'))
except:
print('not valid tweet')
conn.close()
So finally have figured out a solution for this. I ended up using threading to run the stream in it's own thread, then I open another thread every time I do a search. Not sure if I need to close each thread, or if the return takes care of that. If anyone has any thing they thing could be improved, I'm all ears!
Code below:
#!/usr/bin/env python
#python server
import json
import threading
import time
import socket
from twitter import *
import re
#get thread lock ready
thread_lock = threading.Lock()
#set server variables
TCP_IP = '127.0.0.1'
TCP_PORT = 7001
BUFFER_SIZE = 20
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP, TCP_PORT))
s.listen(1)
conn, addr = s.accept()
#print connection address when someone connects
print ('Connection address:', addr)
#fill these in your app!
#twitter auth keys
OAUTH_TOKEN = ''
OAUTH_SECRET = ''
CONSUMER_KEY = ''
CONSUMER_SECRET = ''
t = Twitter(auth=OAuth(OAUTH_TOKEN, OAUTH_SECRET, CONSUMER_KEY, CONSUMER_SECRET))
auth = OAuth(OAUTH_TOKEN, OAUTH_SECRET, CONSUMER_KEY, CONSUMER_SECRET)
stream = TwitterStream(auth = auth, secure = True)
#twitter functions
def pythonSearch():
#lock thread to not interrupt search results
thread_lock.acquire()
print ('request newest tweets')
#get 10 things from timeline
x = t.statuses.home_timeline(count=10)
for i in range(10):
try:
#get username and tweet
userName = x[i]['entities']['user_mentions'][0]['screen_name']
userTweet = x[i]['text']
#print out values
print('username: ', userName)
print('tweet: ', userTweet)
#send json back
delivery = json.dumps({'type':'display','userName':userName,'userTweet':userTweet})
conn.send(delivery.encode('utf-8'))
except:
#not a retweet
print('not valid tweet')
#unlock thread when finished
thread_lock.release()
return
def pythonStream():
#open stream looking for search_term
search_term = 'TESTING'
tweet_iter = stream.statuses.filter(track = search_term)
for tweet in tweet_iter:
# check whether this is a valid tweet
if tweet.get('text'):
#get username and tweet
userName = tweet["user"]["screen_name"]
userTweet = tweet["text"]
# now print our tweet
print ('user: ', userName)
print ('tweet: ', userTweet)
#send json back
delivery1 = json.dumps({'type':'showdown','userName':userName,'userTweet':userTweet})
conn.send(delivery1.encode('utf-8'))
#start main loop
while 1:
#listen for calls
data = conn.recv(BUFFER_SIZE)
data = data.decode('utf-8')
#if someone calls search, do a search
if data == 'SEARCH':
threading.Thread(target = pythonSearch).start()
if data == 'STREAM':
threading.Thread(target = pythonStream).start()
conn.close()
Related
so i have been trying to create a script for a capture the flag level, and lately i have been completely stumped as to how this has not been working out.
#
# Connect to alien server ('localhost', 10000)
#
# Then send each of these values...
# USER
# aliensignal
# PASS
# unlockserver
# SEND
# moonbase
# END
# ...and receive the response from each.
#
# Note: You must receive data back from the server after you send each value
#
from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE,SIG_DFL)
import socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', 10000))
header = "USER: aliensignal"
header1 = "PASS: unlockserver"
header2 = "SEND: moonbase"
header3 = "END"
req = "POST / HTTP/1.1\r\{}\r\n\r\n".format(header)
req1 = "POST / HTTP/1.1\r\{}\r\n\r\n".format(header1)
req3 = "POST / HTTP/1.1\r\{}\r\n\r\n".format(header2)
req4 = "POST / HTTP/1.1\r\{} \r\n\r\n".format(header3)
request1 = bytes(req, "utf-8")
sock.send(request1)
response = sock.recv(2048)
print(response)
request2 = bytes(req1, "utf-8")
sock.send(request2)
response2 = sock.recv(2048)
print(response2)
request3 = bytes(req3, "utf-8")
sock.send(request3)
response3 = sock.recv(2048)
print(response3)
request4 = bytes(req4, "utf-8")
sock.send(request4)
response4 = sock.recv(2048)
print(response4)
sock.close()
I try to setup the socks variable to be used to call to for socket operations, and then i try to define all of the requests that will be made, and type-cast them into bytes type. and i send the requests while waiting for a response, however none is given, and the output just remains blankscreenshot of code editor
ive tried to combine the type-casted request values into a single byte value and then using the socks.sendall(val) to no avail, ive tried to put the values itself into a single string to no avail
I have two programms, who connect via sockets. One is a tweepy StreamListener, where I also preprocess the data with the library "tweet-preprocessor". The other programm shall connect to that socket and analyze the data via Spark Structured Streaming. The Problem is, that Spark doesn't get batches when I preprocess the data before sending them.
This is the StreamListener
import tweepy
import socket
import json
import preprocessor as p
CONSUMER_KEY = ""
CONSUMER_SECRET = ""
ACCESS_TOKEN = ""
ACCESS_TOKEN_SECRET = ""
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
p.set_options(p.OPT.URL, p.OPT.EMOJI, p.OPT.SMILEY)
class MyStreamListener(tweepy.StreamListener):
def __init__(self, csocket):
self.client_socket = csocket
def on_data(self, raw_data):
try:
data = json.loads(raw_data)
clean_text = p.clean(data["text"])
print(clean_text)
self.client_socket.send(clean_text.encode("utf-8"))
return True
except BaseException as e:
print("Error: " + str(e))
return True
def on_error(self, status_code):
print(status_code)
return True
skt = socket.socket()
host = "localhost"
port = 5555
skt.bind((host, port))
skt.listen()
client, address = skt.accept()
myStreamListener = MyStreamListener(csocket=client)
myStream = tweepy.Stream(auth=auth, listener=myStreamListener, )
myStream.filter(track=["Trump"], languages=["en"])
And simple Spark code:
from pyspark.sql import SparkSession
from pyspark.sql.functions import explode, split, size
spark = SparkSession.builder.appName("TwitterSpark").getOrCreate()
lines = spark.readStream.format("socket").option("host", "localhost").option("port", 5555).load()
#tweetlength = lines.select(
# size(split(lines.value, " ")).alias("tweetlength")
#)
query = lines.writeStream.outputMode("update").format("console").start()
query.awaitTermination()
Most likely clean_text does not have a new line character (\n) at the end. Unlike print(clean_text), which automatically adds a new line, socket.send() sends the bytes from clean_text.encode("utf-8") as-is and you need to add the \n explicitly:
self.client_socket.send((clean_text + "\n").encode("utf-8"))
With no \n to separate the lines in the socket data, Spark sees the input as one growing line, unless there are new lines in the tweet text itself.
I am making a multiplayer platforming game, and currently there are server performance issues once I have about 7+ clients connected. I was told earlier last week that I should investigate socketserver since previously I had been writing my own server using merely socket.
Well, after 6 hours of implementation I've got my server working with the new code.
As it stands now, I've seen no performance increase, but that's not surprising. I'm hoping that someone here could point me in the right direction and show me how to get more performance out of this thing.
The server is currently 600 lines of code, but I believe the bottleneck is the socketserver stuff.
So here is (hopefully) the relevant stuff.
And as always... thank you very much :)
import socket
import sys
import threading
import time
import pygame
import random
import json
import levels
import socketserver
class MyUDPHandler(socketserver.BaseRequestHandler):
def handle(self):
data = self.request[0].strip()
socket = self.request[1]
gameserver.listen(data, self.client_address)
class ThreadingUDPServer(socketserver.ThreadingMixIn, socketserver.UDPServer):
pass
class GameServer():
def __init__(self):
self.message_count = 100000
self.message_dump = None
self.text_block = []
self.client_list = []
self.client_hero_dict = {}
self.client_obj_dict = {}
self.queue = []
def listen(self, data, addr):
if addr not in self.client_list:
Client(addr, self.dataHandler(data)) #Makes a client object for future reference
print("Accepted connection: " + repr(addr))
print("List of all connections: " + repr(self.client_list))
for line in servermap.map: #This sends the map to the newly connected client
line = "TILE" + line
threading.Thread(target=self.clientHandler(line, addr))
for obj in identifier.object_dict:
threading.Thread(target=self.clientHandler(identifier.object_dict[obj]))
self.dataHandler(data, addr)
def textHandler(self, data):
if data:
data = "^^%s^^" % str(self.message_count) + data
self.message_count += 1
threading.Thread(target=self.clientHandler(data))
def dataHandler(self, data, addr=None):
data = data.decode()
if data[0:2] == "^^": # if data is a chat message
self.textHandler(data[2:])
if data[0:2] == "*^": # if data is a movement request
self.client_hero_dict[addr].actionfsm(data[2:])
if data[0:6] == "**UN**": #if data is a username
# self.client_obj_dict[addr] = data[6:]
return data[6:]
# self.client_hero_dict[addr].name = data[6:]
# print(self.client_hero_dict[addr].name + " is the Username for " + repr(str(addr)))
def clientHandler(self, message_to_send, specific_client=None):
if type(message_to_send) == dict: #I use dictionaries as simple ways to send object (player/sprite) update information
message_to_send = json.dumps(message_to_send)
if specific_client:
server.socket.sendto(message_to_send.encode(), specific_client)
else:
for client in self.client_list:
server.socket.sendto(message_to_send.encode(), client)
I'm running into issues transferring data over TCP with a remote client and server written in Python. The server is located in a pretty remote region with relatively slow internet connection (<2Mb/sec). When the client is run on the LAN with the server the complete string is transferred (2350 bytes); however, when I run the client outside of the LAN sometimes the string is truncated (1485 bytes) and sometimes the full string comes through (2350 bytes). The size of the truncated string always seems to be 1485 bytes. The full size of the string is well below the set buffer size for the client and server.
I've copied abbreviated versions of the client and server code below, where I have tried to edit out all extraneous details:
Client
import socket
from time import sleep
class FTIRdataClient():
def __init__(self,TCP_IP="xxx.xxx.xxx.xxx",TCP_Port=xxx,BufferSize=4096):
#-----------------------------------
# Configuration parameters of server
#-----------------------------------
self.TCP_IP = TCP_IP
self.TCP_Port = int(TCP_Port)
self.RECV_BUFFER = int(BufferSize)
def writeTCP(self,message):
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((self.TCP_IP,self.TCP_Port))
sock.send(message)
incomming = sock.recv(self.RECV_BUFFER)
sock.close()
except:
print "Unable to connect to data server!!"
incomming = False
return incomming
if __name__ == "__main__":
#----------------------------------
# Initiate remote data client class
#----------------------------------
dataClass = FTIRdataClient(TCP_IP=dataServer_IP,TCP_Port=portNum,BufferSize=4096)
#--------------------------------
# Ask database for all parameters
#--------------------------------
allParms = dataClass.writeTCP("LISTALL")
Server
import os
import sys
import socket
import select
import smtplib
import datetime as dt
class FTIRdataServer(object):
def __init__(self,ctlFvars):
...
def runServer(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.TCP_IP,self.TCP_Port))
#self.server_socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.server_socket.listen(10)
self.connection_list.append(self.server_socket)
#-------------------------------------
# Start loop to listen for connections
#-------------------------------------
while True:
#--------------------
# Get list of sockets
#--------------------
read_sockets,write_sockets,error_sockets = select.select(self.connection_list,[],[],5)
for sock in read_sockets:
#-----------------------
# Handle new connections
#-----------------------
if sock == self.server_socket:
#----------------------------------------------
# New connection recieved through server_socket
#----------------------------------------------
sockfd, addr = self.server_socket.accept()
self.connection_list.append(sockfd)
print "Client (%s, %s) connected" % addr
#-------------------------------------
# Handle incomming request from client
#-------------------------------------
else:
#------------------------
# Handle data from client
#------------------------
try:
data = sock.recv(self.RECV_BUFFER)
#------------------------------------------------
# Three types of call to server:
# 1) set -- sets the value of a data parameter
# 2) get -- gets the value of a data parameter
# 3) write -- write data to a file
#------------------------------------------------
splitVals = data.strip().split()
...
elif splitVals[0].upper() == 'LISTALL':
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParams:
msgLst.append("{0:}={1:}".format(k," ".join(self.dataParams[k])))
msg = ";".join(msgLst)
sock.sendall(msg)
...
else:
pass
#---------------------------------------------------
# Remove client from socket list after disconnection
#---------------------------------------------------
except:
sock.close()
self.connection_list.remove(sock)
continue
#-------------
# Close server
#-------------
self.closeServer()
def closeServer(self):
''' Close the TCP data server '''
self.server_socket.close()
Your help is greatly appreciated!!!
For anyone who is interested I found the solution to this problem. John Nielsen has a pretty good explanation here. Basically, TCP stream only guarantees that bytes will not arrive out of order or be duplicated; however, it does not guarantee how many groups the data will be sent in. So one needs to continually read (socket.recv) until all the data is sent. The previous code work on the LAN because the server was sending the entire string in one group. Over a remote connection the string was split into several groups.
I modified the client to continually loop on socket.recv() until the socket is closed and I modified the server to immediately close the socket after sending the data. There are several other ways to do this mentioned in the above link. The new code looks like:
Client
class FTIRdataClient(object):
def __init__(self,TCP_IP="xxx.xxx.xx.xxx",TCP_Port=xxxx,BufferSize=4024):
#-----------------------------------
# Configuration parameters of server
#-----------------------------------
self.TCP_IP = TCP_IP
self.TCP_Port = int(TCP_Port)
self.RECV_BUFFER = int(BufferSize)
def setParam(self,message):
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((self.TCP_IP,self.TCP_Port))
sock.sendall("set "+message)
#-------------------------
# Loop to recieve all data
#-------------------------
incommingTotal = ""
while True:
incommingPart = sock.recv(self.RECV_BUFFER)
if not incommingPart: break
incommingTotal += incommingPart
sock.close()
except:
print "Unable to connect to data server!!"
incommingTotal = False
return incommingTotal
Server
class FTIRdataServer(object):
def __init__(self,ctlFvars):
...
def runServer(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.TCP_IP,self.TCP_Port))
#self.server_socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.server_socket.listen(10)
self.connection_list.append(self.server_socket)
#-------------------------------------
# Start loop to listen for connections
#-------------------------------------
while True:
#--------------------
# Get list of sockets
#--------------------
read_sockets,write_sockets,error_sockets = select.select(self.connection_list,[],[],5)
for sock in read_sockets:
#-----------------------
# Handle new connections
#-----------------------
if sock == self.server_socket:
#----------------------------------------------
# New connection recieved through server_socket
#----------------------------------------------
sockfd, addr = self.server_socket.accept()
self.connection_list.append(sockfd)
print "Client (%s, %s) connected" % addr
#-------------------------------------
# Handle incomming request from client
#-------------------------------------
else:
#------------------------
# Handle data from client
#------------------------
try:
data = sock.recv(self.RECV_BUFFER)
...
elif splitVals[0].upper() == 'LISTALL':
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParams:
msgLst.append("{0:}={1:}".format(k," ".join(self.dataParams[k])))
msg = ";".join(msgLst)
sock.sendall(msg)
elif splitVals[0].upper() == 'LISTALLTS': # List all time stamps
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParamTS:
msgLst.append("{0:}={1:}".format(k,self.dataParamTS[k]))
msg = ";".join(msgLst)
sock.sendall(msg)
...
else:
pass
#------------------------
# Close socket connection
#------------------------
sock.close()
self.connection_list.remove(sock)
#------------------------------------------------------
# Remove client from socket list if client discconnects
#------------------------------------------------------
except:
sock.close()
self.connection_list.remove(sock)
continue
#-------------
# Close server
#-------------
self.closeServer()
Whatever. This is probably common knowledge and I'm just a little slow.
I have set up an experiment where I pass Modbus traffic over a SSL tunnel (this being the first thing I've ever done in python). I am able to send and receive data but when I send one request numerous requests are actually sent (see screenshot)
I've tried numerous configurations including (in both client and server):
send()--no change
sendall() --no change
setblocking(1)
setblocking(0)--doesn't read all the data
On the server side:
if data == Read_Coils_Answer-- I don't think I'm converting the big endian properly for comparison and this didn't work
while data: --the while loop seems to be the only way to prevent either side from stopping short with a "Broken Pipe" error. So this is what I'm using.
I eventually plan to use a for loop (now commented out and set to 4).
My Server code:
from ModLib import *
import socket, ssl, sys, pprint
try:
bindsocket = socket.socket()
bindsocket.bind(('', 502))
bindsocket.listen(5)
bindsocket.setblocking(1)
def do_something(connstream, data):
readCoilsReq = str('\x01\x01\x00')
answer = str(ModbusPDU01_Read_Coils_Answer)
while data:
print ("Request Recevied from Client:")
print pprint.pformat(data)
connstream.send(answer)
print ("Answer Sent to Client")
print pprint.pformat(answer)
return False
def deal_with_client(connstream):
data = connstream.recv(64)
while data:
if not do_something(connstream, data):
break
data = connstream.recv(64)
while True:
newsocket, fromaddr = bindsocket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile="server.crt",
keyfile="server.key",
ssl_version=ssl.PROTOCOL_TLSv1)
try:
deal_with_client(connstream)
finally:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except KeyboardInterrupt:
print ("\nTerminating Session at User Request")
print ("No More Data Will be Sent/Recieved\n")
sys.exit(1)
My Client Side code:
from ModLib import *
from time import sleep
import socket, ssl, pprint
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_sock = ssl.wrap_socket(s,
ca_certs="server.crt",
cert_reqs=ssl.CERT_REQUIRED)
ssl_sock.connect(('localhost', 502))
ssl_sock.setblocking(1)
readCoils = ModbusPDU01_Read_Coils()
#for i in range(4):
sleep(2)
ssl_sock.sendall(str(readCoils))
print ("Request for Read Coils Sent")
#start receive
data = ssl_sock.recv(64)
print ("Response from Server:")
print pprint.pformat(data)
if False: #from the python docs
ssl_sock.write("""GET / HTTP/1.0\r
Host: www.verisign.com\n\n""")
data = ssl_sock.read()
ssl_sock.close()
The do_something() loop was not necessary, as the deal_with_client() loop was doing the same thing. I removed do_something() and put the code in deal_with_client() which allows me to keep the connection open (see below)
from ModLib import *
import socket, ssl, sys, pprint
try:
bindsocket = socket.socket()
bindsocket.bind(('', 502))
bindsocket.listen(5)
bindsocket.setblocking(1)
def deal_with_client(connstream):
data = connstream.recv(1120)
answer = str(ModbusPDU01_Read_Coils_Answer())
while data:
print ("Request Received from Client:")
print pprint.pformat(data)
connstream.send(answer)
print ("Answer Sent to Client")
print pprint.pformat(answer)
data = connstream.recv(1120)
while True:
newsocket, fromaddr = bindsocket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile="server.crt",
keyfile="server.key",
ssl_version=ssl.PROTOCOL_TLSv1)
try:
deal_with_client(connstream)
finally:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except KeyboardInterrupt:
print ("\nTerminating Session at User Request")
print ("No More Data Will be Sent/Received\n")
sys.exit(1)