I need to create a MultiService that loads asynchronously in a twisted plugin, is there a way of achieving that ?
Here what I've done so far:
app/plugins.py
class Options(usage.Options):
synopsis = '[options]'
longdesc = (
'Starts app modules services. The modules must '
'be configured and enabled for the current server '
'before being started. To see the list installed modules '
'use the --list switch.')
optFlags = [['list', 'l', 'Display the list of installed modules.']]
def makeService(options):
from twisted.internet import reactor
debug = options['debug']
return app.ModuleService.load(reactor)
twisted/plugins/app_plugins.py
TestService = ServiceMaker(
'test',
'app.plugins',
'Test service.',
'test')
app.py
class ModuleService(service.MultiService):
def __init__(self, reactor=None):
# Twisted old style classes
service.MultiService.__init__(self)
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
#classmethod
#defer.inlineCallbacks
def load(cls, reactor=None):
modules = yield get_modules()
service = cls(reactor)
for module in modules:
# module tcp server
mod_endpoint = endpoints.TCP4ServerEndpoint(reactor, module.port)
module_service = internet.StreamServerEndpointService(
mod_endpoint,
pb.PBServerFactory(spread.RunnerServer()))
module_service.setServiceParent(service)
defer.returnValue(service)
So my problem is that the MultiService is loaded asynchronously and therefore cannot be used in the makeService function, someone can help me with this ?
Services in twisted are not supposed to start asynchronously. Service start is done before reactoru startup. You should change your application architecture. I am currently studying how :D
Related
I have programmed two servers that essentiall do the same with different methods. Both are webservers with a non-blocking ssl wrapper around the socket object. Threads allow multiple requests to be served at the same time.
One uses ThreadingTCPServer and the other calls ThreadingMixIn on the HTTPServer module.
Which one should I use and why if any of them is better than the other?
ThreadingMixIn on the HTTPServer:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os, ssl
from socketserver import ThreadingMixIn
from http.server import SimpleHTTPRequestHandler, HTTPServer
MYSERV_WORKDIR = "/media/kingdian/server_pub"
#MYSERV_CLIENTCRT = "/home/ran/keys/client.pem"
MYSERV_FULLCHAIN = "/home/ran/.acme.sh/example.com_ecc/fullchain.cer"
MYSERV_PRIVKEY = "/home/ran/.acme.sh/example.com_ecc/example.com.key"
global sslcontext
sslcontext = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
sslcontext.options |= ssl.OP_NO_TLSv1
sslcontext.options |= ssl.OP_NO_TLSv1_1
#sslcontext.options |= ssl.OP_NO_TLSv1_2
#sslcontext.protocol = ssl.PROTOCOL_TLS
#sslcontext.verify_mode = ssl.CERT_REQUIRED
sslcontext.set_ciphers("ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305")
sslcontext.set_ecdh_curve("secp384r1")
#sslcontext.load_verify_locations(MYSERV_CLIENTCRT)
sslcontext.load_cert_chain(MYSERV_FULLCHAIN, MYSERV_PRIVKEY)
class HSTSHandler(SimpleHTTPRequestHandler):
def end_headers(self):
self.send_header("Strict-Transport-Security", "max-age=63072000; includeSubDomains; preload")
self.send_header("Content-Security-Policy", "default-src 'self'")
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("X-Frame-Options", "SAMEORIGIN")
self.send_header("X-XSS-Protection", "1; mode=block")
self.send_header("Referrer-Policy", "no-referrer")
SimpleHTTPRequestHandler.end_headers(self)
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
daemon_threads = True
def main():
try:
os.chdir(MYSERV_WORKDIR)#auto-change working directory
SimpleHTTPRequestHandler.sys_version = "002"#display custom Python system version
SimpleHTTPRequestHandler.server_version = "001"#display custom server software version
my_server = ThreadedHTTPServer(('', 443), HSTSHandler)
my_server.socket = sslcontext.wrap_socket(my_server.socket, do_handshake_on_connect=False, server_side=True)
print('Starting server, use <Ctrl-C> to stop')
my_server.serve_forever()
except KeyboardInterrupt:
print(' received, shutting down server')
my_server.shutdown()
if __name__ == '__main__':
main()
ThreadingTCPServer:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os, ssl, socketserver
from http.server import SimpleHTTPRequestHandler
MYSERV_WORKDIR = "/media/kingdian/server_pub"
#MYSERV_CLIENTCRT = "/home/ran/keys/client.pem"
MYSERV_FULLCHAIN = "/home/ran/.acme.sh/example.com_ecc/fullchain.cer"
MYSERV_PRIVKEY = "/home/ran/.acme.sh/example.com_ecc/example.com.key"
global sslcontext
sslcontext = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
sslcontext.options |= ssl.OP_NO_TLSv1
sslcontext.options |= ssl.OP_NO_TLSv1_1
#sslcontext.options |= ssl.OP_NO_TLSv1_2
#sslcontext.protocol = ssl.PROTOCOL_TLS
#sslcontext.verify_mode = ssl.CERT_REQUIRED
sslcontext.set_ciphers("ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305")
sslcontext.set_ecdh_curve("secp384r1")
#sslcontext.load_verify_locations(MYSERV_CLIENTCRT)
sslcontext.load_cert_chain(MYSERV_FULLCHAIN, MYSERV_PRIVKEY)
class HSTSHandler(SimpleHTTPRequestHandler):
def end_headers(self):
self.send_header("Strict-Transport-Security", "max-age=63072000; includeSubDomains; preload")
self.send_header("Content-Security-Policy", "default-src 'self'")
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("X-Frame-Options", "SAMEORIGIN")
self.send_header("X-XSS-Protection", "1; mode=block")
self.send_header("Referrer-Policy", "no-referrer")
SimpleHTTPRequestHandler.end_headers(self)
def main():
try:
os.chdir(MYSERV_WORKDIR)#auto-change working directory
SimpleHTTPRequestHandler.sys_version = "002"#display custom Python system version
SimpleHTTPRequestHandler.server_version = "001"#display custom server software version
my_server = socketserver.ThreadingTCPServer(('', 443), HSTSHandler)
my_server.socket = sslcontext.wrap_socket(my_server.socket, do_handshake_on_connect=False, server_side=True)
print('Starting server, use <Ctrl-C> to stop')
my_server.serve_forever()
except KeyboardInterrupt:
print(' received, shutting down server')
my_server.shutdown()
if __name__ == '__main__':
main()
Those are pretty much two different ways to get to the same end result. If you look at the implementation of ThreadingTCPServer, it's just:
class ThreadingTCPServer(ThreadingMixIn, TCPServer): pass
In the other example, HTTPServer is just a very small wrapper around TCPServer (adds an override in server_bind to store the server hostname and automatically sets allow_reuse_address) and then your code is directly adding the ThreadingMixIn.
So, ultimately, you're going through the same basic code either way. The class hierarchy in the python library has changed slightly over time and so different code examples have ended up doing things in different combinations.
Personally, for your examples, I find the version that uses ThreadingTCPServer cleaner, just because one extra class is handled elsewhere and the code is slightly smaller therefore. OTOH, the other version allows slightly more flexibility since you've already defined a point at which you can control attributes of the threading mix-in, plus you get the additional HTTPServer wrapper bits if that should become important to you.
I am using Klein, a micro web-framework based on twisted. I have a server (running on windows!), which will spawn an external long running process (end-to-end test) via reactor.spawnProcess().
To send status information about the running test, I implemented a ProcessProtocol:
class IPCProtocol(protocol.ProcessProtocol):
def __init__(self, status: 'Status', history: 'History'):
super().__init__()
self.status: Status = status
self.history: History = history
self.pid = None
def connectionMade(self):
self.pid = self.transport.pid
log.msg("process started, pid={}".format(self.pid))
def processExited(self, reason):
log.msg("process exited, status={}".format(reason.value.exitCode))
# add current run to history
self.history.add(self.status.current_run)
# create empty testrun and save status
self.status.current_run = Testrun()
self.status.status = StatusEnum.ready
self.status.save()
# check for more queue items
if not self.status.queue.is_empty():
start_testrun()
def outReceived(self, data: bytes):
data = data.decode('utf-8').strip()
if data.startswith(constants.LOG_PREFIX_FAILURE):
self.failureReceived()
if data.startswith(constants.LOG_PREFIX_SERVER):
data = data[len(constants.LOG_PREFIX_SERVER):]
log.msg("Testrunner: " + data)
self.serverMsgReceived(data)
I start the process with the following command:
ipc_protocol = IPCProtocol(status=app.status, history=app.history)
args = [sys.executable, 'testrunner.py', next_entry.suite, json.dumps(next_entry.testscripts)]
log.msg("Starting testrunn.py with args: {}".format(args))
reactor.spawnProcess(ipc_protocol, sys.executable, args=args)
To send information, I just print out messages (with a prefix to distinct them) in my testrunner.py.
The problem is that if I send the print commands to fast, then outReceived will merge the messages.
I already tried adding a flush=True for print() calls in the external process, but this didn't fix the problem. Some other question suggested using usePTY=True for the spawnProcess but this is not supported under windows.
Is there a better way to fix this, than adding a small delay (like time.sleep(0.1)) to each print()call?
You didn't say it, but it seems like the child process writes lines to its stdout.
You need to parse the output to find the line boundaries if you want to operate on these lines.
You can use LineOnlyReceiver to help you with this. Since processes aren't stream transports, you can't just use LineOnlyReceiver directly. You have to adapt it to the process protocol interface. You can do this yourself or you can use ProcessEndpoint (instead of spawnProcess) to do it for you.
For example:
from twisted.protocols.basic import LineOnlyReceiver
from twisted.internet.protocol import Factory
from twisted.internet.endpoints import ProcessEndpoint
from twisted.internet import reactor
endpoint = ProcessEndpoint(reactor, b"/some/some-executable", ...)
spawning_deferred = endpoint.connect(Factory.forProtocol(LineOnlyReceiver))
...
I am trying to build a COM server to get real time information. The problem is that all other functionalities stop while localserver is opened. The rest of the code just runs when the localserver is closed.
I have searched for solutions and failed trying multiprocessing, not because this wouldn't work, I guess because I suck. Anyway, I am stuck in this part.
import pythoncom
import win32com
from win32com.server import localserver
from multiprocessing import Process
class PythonUtilities(object):
_reg_clsid_ = '{D9C54599-9011-4678-B1EB-A07FD272F0AF}'
_reg_desc_ = "Change information between programs"
_reg_progid_ = "Python.LetsTalk"
_public_attrs_ = ['speech', 'roger']
_readonly_attrs_ = ['roger']
_public_methods_ = ['talktome']
def __init__(self):
self.roger = 'roger'
self.speech = None
def talktome(self,speech):
self.speech = speech
print ('New speech received: ' + self.speech)
return self.roger
### ___ ###
def runserver(mess):
print(mess)
localserver.serve(['{D9C54599-9011-4678-B1EB-A07FD272F0AF}'])
if __name__=='__main__':
pu = PythonUtilities
print ("Registering COM Server ")
win32com.server.register.UseCommandLine(pu)
# Fine so far.
# The problem starts here:
localserver.serve(['{D9C54599-9011-4678-B1EB-A07FD272F0AF}'])
#... rest of the code waiting for localserver be closed
# Experiment... Doesnt work:
#proc = Process(target=runserver, args = ('starting process',))
#proc.start()
#proc.join()
It's important to say that all messages sent from the client seem to be correctly displayed BUT ONLY AFTER I close the local server manually. I want to receive it in real time like a chat app. I mean, I want to keep the localserver opened and being able to work with the information received along the rest of the code.
My problem was solved rewriting the localserver.serve() function and starting it in a new thread as the code below.
import pythoncom
from win32com.client import Dispatch # to get attributes
from win32com.server import register, factory
from threading import Thread
from queue import Queue
class PythonUtilities(object):
_reg_clsid_ = '{D9C54599-9011-4678-B1EB-A07FD272F0AF}'
_reg_desc_ = "Change information between programs"
_reg_progid_ = "Python.LetsTalk"
_public_attrs_ = ['speech']
_public_methods_ = ['talktome']
queue_speech = Queue()
def talktome(self,speech):
self.queue_speech.put(speech)
print ('New speech received: ' + speech)
return 'roger'
### ___ ###
# use instead localserver.serve()
def runserver():
# added - multithread support
pythoncom.CoInitialize()
clsids = ['{D9C54599-9011-4678-B1EB-A07FD272F0AF}']
infos = factory.RegisterClassFactories(clsids)
# commented - from original localserver.serve() method
#pythoncom.EnableQuitMessage(win32api.GetCurrentThreadId())
pythoncom.CoResumeClassObjects()
pythoncom.PumpMessages()
factory.RevokeClassFactories( infos )
pythoncom.CoUninitialize()
if __name__=='__main__':
#use this
server_thread = Thread(target=runserver) # Process works as well
server_thread.start()
#instead this
#localserver.serve(['{D9C54599-9011-4678-B1EB-A07FD272F0AF}'])
#... rest of the code now works in parallel
Also I have made some improvements like Queue to get data later. I hope it can help others.
I need to implement a scheduled task in our Django app. DBader's schedule seems to be a good candidate for the job, however when run it as part of a Django project, it doesn't seem to produce the desired effect.
Specifically, this works fine as an independent program:
import schedule
import time
import logging
log = logging.getLogger(__name__)
def handleAnnotationsWithoutRequests(settings):
'''
From settings passed in, grab job-ids list
For each job-id in that list, perform annotation group/set logic [for details, refer to handleAnnotationsWithRequests(requests, username)
sans requests, those are obtained from db based on job-id ]
'''
print('Received settings: {}'.format(str(settings)))
def job():
print("I'm working...")
#schedule.every(3).seconds.do(job)
#schedule.every(2).seconds.do(handleAnnotationsWithoutRequests, settings={'a': 'b'})
invoc_time = "10:33"
schedule.every().day.at(invoc_time).do(handleAnnotationsWithoutRequests, settings={'a': 'b'})
while True:
schedule.run_pending()
time.sleep(1)
But this (equivalent) code run in Django context doesn't result in an invocation.
def handleAnnotationsWithoutRequests(settings):
'''
From settings passed in, grab job-ids list
For each job-id in that list, perform annotation group/set logic [for details, refer to handleAnnotationsWithRequests(requests, username)
sans requests, those are obtained from db based on job-id ]
'''
log.info('Received settings: {}'.format(str(settings)))
def doSchedule(settings):
'''
with scheduler library
Based on time specified in settings, invoke .handleAnnotationsWithoutRequests(settings)
'''
#settings will need to be reconstituted from the DB first
#settings = {}
invocationTime = settings['running_at']
import re
invocationTime = re.sub(r'([AaPp][Mm])', "", invocationTime)
log.info("Invocation time to be used: {}".format(invocationTime))
schedule.every().day.at(invocationTime).do(handleAnnotationsWithoutRequests, settings=settings)
while True:
schedule.run_pending()
time.sleep(1)
so the log from handleAnnotationsWithoutRequests() doesn't appear on the console.
Is this scheduling library compatible with Django? Are there any usage samples that one could refer me to?
I'm suspecting some thread issues are at work here. Perhaps there are better alternatives to be used? Suggestions are welcome.
Thank you in advance.
For web servers, you probably don't want something that runs in-process:
An in-process scheduler for periodic jobs [...]
https://github.com/Tivix/django-cron has proven a working solution.
There's also the heavyweight champion Celery and Celerybeat.
I do this a lot with Django Commands
The pattern I use is to setup a new Django command in my app and then make it a long-running process inside a never-ending while() loop.
I the loop iterates continuously with a custom defined sleep(1) timer.
The short version is here, with a bit of pseudo-code thrown in. You can see a working version of this pattern in my Django Reference Implementation.
class Command(BaseCommand):
help = 'My Long running job'
def handle(self, *args, **options):
self.stdout.write(self.style.SUCCESS(f'Starting long-running job.'))
while True:
if conditions met for job:
self.job()
sleep(5)
def job(self):
self.stdout.write(self.style.SUCCESS(f'Running the job...'))
I currently have a Python file that when run using python file_name.py installs a Windows service that is viewable in Event Viewer under application logs and stoppable using sc stop service_name. However, when converted into an executable using cx_Freeze, the executable runs with no errors but the service no longer installs. This happens if I run just the executable by itself, if I run service_name.exe --install service_name, or if I run sc create service_name binPath=service_path
My setup.py file looks something like:
from cx_Freeze import setup, Executable
options = {
'build_exe': {
'packages': ['packagename'],
'includes': ['ServiceHandler', 'cx_Logging']}
}
setup(name='cx_FreezeSampleService',
version='0.1',
description='Sample cx_Freeze Windows serice',
executables=Executable('Config.py', base='Win32Service',
targetName='cx_FreezeSampleService.exe'),
options=options
)
My Config.py looks something like:
NAME = 'cx_FreezeSampleService%s'
DISPLAY_NAME = 'cx_Freeze Sample Service - %s'
MODULE_NAME = 'ServiceHandler'
CLASS_NAME = 'Handler'
DESCRIPTION = 'Sample service description'
AUTO_START = True
SESSION_CHANGES = False
And finally, my ServiceHandler.py looks something like:
class Handler(object):
def Initialize(self, Config):
pass
def Run(self):
#code to run service
def Stop(self):
#code to stop service
This code follows the example at the cx_Freeze source code here (https://bitbucket.org/anthony_tuininga/cx_freeze/src/1282b6b6ee637738210113dd88c3c198d475340f/cx_Freeze/samples/service/?at=default) almost exactly, but neither this nor the example seem to work in actually installing a service.
Thank you in advance!
This is an old question, but I manage to get it working as a window service for a simple flask application with the help of the developers.
[https://github.com/marcelotduarte/cx_Freeze/tree/master/cx_Freeze/samples/service]
You have to set up all the windows service actions you want performance.
this is how the ServiceHandler.py should look like as a template you still need to adapt to run your application.
"""
Implements a simple service using cx_Freeze.
See below for more information on what methods must be implemented and how they
are called.
"""
import threading
import os
import sys
import cx_Logging
class Handler:
# no parameters are permitted; all configuration should be placed in the
# configuration file and handled in the Initialize() method
def __init__(self):
self.stopEvent = threading.Event()
self.stopRequestedEvent = threading.Event()
# called when the service is starting
def initialize(self, configFileName):
self.directory = os.path.dirname(sys.executable)
cx_Logging.StartLogging(os.path.join(self.directory, "teste.log"), cx_Logging.DEBUG)
#pass
# called when the service is starting immediately after Initialize()
# use this to perform the work of the service; don't forget to set or check
# for the stop event or the service GUI will not respond to requests to
# stop the service
def run(self):
cx_Logging.Debug("stdout=%r", sys.stdout)
sys.stdout = open(os.path.join(self.directory, "stdout.log"), "a")
sys.stderr = open(os.path.join(self.directory, "stderr.log"), "a")
self.stopRequestedEvent.wait()
self.stopEvent.set()
# called when the service is being stopped by the service manager GUI
def stop(self):
self.stopRequestedEvent.set()
self.stopEvent.wait()