We Keep Coding

Running bokeh server programmatically to show in browser locally

I am making an interactive data manipulation with bokeh (0.12.6) utility that I will deploy within a package. The idea is that a user can run a some routine module.utility() that will start the bokeh server, launch the application in a browser, and when the tab or browser are closed, the server will be stopped.
My application launches fine if I run bokeh serve --show myapp, but it hangs when connecting to the localhost using my method described below. I've inspected the handlers, everything looks as it should.
Is this a reasonable thing to do, and am I going about it correctly?
Directory format
<installed module path>/myapp
└── main.py
Where ./myapp will reside in venv/lib/python3.5/site-packages/mymodule etc.
main.py
from bokeh.io import curdoc
from bokeh.layouts import column
from bokeh.plotting import figure
from bokeh.models.sources import ColumnDataSource
source = ColumnDataSource(dict(x=list(range(5)), y=list(range(5))))
p = figure(width=300, height=300, tools=[], toolbar_location=None)
p.line(x='x', y='y', source=source)
curdoc().add_root(column(p, sizing_mode='scale_width'))
Run script
def run_single_server(abs_app_path, port=5000):
'''Run bokeh application for single session from server`'''
from bokeh.application import Application
from bokeh.application.handlers import DirectoryHandler
from bokeh.server.server import Server
import os
app_name = os.path.split(abs_app_path)[1]
url = '/{}'.format(app_name)
# Start a bokeh server
apps = {url:Application(DirectoryHandler(filename=abs_app_path))}
server = Server(apps, port=port)
server.start()
server.show(url)
# somehow wait for session to end, perhaps using `server_lifecycle.py`
server.stop()
return
def utility():
import mymodule
module_path = os.path.split(mymodule.__file__)[0]
abs_app_path = os.path.join(module_path, 'myapp')
run_single_server(abs_app_path, port=5000)
return
Perhaps have that routine in the main __init__.py, and have it work like this:
import mymodule
mymodule.utility()
# 1. Browser launches
# 2. user does stuff
# 3. user closes window
# 4. bokeh server is shutdown
Update
I found the build_single_handler_application routine and tried that, but it also appears to hang.
from bokeh.command.util import build_single_handler_application
import os
app_name = os.path.split(abs_app_path)[1]
url = '/{}'.format(app_name)
# Start a bokeh server
apps = build_single_handler_application(abs_app_path)

It looks like I had a couple of problems. I ended up finding and adapting some code that I found on the mail group here for my use-case.
I managed to get everything to work by using separate process for 1) starting the server, 2) launching the app urls with webbrowser, and 3) checking for closed connections and shutting down.
I think I could perhaps do away with initiating the tornado server instance as was done in the flask example I adapted, but I'm happy here.
Note: this example uses single file apps, but you can pass the paths of directory formatted apps as well.
def create_bokeh_server(io_loop, files, argvs, host, port):
'''Start bokeh server with applications paths'''
from bokeh.server.server import Server
from bokeh.command.util import build_single_handler_applications
# Turn file paths into bokeh apps
apps = build_single_handler_applications(files, argvs)
# kwargs lifted from bokeh serve call to Server, with created io_loop
kwargs = {
'io_loop':io_loop,
'generate_session_ids':True,
'redirect_root':True,
'use_x_headers':False,
'secret_key':None,
'num_procs':1,
'host': host,
'sign_sessions':False,
'develop':False,
'port':port,
'use_index':True
}
server = Server(apps,**kwargs)
return server
def run_single_app(files, port=5000, new='tab'):
def start_bokeh(io_loop):
'''Start the `io_loop`'''
io_loop.start()
return None
def launch_app(host, app_name, new):
'''Lauch app in browser
Ideally this would `bokeh.util.browser.view()`, but it doesn't work
'''
import webbrowser
# Map method strings to webbrowser method
options = {'current':0, 'window':1, 'tab':2}
# Concatenate url and open in browser, creating a session
app_url = 'http://{}/{}'.format(host, app_name)
print('Opening `{}` in browser'.format(app_url))
webbrowser.open(app_url, new=options[new])
return None
def server_loop(server, io_loop):
'''Check connections once session created and close on disconnect'''
import time
connected = [True,]
session_loaded = False
while any(connected):
# Check if no session started on server
sessions = server.get_sessions()
if not session_loaded:
if sessions:
session_loaded = True
# Once 1+ sessions started, check for no connections
else:
# List of bools for each session
connected = [True,]*len(sessions)
# Set `connected` item false no connections on session
for i in range(len(sessions)):
if sessions[i].connection_count == 0:
connected[i] = False
# Keep the pace down
time.sleep(2)
# Stop server once opened session connections closed
io_loop.stop()
return None
import os
import threading
import tornado.ioloop
import tornado.autoreload
import time
# Initialize some values, sanatize the paths to the bokeh plots
argvs = {}
app_names = []
for path in files:
argvs[path] = None
app_names.append(os.path.splitext(os.path.split(path)[1])[0])
# Concate hostname/port for creating handlers, launching apps
host = 'localhost:{}'.format(port)
# Initialize the tornado server
io_loop = tornado.ioloop.IOLoop.instance()
tornado.autoreload.start(io_loop)
# Add the io_loop to the bokeh server
server = run_bokeh_server(io_loop, files, argvs, host, port)
print('Starting the server on {}'.format(host))
args = (io_loop,)
th_startup = threading.Thread(target=start_bokeh, args=args)
th_startup.start()
# Launch each application in own tab or window
th_launch = [None,]*len(app_names)
for i in range(len(app_names)):
args = (host, app_names[i], new)
th_launch[i] = threading.Thread(target=launch_app, args=args)
th_launch[i].start()
# Delay to allow tabs to open in same browser window
time.sleep(2)
# Run session connection test, then stop `io_loop`
args = (server, io_loop)
th_shutdown = threading.Thread(target=server_loop, args=args)
th_shutdown.start()
return None
if __name__ == "__main__":
import os
files = [os.path.join('bokeh', fname) for fname in ['ex1.py','ex2.py']]
run_single_app(files, port=5006)

How to add jobs to apscheduler via request using uwsgi + flask_apscheduler

When run under Flask's local development server, jobs are added and run normally. When run under uWSGI, the job appears to get added to the job store but never is executed. A simple example with the described undesired behavior is given below:
__init__.py
import flask
from datetime import datetime, timedelta
from flask_apscheduler import APScheduler
app = flask.Flask("apscheduler_test")
app.config["SCHEDULER_API_ENABLED"] = True
scheduler = APScheduler()
scheduler.init_app(app)
scheduler.start()
def test_job():
print("test job run")
#app.route("/test")
def apscheduler_test():
print("Adding Job")
scheduler.add_job(id="101",
func=test_job,
next_run_time=(datetime.now() + timedelta(seconds=10)))
return "view", 200
if __name__ == '__main__':
app.run(port=5050)
apschedule_test.ini
[uwsgi]
pidfile = /var/run/%n.pid
chdir = /opt/apscheduler
master = true
enable-threads = true
threads = 20
http-socket = :48197
logto = /var/log/%n.log
plugin = python3
module = %n
callable = app
processes = 1
uid = root
gid = root
daemonize = /var/log/apscheduler_test.log

Try adding this flag to your ini file:
lazy-apps=true
Similar issue: uWSGI lazy-apps and ThreadPool

Twisted makeService() async

I need to create a MultiService that loads asynchronously in a twisted plugin, is there a way of achieving that ?
Here what I've done so far:
app/plugins.py
class Options(usage.Options):
synopsis = '[options]'
longdesc = (
'Starts app modules services. The modules must '
'be configured and enabled for the current server '
'before being started. To see the list installed modules '
'use the --list switch.')
optFlags = [['list', 'l', 'Display the list of installed modules.']]
def makeService(options):
from twisted.internet import reactor
debug = options['debug']
return app.ModuleService.load(reactor)
twisted/plugins/app_plugins.py
TestService = ServiceMaker(
'test',
'app.plugins',
'Test service.',
'test')
app.py
class ModuleService(service.MultiService):
def __init__(self, reactor=None):
# Twisted old style classes
service.MultiService.__init__(self)
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
#classmethod
#defer.inlineCallbacks
def load(cls, reactor=None):
modules = yield get_modules()
service = cls(reactor)
for module in modules:
# module tcp server
mod_endpoint = endpoints.TCP4ServerEndpoint(reactor, module.port)
module_service = internet.StreamServerEndpointService(
mod_endpoint,
pb.PBServerFactory(spread.RunnerServer()))
module_service.setServiceParent(service)
defer.returnValue(service)
So my problem is that the MultiService is loaded asynchronously and therefore cannot be used in the makeService function, someone can help me with this ?

Services in twisted are not supposed to start asynchronously. Service start is done before reactoru startup. You should change your application architecture. I am currently studying how :D

How to install a Python Windows service using cx_Freeze?

I currently have a Python file that when run using python file_name.py installs a Windows service that is viewable in Event Viewer under application logs and stoppable using sc stop service_name. However, when converted into an executable using cx_Freeze, the executable runs with no errors but the service no longer installs. This happens if I run just the executable by itself, if I run service_name.exe --install service_name, or if I run sc create service_name binPath=service_path
My setup.py file looks something like:
from cx_Freeze import setup, Executable
options = {
'build_exe': {
'packages': ['packagename'],
'includes': ['ServiceHandler', 'cx_Logging']}
}
setup(name='cx_FreezeSampleService',
version='0.1',
description='Sample cx_Freeze Windows serice',
executables=Executable('Config.py', base='Win32Service',
targetName='cx_FreezeSampleService.exe'),
options=options
)
My Config.py looks something like:
NAME = 'cx_FreezeSampleService%s'
DISPLAY_NAME = 'cx_Freeze Sample Service - %s'
MODULE_NAME = 'ServiceHandler'
CLASS_NAME = 'Handler'
DESCRIPTION = 'Sample service description'
AUTO_START = True
SESSION_CHANGES = False
And finally, my ServiceHandler.py looks something like:
class Handler(object):
def Initialize(self, Config):
pass
def Run(self):
#code to run service
def Stop(self):
#code to stop service
This code follows the example at the cx_Freeze source code here (https://bitbucket.org/anthony_tuininga/cx_freeze/src/1282b6b6ee637738210113dd88c3c198d475340f/cx_Freeze/samples/service/?at=default) almost exactly, but neither this nor the example seem to work in actually installing a service.
Thank you in advance!

This is an old question, but I manage to get it working as a window service for a simple flask application with the help of the developers.
[https://github.com/marcelotduarte/cx_Freeze/tree/master/cx_Freeze/samples/service]
You have to set up all the windows service actions you want performance.
this is how the ServiceHandler.py should look like as a template you still need to adapt to run your application.
"""
Implements a simple service using cx_Freeze.
See below for more information on what methods must be implemented and how they
are called.
"""
import threading
import os
import sys
import cx_Logging
class Handler:
# no parameters are permitted; all configuration should be placed in the
# configuration file and handled in the Initialize() method
def __init__(self):
self.stopEvent = threading.Event()
self.stopRequestedEvent = threading.Event()
# called when the service is starting
def initialize(self, configFileName):
self.directory = os.path.dirname(sys.executable)
cx_Logging.StartLogging(os.path.join(self.directory, "teste.log"), cx_Logging.DEBUG)
#pass
# called when the service is starting immediately after Initialize()
# use this to perform the work of the service; don't forget to set or check
# for the stop event or the service GUI will not respond to requests to
# stop the service
def run(self):
cx_Logging.Debug("stdout=%r", sys.stdout)
sys.stdout = open(os.path.join(self.directory, "stdout.log"), "a")
sys.stderr = open(os.path.join(self.directory, "stderr.log"), "a")
self.stopRequestedEvent.wait()
self.stopEvent.set()
# called when the service is being stopped by the service manager GUI
def stop(self):
self.stopRequestedEvent.set()
self.stopEvent.wait()

How to create Celery Windows Service?

I'm trying to create a Windows Service to launch Celery. I have come across an article that does it using Task Scheduler. However it seems to launch numerous celery instances and keeps eating up memory till the machine dies. Is there any way to launch it as a Windows service?

I got the answer from another website. Celeryd (daemon service for Celery) runs as a paster application, searching for 'Paster Windows Service' lead me here. It describes how to run a Pylons application as a Windows Service. Being new to paster framework and hosting python web services, it didn't cross my mind to check it at first. But that solution works for Celery with a slight change here and there in the script.
I've modified the script to make it easier for modifying Celery settings. The essential changes are:
Create an INI file with the settings for Celery service (shown below)
Create a python script to create a Windows service.
INI file settings (celeryd.ini):
[celery:service]
service_name = CeleryService
service_display_name = Celery Service
service_description = WSCGI Windows Celery Service
service_logfile = celeryd.log
Python script to create Windows Service (CeleryService.py):
"""
The most basic (working) Windows service possible.
Requires Mark Hammond's pywin32 package.
Most of the code was taken from a CherryPy 2.2 example of how to set up a service
"""
import pkg_resources
import win32serviceutil
from paste.script.serve import ServeCommand as Server
import os, sys
import ConfigParser
import win32service
import win32event
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
INI_FILE = 'celeryd.ini'
SERV_SECTION = 'celery:service'
SERV_NAME = 'service_name'
SERV_DISPLAY_NAME = 'service_display_name'
SERV_DESC = 'service_description'
SERV_LOG_FILE = 'service_logfile'
SERV_APPLICATION = 'celeryd'
SERV_LOG_FILE_VAR = 'CELERYD_LOG_FILE'
# Default Values
SERV_NAME_DEFAULT = 'CeleryService'
SERV_DISPLAY_NAME_DEFAULT = 'Celery Service'
SERV_DESC_DEFAULT = 'WSCGI Windows Celery Service'
SERV_LOG_FILE_DEFAULT = r'D:\logs\celery.log'
class DefaultSettings(object):
def __init__(self):
if SCRIPT_DIR:
os.chdir(SCRIPT_DIR)
# find the ini file
self.ini = os.path.join(SCRIPT_DIR,INI_FILE)
# create a config parser opject and populate it with the ini file
c = ConfigParser.SafeConfigParser()
c.read(self.ini)
self.c = c
def getDefaults(self):
'''
Check for and get the default settings
'''
if (
(not self.c.has_section(SERV_SECTION)) or
(not self.c.has_option(SERV_SECTION, SERV_NAME)) or
(not self.c.has_option(SERV_SECTION, SERV_DISPLAY_NAME)) or
(not self.c.has_option(SERV_SECTION, SERV_DESC)) or
(not self.c.has_option(SERV_SECTION, SERV_LOG_FILE))
):
print 'setting defaults'
self.setDefaults()
service_name = self.c.get(SERV_SECTION, SERV_NAME)
service_display_name = self.c.get(SERV_SECTION, SERV_DISPLAY_NAME)
service_description = self.c.get(SERV_SECTION, SERV_DESC)
iniFile = self.ini
service_logfile = self.c.get(SERV_SECTION, SERV_LOG_FILE)
return service_name, service_display_name, service_description, iniFile, service_logfile
def setDefaults(self):
'''
set and add the default setting to the ini file
'''
if not self.c.has_section(SERV_SECTION):
self.c.add_section(SERV_SECTION)
self.c.set(SERV_SECTION, SERV_NAME, SERV_NAME_DEFAULT)
self.c.set(SERV_SECTION, SERV_DISPLAY_NAME, SERV_DISPLAY_NAME_DEFAULT)
self.c.set(SERV_SECTION, SERV_DESC, SERV_DESC_DEFAULT)
self.c.set(SERV_SECTION, SERV_LOG_FILE, SERV_LOG_FILE_DEFAULT)
cfg = file(self.ini, 'wr')
self.c.write(cfg)
cfg.close()
print '''
you must set the celery:service section service_name, service_display_name,
and service_description options to define the service
in the %s file
''' % self.ini
sys.exit()
class CeleryService(win32serviceutil.ServiceFramework):
"""NT Service."""
d = DefaultSettings()
service_name, service_display_name, service_description, iniFile, logFile = d.getDefaults()
_svc_name_ = service_name
_svc_display_name_ = service_display_name
_svc_description_ = service_description
def __init__(self, args):
win32serviceutil.ServiceFramework.__init__(self, args)
# create an event that SvcDoRun can wait on and SvcStop
# can set.
self.stop_event = win32event.CreateEvent(None, 0, 0, None)
def SvcDoRun(self):
os.chdir(SCRIPT_DIR)
s = Server(SERV_APPLICATION)
os.environ[SERV_LOG_FILE_VAR] = self.logFile
s.run([self.iniFile])
win32event.WaitForSingleObject(self.stop_event, win32event.INFINITE)
def SvcStop(self):
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
#win32event.SetEvent(self.stop_event)
self.ReportServiceStatus(win32service.SERVICE_STOPPED)
sys.exit()
if __name__ == '__main__':
win32serviceutil.HandleCommandLine(CeleryService)
To install the service run python CeleryService.py install and then python CeleryService.py start to start the service. NOTE: These commands should be run in command-line with administrator rights.
If the service needs to be removed, run python CeleryService.py remove.
I was trying to host Celery as part of enhancing my RhodeCode installation. This solution seems to work. Hope this will help someone.

The accepted answer does not apply for running celery with a Django application. But it inspired me to come up with a solution for running celery as a Windows service with Django. Note that the following is for Django projects only. It may work with other applications with some modifications.
The following discussion assumes Python >= 3.6 and RabbitMQ are already installed, and rabbitmq-server is running on localhost.
Create a file celery_service.py (or whatever you like) inside your Django project's top level folder, same level as manage.py, with the following content:
'''Usage : python celery_service.py install (start / stop / remove)
Run celery as a Windows service
'''
import win32service
import win32serviceutil
import win32api
import win32con
import win32event
import subprocess
import sys
import os
from pathlib import Path
import shlex
import logging
import time
# The directory for celery.log and celery_service.log
# Default: the directory of this script
INSTDIR = Path(__file__).parent
# The path of python Scripts
# Usually it is in path_to/venv/Scripts.
# If it is already in system PATH, then it can be set as ''
PYTHONSCRIPTPATH = INSTDIR / 'venvcelery/Scripts'
# The directory name of django project
# Note: it is the directory at the same level of manage.py
# not the parent directory
PROJECTDIR = 'proj'
logging.basicConfig(
filename = INSTDIR / 'celery_service.log',
level = logging.DEBUG,
format = '[%(asctime)-15s: %(levelname)-7.7s] %(message)s'
)
class CeleryService(win32serviceutil.ServiceFramework):
_svc_name_ = "Celery"
_svc_display_name_ = "Celery Distributed Task Queue Service"
def __init__(self, args):
win32serviceutil.ServiceFramework.__init__(self, args)
self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
def SvcStop(self):
logging.info('Stopping {name} service ...'.format(name=self._svc_name_))
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
self.ReportServiceStatus(win32service.SERVICE_STOPPED)
sys.exit()
def SvcDoRun(self):
logging.info('Starting {name} service ...'.format(name=self._svc_name_))
os.chdir(INSTDIR) # so that proj worker can be found
logging.info('cwd: ' + os.getcwd())
self.ReportServiceStatus(win32service.SERVICE_RUNNING)
command = '"{celery_path}" -A {proj_dir} worker -f "{log_path}" -l info -P eventlet'.format(
celery_path=PYTHONSCRIPTPATH / 'celery.exe',
proj_dir=PROJECTDIR,
log_path=INSTDIR / 'celery.log')
logging.info('command: ' + command)
args = shlex.split(command)
proc = subprocess.Popen(args)
logging.info('pid: {pid}'.format(pid=proc.pid))
self.timeout = 3000
while True:
rc = win32event.WaitForSingleObject(self.hWaitStop, self.timeout)
if rc == win32event.WAIT_OBJECT_0:
# stop signal encountered
# terminate process 'proc'
PROCESS_TERMINATE = 1
handle = win32api.OpenProcess(PROCESS_TERMINATE, False, proc.pid)
win32api.TerminateProcess(handle, -1)
win32api.CloseHandle(handle)
break
if __name__ == '__main__':
win32serviceutil.HandleCommandLine(CeleryService)
Before the script can be run, you need to
Optionally create a python virtual environment e.g. 'venvcelery'.
Install the following requirements:
django>=2.0.0
sqlalchemy>=1.0.14
celery>=4.3.0,<5.0
pywin32>=227
eventlet>=0.25
Fix pywin32 pywintypes36.dll location. ref
Correctly set PYTHONSCRIPTPATH and PROJECTDIR in celery_service.py
PYTHONSCRIPTPATH is usually the "Scripts" folder under your python's installation path or current virtual environment
PROJECTDIR is the directory name of the Django project.
It is the directory at the same level of manage.py, not the parent directory.
Now you can install / start / stop / remove the service with:
python celery_service.py install
python celery_service.py start
python celery_service.py stop
python celery_service.py remove
I created a demo Django project with celery running as a Windows service:
https://github.com/azalea/django_celery_windows_service
In case you are interested in a running example.
Note: this is an updated version assuming Python >= 3.6, Django 2.2 and Celery 4.
An older version with Python 2.7, Django 1.6 and Celery 3 can be viewed in edit history.

#azalea 's answer helped me a lot, but one thing I like to highlight here is, the service (celery_service.py) needs to be installed with your user/password, otherwise, when you run subprocess.Popen(args) in SvcDoRun() function, nothing will happen as there will be a permission issue. To set the user/password, you can choose one of two methods:
Using command line:
python33 .\celeryService1.py --username .\USERNAME --password PASSWORD
Go to Computer Management(local) > Services and Applications > Services, find your server (in #azalea's example, it is "Celery Distributed Task Queue Service"), and right click to open Properties page, input "This account" in Log On tab

A good project here but didn't succeed to use it :
Link to the GitHub of the django-windows-tools.
It gave me a timeout at the last command line. Doesn't have enough time to search why.
The package allow settings FastCGI, Celery and Static files of a Django project on IIS.

Thanks to Azalea as this led me to the path in being able to create 2 windows services with Celery 4 on Windows.
One to be able to start/stop multiple workers T
Second to be able to start/stop the beat service and tidy up the pid using Celery 4.
Only caveat in this that I do not have a solution for is you can not restart the workers as you need to ensure the spawned processes for multiple are stopped before starting back up.
Workers.py:
'''Usage : python celery_service.py install (start / stop / remove)
Run celery as a Windows service
'''
import win32service
import win32serviceutil
import win32api
import win32con
import win32event
import subprocess
import sys
import os
import shlex
import logging
import time
# The directory for celery_worker.log and celery_worker_service.log
# Default: the directory of this script
INSTDIR = 'X:\Application\Project'
LOGDIR = 'X:\Application\LogFiles'
# The path of python Scripts
# Usually it is in PYTHON_INSTALL_DIR/Scripts. e.g.
# r'C:\Python27\Scripts'
# If it is already in system PATH, then it can be set as ''
PYTHONSCRIPTPATH = 'C:\Python36\Scripts'
# The directory name of django project
# Note: it is the directory at the same level of manage.py
# not the parent directory
PROJECTDIR = 'Project'
logging.basicConfig(
filename = os.path.join(LOGDIR, 'celery_worker_service.log'),
level = logging.DEBUG,
format = '[%(asctime)-15s: %(levelname)-7.7s] %(message)s'
)
class CeleryService(win32serviceutil.ServiceFramework):
_svc_name_ = "CeleryWorkers"
_svc_display_name_ = "CeleryWorkers"
def __init__(self, args):
win32serviceutil.ServiceFramework.__init__(self, args)
self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
def SvcStop(self):
logging.info('Stopping {name} service ...'.format(name=self._svc_name_))
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
logging.info('Stopped1 {name} service ...'.format(name=self._svc_name_))
logging.info('Stopped3 {name} service ...'.format(name=self._svc_name_))
command = '"{celery_path}" -A {proj_dir} --workdir=X:/Application/Project control shutdown --timeout=10'.format(
celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
proj_dir=PROJECTDIR,
log_path=os.path.join(LOGDIR,'celery_worker.log'))
logging.info('command: ' + command)
args = shlex.split(command)
proc = subprocess.Popen(args)
logging.info('Stopped celery shutdown ...')
self.ReportServiceStatus(win32service.SERVICE_STOPPED)
logging.info('Stopped2 {name} service ...'.format(name=self._svc_name_))
sys.exit()
def SvcDoRun(self):
logging.info('Starting {name} service ...'.format(name=self._svc_name_))
os.chdir(INSTDIR) # so that proj worker can be found
logging.info('cwd: ' + os.getcwd())
self.ReportServiceStatus(win32service.SERVICE_RUNNING)
command = '"{celery_path}" -A {proj_dir} -c 8 worker --workdir=X:/Application/Project --pidfile=celeryservice.pid -f "{log_path}" -l info'.format(
celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
proj_dir=PROJECTDIR,
log_path=os.path.join(LOGDIR,'celery_worker.log'))
logging.info('command: ' + command)
args = shlex.split(command)
proc = subprocess.Popen(args)
logging.info('pid: {pid}'.format(pid=proc.pid))
self.timeout = 3000
while True:
rc = win32event.WaitForSingleObject(self.hWaitStop, self.timeout)
if rc == win32event.WAIT_OBJECT_0:
# stop signal encountered
# terminate process 'proc'
PROCESS_TERMINATE = 1
handle = win32api.OpenProcess(PROCESS_TERMINATE, False, proc.pid)
win32api.TerminateProcess(handle, -1)
win32api.CloseHandle(handle)
break
if __name__ == '__main__':
win32serviceutil.HandleCommandLine(CeleryService)
Beatservice.py:
'''Usage : python celery_service.py install (start / stop / remove)
Run celery as a Windows service
'''
import win32service
import win32serviceutil
import win32api
import win32con
import win32event
import subprocess
import sys
import os
import shlex
import logging
import time
import signal
# The directory for celery_beat.log and celery_beat_service.log
# Default: the directory of this script
INSTDIR = os.path.dirname(os.path.realpath(__file__))
LOGPATH = 'X:\Application\Logs'
# The path of python Scripts
# Usually it is in PYTHON_INSTALL_DIR/Scripts. e.g.
# r'C:\Python27\Scripts'
# If it is already in system PATH, then it can be set as ''
PYTHONSCRIPTPATH = 'C:\Python36\Scripts'
# The directory name of django project
# Note: it is the directory at the same level of manage.py
# not the parent directory
PROJECTDIR = 'PROJECT'
logging.basicConfig(
filename = os.path.join(LOGPATH, 'celery_beat_service.log'),
level = logging.DEBUG,
format = '[%(asctime)-15s: %(levelname)-7.7s] %(message)s'
)
class CeleryService(win32serviceutil.ServiceFramework):
_svc_name_ = "CeleryBeat"
_svc_display_name_ = "CeleryBeat"
def __init__(self, args):
win32serviceutil.ServiceFramework.__init__(self, args)
self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
def SvcStop(self):
logging.info('Stopping 1 {name} service ...'.format(name=self._svc_name_))
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
pidno = open("X:\Aplication\Project\celerybeat.pid", "r")
_pid_id_ = pidid=pidno.read()
pidno.close()
logging.info(_pid_id_)
logging.info('taskkill /F /PID {pidid} ..'.format(pidid=_pid_id_))
cmdcom = 'taskkill /F /PID {pidid}'.format(pidid=_pid_id_)
logging.info(cmdcom)
killargs = shlex.split(cmdcom)
process = subprocess.Popen(killargs)
output, error = process.communicate()
logging.info(output)
logging.info('Stopping 2 {name} service ...'.format(name=self._svc_name_))
os.remove("X:\Application\PROJECT\celerybeat.pid")
logging.info('X:\Application\PROJECT\celerybeat.pid file removed')
self.ReportServiceStatus(win32service.SERVICE_STOPPED)
sys.exit()
def SvcDoRun(self):
logging.info('Starting {name} service ...'.format(name=self._svc_name_))
os.chdir(INSTDIR) # so that proj worker can be found
logging.info('cwd: ' + os.getcwd())
self.ReportServiceStatus(win32service.SERVICE_RUNNING)
command = '"{celery_path}" -A {proj_dir} beat --workdir=X:/Application/Project -f X:/Application/logs/beat.log -l info'.format(
celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
proj_dir=PROJECTDIR,
log_path=os.path.join(LOGPATH,'celery_beat.log'))
logging.info('command: ' + command)
args = shlex.split(command)
proc = subprocess.Popen(args)
logging.info('pid: {pid}'.format(pid=proc.pid))
self.timeout = 3000
while True:
rc = win32event.WaitForSingleObject(self.hWaitStop, self.timeout)
if rc == win32event.WAIT_OBJECT_0:
# stop signal encountered
# terminate process 'proc'
PROCESS_TERMINATE = 1
handle = win32api.OpenProcess(PROCESS_TERMINATE, False, proc.pid)
win32api.TerminateProcess(handle, -1)
win32api.CloseHandle(handle)
break
if __name__ == '__main__':
win32serviceutil.HandleCommandLine(CeleryService)