I am trying to understand a script found on the Python documentation page.
I do not understand exactly, how to introduce the arguments, I know that I must put the name of the directory where the virtual environment will be installed so that the argument dirs stores the tuple of directories, however I do not understand how to modify the parameters of the other arguments such as nopip.
Code:
import os
import os.path
from subprocess import Popen, PIPE
import sys
from threading import Thread
from urllib.parse import urlparse
from urllib.request import urlretrieve
import venv
class ExtendedEnvBuilder(venv.EnvBuilder):
"""
This builder installs setuptools and pip so that you can pip or
easy_install other packages into the created virtual environment.
:param nodist: If true, setuptools and pip are not installed into the
created virtual environment.
:param nopip: If true, pip is not installed into the created
virtual environment.
:param progress: If setuptools or pip are installed, the progress of the
installation can be monitored by passing a progress
callable. If specified, it is called with two
arguments: a string indicating some progress, and a
context indicating where the string is coming from.
The context argument can have one of three values:
'main', indicating that it is called from virtualize()
itself, and 'stdout' and 'stderr', which are obtained
by reading lines from the output streams of a subprocess
which is used to install the app.
If a callable is not specified, default progress
information is output to sys.stderr.
"""
def __init__(self, *args, **kwargs):
self.nodist = kwargs.pop('nodist', False)
self.nopip = kwargs.pop('nopip', False)
self.progress = kwargs.pop('progress', None)
self.verbose = kwargs.pop('verbose', False)
super().__init__(*args, **kwargs)
def post_setup(self, context):
"""
Set up any packages which need to be pre-installed into the
virtual environment being created.
:param context: The information for the virtual environment
creation request being processed.
"""
os.environ['VIRTUAL_ENV'] = context.env_dir
if not self.nodist:
self.install_setuptools(context)
# Can't install pip without setuptools
if not self.nopip and not self.nodist:
self.install_pip(context)
def reader(self, stream, context):
"""
Read lines from a subprocess' output stream and either pass to a progress
callable (if specified) or write progress information to sys.stderr.
"""
progress = self.progress
while True:
s = stream.readline()
if not s:
break
if progress is not None:
progress(s, context)
else:
if not self.verbose:
sys.stderr.write('.')
else:
sys.stderr.write(s.decode('utf-8'))
sys.stderr.flush()
stream.close()
def install_script(self, context, name, url):
_, _, path, _, _, _ = urlparse(url)
fn = os.path.split(path)[-1]
binpath = context.bin_path
distpath = os.path.join(binpath, fn)
# Download script into the virtual environment's binaries folder
urlretrieve(url, distpath)
progress = self.progress
if self.verbose:
term = '\n'
else:
term = ''
if progress is not None:
progress('Installing %s ...%s' % (name, term), 'main')
else:
sys.stderr.write('Installing %s ...%s' % (name, term))
sys.stderr.flush()
# Install in the virtual environment
args = [context.env_exe, fn]
p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath)
t1 = Thread(target=self.reader, args=(p.stdout, 'stdout'))
t1.start()
t2 = Thread(target=self.reader, args=(p.stderr, 'stderr'))
t2.start()
p.wait()
t1.join()
t2.join()
if progress is not None:
progress('done.', 'main')
else:
sys.stderr.write('done.\n')
# Clean up - no longer needed
os.unlink(distpath)
def install_setuptools(self, context):
"""
Install setuptools in the virtual environment.
:param context: The information for the virtual environment
creation request being processed.
"""
url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py'
self.install_script(context, 'setuptools', url)
# clear up the setuptools archive which gets downloaded
pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz')
files = filter(pred, os.listdir(context.bin_path))
for f in files:
f = os.path.join(context.bin_path, f)
os.unlink(f)
def install_pip(self, context):
"""
Install pip in the virtual environment.
:param context: The information for the virtual environment
creation request being processed.
"""
url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py'
self.install_script(context, 'pip', url)
def main(args=None):
compatible = True
if sys.version_info < (3, 3):
compatible = False
elif not hasattr(sys, 'base_prefix'):
compatible = False
if not compatible:
raise ValueError('This script is only for use with '
'Python 3.3 or later')
else:
import argparse
parser = argparse.ArgumentParser(prog=__name__,
description='Creates virtual Python '
'environments in one or '
'more target '
'directories.')
parser.add_argument('dirs', metavar='ENV_DIR', nargs='+ Pruebas',
help='A directory in which to create the'
'virtual environment.')
parser.add_argument('--no-setuptools', default=False,
action='store_true', dest='nodist',
help="Don't install setuptools or pip in the "
"virtual environment.")
parser.add_argument('--no-pip', default=False,
action='store_true', dest='nopip',
help="Don't install pip in the virtual "
"environment.")
parser.add_argument('--system-site-packages', default=False,
action='store_true', dest='system_site',
help='Give the virtual environment access to the '
'system site-packages dir.')
if os.name == 'nt':
use_symlinks = False
else:
use_symlinks = True
parser.add_argument('--symlinks', default=use_symlinks,
action='store_true', dest='symlinks',
help='Try to use symlinks rather than copies, '
'when symlinks are not the default for '
'the platform.')
parser.add_argument('--clear', default=False, action='store_true',
dest='clear', help='Delete the contents of the '
'virtual environment '
'directory if it already '
'exists, before virtual '
'environment creation.')
parser.add_argument('--upgrade', default=False, action='store_true',
dest='upgrade', help='Upgrade the virtual '
'environment directory to '
'use this version of '
'Python, assuming Python '
'has been upgraded '
'in-place.')
parser.add_argument('--verbose', default=False, action='store_true',
dest='verbose', help='Display the output '
'from the scripts which '
'install setuptools and pip.')
options = parser.parse_args(args)
if options.upgrade and options.clear:
raise ValueError('you cannot supply --upgrade and --clear together.')
builder = ExtendedEnvBuilder(system_site_packages=options.system_site,
clear=options.clear,
symlinks=options.symlinks,
upgrade=options.upgrade,
nodist=options.nodist,
nopip=options.nopip,
verbose=options.verbose)
for d in options.dirs:
builder.create(d)
if __name__ == '__main__':
rc = 1
try:
main()
rc = 0
except Exception as e:
print('Error: %s' % e, file=sys.stderr)
sys.exit(rc)
All the other options could be modified through command line arguments. You could see available options if run script with --help flag:
python3 pyvenvex.py --help
Output:
Usage: __main__ [-h] [--no-setuptools] [--no-pip] [--system-site-packages]
[--symlinks] [--clear] [--upgrade] [--verbose]
ENV_DIR [ENV_DIR ...]
Creates virtual Python environments in one or more target directories.
positional arguments:
ENV_DIR A directory to create the environment in.
optional arguments:
-h, --help show this help message and exit
--no-setuptools Don't install setuptools or pip in the virtual
environment.
--no-pip Don't install pip in the virtual environment.
--system-site-packages
Give the virtual environment access to the system
site-packages dir.
--symlinks Try to use symlinks rather than copies, when symlinks
are not the default for the platform.
--clear Delete the contents of the environment directory if it
already exists, before environment creation.
--upgrade Upgrade the environment directory to use this version
of Python, assuming Python has been upgraded in-place.
--verbose Display the output from the scripts which install
setuptools and pip.
This arguments handled with argparse module from standard library. You could see a lot of boilerplate code like this in the script:
parser.add_argument('--no-pip', default=False,
action='store_true', dest='nopip',
help="Don't install pip in the virtual "
"environment.")
The only tricky thing here is the keyword argument dest='nopip', which means "parse --no-pip argument and put result into nopip attribute".
Related
Without getting confused, there are tons of questions about installing packages, how to import the resulting modules, and listing what packages are available. But there doesn't seem to be the equivalent of a "--what-provides" option for pip, if you don't have a requirements.txt or pipenv. This question is similar to a previous question, but asks for the parent package, and not additional metadata. That said, these other questions did not get a lot of attention or many accepted answers - eg. How do you find python package metadata information given a module. So forging ahead... .
By way of example, there are two packages (to name a few) that will install a module called "serial" - namely "pyserial" and "serial". So assuming that one of the packages was installed, we might find it by using pip list:
python3 -m pip list | grep serial
However, the problem comes in if the name of the package does not match the name of the module, or if you just want to find out what package to install, working on a legacy server or dev machine.
You can check the path of the imported module - which can give you a clue. But continuing the example...
>>> import serial
>>> print(serial.__file__)
/usr/lib/python3.6/site-packages/serial/__init__.py
It is in a "serial" directory, but only pyserial is in fact installed, not serial:
> python3 -m pip list | grep serial
pyserial 3.4
The closest I can come is to generate a requirements.txt via "pipreqs ./" which may fail on a dependent child file (as it does with me), or to reverse check dependencies via pipenv (which brings a whole set of new issues along to get it all setup):
> pipenv graph --reverse
cymysql==0.9.15
ftptool==0.7.1
netifaces==0.10.9
pip==20.2.2
PyQt5-sip==12.8.1
- PyQt5==5.15.0 [requires: PyQt5-sip>=12.8,<13]
setuptools==50.3.0
wheel==0.35.1
Does anyone know of a command that I have missed for a simple solution to finding what pip package provides a particular module?
Use the packages_distributions() function from importlib.metadata (or importlib-metadata). So for example, in your case where serial is the name of the "import package":
import importlib.metadata # or: `import importlib_metadata`
importlib.metadata.packages_distributions()['serial']
This should return a list containing pyserial, which is the name of the "distribution package" (the name that should be used to pip-install).
References
https://importlib-metadata.readthedocs.io/en/stable/using.html#package-distributions
https://github.com/python/importlib_metadata/pull/287/files
For older Python versions and/or older versions of importlib-metadata...
I believe something like the following should work:
#!/usr/bin/env python3
import importlib.util
import pathlib
import importlib_metadata
def get_distribution(file_name):
result = None
for distribution in importlib_metadata.distributions():
try:
relative = (
pathlib.Path(file_name)
.relative_to(distribution.locate_file(''))
)
except ValueError:
pass
else:
if distribution.files and relative in distribution.files:
result = distribution
break
return result
def alpha():
file_name = importlib.util.find_spec('serial').origin
distribution = get_distribution(file_name)
print("alpha", distribution.metadata['Name'])
def bravo():
import serial
file_name = serial.__file__
distribution = get_distribution(file_name)
print("bravo", distribution.metadata['Name'])
if __name__ == '__main__':
alpha()
bravo()
This is just an example of code showing how to get the metadata of the installed project a specific module belongs to.
The important bit is the get_distribution function, it takes a file name as an argument. It could be the file name of a module or package data. If that file name belongs to a project installed in the environment (via pip install for example) then the importlib.metadata.Distribution object is returned.
Edit 2023/01/31: This issue is now solved via the importlib_metadata library. See Provide mapping from "Python packages" to "distribution packages", specifically "Note 2" deals with this exact issue. As such, see comments by #sinoroc, you can locate the package (eg. package "pyserial" providing module "serial") with something like this:
>>> import importlib_metadata
>>> print(importlib_metadata.packages_distributions()['serial'])
['pyserial']
Building on #sinoroc's much-published answer, I came up with the following code (incorporating the mentioned importlib.util.find_spec method, but with a bash-based search against the RECORD file in the path returned). I also tried to implement #sinoroc's version - but was not successful. Both methods are included to demonstrate.
Run as "python3 python_find-module-package.py -m [module-name-here] -d", which will also print debug. Leave off the "-d" switch to get just the package name returned (and errors).
TLDR: Code on github.
#!/usr/bin/python3
import sys
import os.path
import importlib.util
import importlib_metadata
import pathlib
import subprocess
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--module", help="Find matching package for the specified Python module",
type=str)
#parser.add_argument("-u", "--username", help="Database username",
# type=str)
#parser.add_argument("-p", "--password", help="Database password",
# type=str)
parser.add_argument("-d", "--debug", help="Debug messages are enabled",
action="store_true")
args = parser.parse_args()
TESTMODULE='serial'
def debugPrint (message="Nothing"):
if args.debug:
print ("[DEBUG] %s" % str(message))
class application ():
def __init__(self, argsPassed):
self.argsPassed = argsPassed
debugPrint("Got these arguments:\n%s" % (argsPassed))
def run (self):
#debugPrint("Running with args:\n%s" % (self.argsPassed))
try:
if self.argsPassed.module is not None:
self.moduleName=self.argsPassed.module #i.e. the module that you're trying to match with a package.
else:
self.moduleName=TESTMODULE
print("[WARN] No module name supplied - defaulting to %s!" % (TESTMODULE))
self.location=importlib.util.find_spec(self.moduleName).origin
debugPrint(self.location)
except:
print("[ERROR] Parsing module name!")
exit(1)
try:
self.getPackage()
except Exception as e:
print ("[ERROR] getPackage failed: %s" % str(e))
try:
distResult=self.getDistribution(self.location)
self.packageStrDist=distResult.metadata['Name']
print(self.packageStrDist)
except Exception as e:
print ("[ERROR] getDistribution failed: %s" % str(e))
debugPrint("Parent package for \"%s\" is: \"%s\"" % (self.moduleName, self.packageStr))
return self.packageStr
def getPackage (self):
locationStr=self.location.split("site-packages/",1)[1]
debugPrint(locationStr)
#serial/__init__.py
locationDir=self.location.split(locationStr,1)[0]
debugPrint(locationDir)
#/usr/lib/python3.6/site-packages
cmd='find \"' + locationDir + '\" -type f -iname \'RECORD\' -printf \'\"%p\"\\n\' | xargs grep \"' + locationStr + '\" -l -Z'
debugPrint(cmd)
#find "/usr/lib/python3.6/site-packages" -type f -iname 'RECORD' -printf '"%p"\n' | xargs grep "serial/__init__.py" -l -Z
#return_code = os.system(cmd)
#return_code = subprocess.run([cmd], stdout=subprocess.PIPE, universal_newlines=True, shell=False)
#findResultAll = return_code.stdout
findResultAll = subprocess.check_output(cmd, shell=True) # Returns stdout as byte array, null terminated.
findResult = str(findResultAll.decode('ascii').strip().strip('\x00'))
debugPrint(findResult)
#/usr/lib/python3.6/site-packages/pyserial-3.4.dist-info/RECORD
findDir = os.path.split(findResult)
self.packageStr=findDir[0].replace(locationDir,"")
debugPrint(self.packageStr)
def getDistribution(self, fileName=TESTMODULE):
result = None
for distribution in importlib_metadata.distributions():
try:
relative = (pathlib.Path(fileName).relative_to(distribution.locate_file('')))
#except ValueError:
#except AttributeError:
except:
pass
else:
if relative in distribution.files:
result = distribution
return result
if __name__ == '__main__':
result=1
try:
prog = application(args)
result = prog.run()
except Exception as E:
print ("[ERROR] Prog Exception: %s" % str(E))
finally:
sys.exit(result)
# exit the program if we haven't already
print ("Shouldn't get here.")
sys.exit(result)
I am a complete Python beginner and try to write a Python script to automate the setup of a SDK on Linux machines from remote Github repositories.
The script starts by performing some basic preliminary operations, especially the check/setup of several packages (git, docker, pip, etc.).
For now, I target Debian (Stretch, Buster), Centos (6, 7) and Ubuntu Server 18.04LTS.
Of course, I want the script to run on the widest range of linux machines.
Today I rely on available package managers (apt-get and yum), roughly requested through subprocess.call() statements.
I customize the related commands using nasty script configuration variables like below :
import platform
distribution = platform.dist()[0]
version = platform.dist()[1]
if distribution == 'debian':
pkgInstaller = 'dpkg'
pkmManager = 'apt-get'
checkIfInstalled = '-s'
installPackage = 'install'
yesToAll = '-y'
dockerPackage = 'docker-ce'
elif distribution == 'centos':
pkgInstaller = 'rpm'
pkgManager = 'yum'
checkIfInstalled = '-q'
installPackage = 'install'
yesToAll = '-y'
dockerPackage = 'docker'
I then simply loop on an array containing the names of packages to be installed, then run the command through subprocess.call() :
prerequisites = ['git', dockerPackage, 'doxygen', 'python2-pip']
for pkg in prerequisites:
pgkInstallation = subprocess.call(['sudo', pkgManager, yesToAll, installPackage, pkg])
While this approach may have the benefit of not having too much bonding to third-party Python modules, I guess there are... some smarter ways of doing such simple operation ?
Usually when doing switch statements like this, a dictionary might be a bit more useful. Also, normally I'm not one to try to PEP-8 things, but this is an instance where PEP-8 might really help your readability by not matching up your equals signs for all of your lines of code.
The dict will hold your distro as the key, and your vars as a value wrapped in a tuple
options = {
'debian': ('dpkg', 'apt-get', '-s', 'install', '-y', 'docker-ce'),
'centos': ('rpm', 'yum', '-q', 'install', '-y', 'docker'),
}
# unpack this function call here
distribution, version, *_ = platform.dist()
# now get the match
pkg_installer, pkg_manager, check, install_pkg, yes_to_all, docker = options[distribution]
requisites = ['git', docker, 'doxygen', 'python2-pip']
for pkg in requisites:
pgkInstallation = subprocess.call(['sudo', pkg_manager, yes_to_all, install_pkg, pkg])
The options[distribution] call will raise a KeyError for unsupported distributions, so you can probably catch that and raise something a bit more useful like:
try:
pkg_installer, pkg_manager, check, install_pkg, yes_to_all, docker = options[distribution]
except KeyError as e:
raise ValueError(f"Got unsupported OS, expected one of {', '.join(options.keys())}") from e
To make it less verbose, the only var you use out of order is docker, so you can house all of the others in a single var:
try:
*args, docker = options[distribution]
except KeyError as e:
raise ValueError(f"Got unsupported OS, expected one of {', '.join(options.keys())}") from e
requisites = ['git', docker, 'doxygen', 'python2-pip']
for pkg in requisites:
pgkInstallation = subprocess.call(['sudo', *args, pkg])
I am using sonarqube/soanrpython on windows to analyse my python code and would like to be able to initiate the scan using setuptools instead of calling the scanner from the DOS prompt. Is this possible?. I have searched the web and cannot find anything.
I call the scanner using the following command
C:> sonar-scanner -Dsonar.projectKey=TL:python -Dsonar.sources=mypackage
But would like to be able to call
C:> python setup.py sonar
Or something similar
Edit:
To get this to work I put the following code in my setup.py file
A class:
class SonarPython(Command):
""" Run sonar-scanner via setuptools.
"""
description = 'running sonar-scanner for project '+name
user_options = [
('project-key=', 'k', 'project key (eg TL:python)'),
('source-dir=', 's', 'source dir location'),
]
def initialize_options(self):
self.project_key = None
self.source_dir = None
def finalize_options(self):
print("Sonar project_key is", self.project_key)
if self.project_key is None:
raise Exception("Parameter --project-key is missing (e.g. TL:python)")
print("Sonar using source_dir ", self.source_dir)
if self.source_dir is None:
raise Exception("Parameter --source-dir is missing (relative to setup.py)")
def run(self):
"""Run command.
"""
command = ['cmd', '/c', ]
command.append('sonar-scanner'+' -Dsonar.projectKey='+self.project_key+' -Dsonar.sources='+self.source_dir)
self.announce('Running command: %s' % str(command),level=distutils.log.INFO)
subprocess.check_call(command)
and and alteration to cmdclass and command_options
cmdclass={'sonar' : SonarPython},
command_options={
'sonar': {
'project_key': ('setup.py', 'TL:python'),
'source_dir': ('setup.py', 'mypackage')
}
},
You can then call sonar with the command
python setup.py sonar
You could leave out the command_options entry and just pass them on the command line as
python setup.py sonar --project_key 'TL:python' --source_dir 'myproject'
You can create a new distutils/setuptools command:
from distutils.core import setup, Command
import os
class SonarCommand(Command):
description = "Run sonarqube's sonar"
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self):
os.system('sonar-scanner -Dsonar.projectKey=TL:python -Dsonar.sources=mypackage')
To enable the command you must reference it in setup():
setup(
# stuff omitted for conciseness
cmdclass={
'sonar': SonarCommand
}
See the docs and examples (1, 2).
Unfortunately, this is not available.
Tensorboard should be started from commnad line like that:
tensorboard --logdir=path
I need to run it from code. Until now I use this:
import os
os.system('tensorboard --logdir=' + path)
However tensorboard do not start because is not included in the system path. I use PyCharm with virtualenv on windows. I don't want to change system paths so the only option is to run it from virtualenv. How to do this?
Using Tensorboard 2 API (2019):
from tensorboard import program
tracking_address = log_path # the path of your log file.
if __name__ == "__main__":
tb = program.TensorBoard()
tb.configure(argv=[None, '--logdir', tracking_address])
url = tb.launch()
print(f"Tensorflow listening on {url}")
Note: tb.launch() create a daemon thread that will die automatically when your process is finished
Probably a bit late for an answer, but this is what worked for me in Python 3.6.2:
import tensorflow as tf
from tensorboard import main as tb
tf.flags.FLAGS.logdir = "/path/to/graphs/"
tb.main()
That runs tensorboard with the default configuration and looks for graphs and summaries in "/path/to/graphs/". You can of course change the log directory and set as many variables as you like using:
tf.flags.FLAGS.variable = value
Hope it helps.
You should launch tensorBoard in the separate thread:
def launchTensorBoard():
import os
os.system('tensorboard --logdir=' + tensorBoardPath)
return
import threading
t = threading.Thread(target=launchTensorBoard, args=([]))
t.start()
As I get the same problem, you can use this lines inspired by tensorboard\main.py:
from tensorboard import default
from tensorboard import program
tb = program.TensorBoard(default.PLUGIN_LOADERS, default.get_assets_zip_provider())
tb.configure(argv=['--logdir', my_directory])
tb.main()
With my_directory as the folder you want to check. Don't forget to create a separate Thread if you want to avoid to be block after tb.main().
Best regards
EDIT Tensorboard V1.10:
For some personnal reasons, I write it in a different way:
class TensorBoardTool:
def __init__(self, dir_path):
self.dir_path = dir_path
def run(self):
# Remove http messages
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
# Start tensorboard server
tb = program.TensorBoard(default.PLUGIN_LOADERS, default.get_assets_zip_provider())
tb.configure(argv=['--logdir', self.dir_path])
url = tb.launch()
sys.stdout.write('TensorBoard at %s \n' % url)
EDIT Tensorboard V1.12:
According to Elad Weiss and tsbertalan for the version 1.12 of tensorboard.
def run(self):
# Remove http messages
log = logging.getLogger('werkzeug').setLevel(logging.ERROR)
# Start tensorboard server
tb = program.TensorBoard(default.get_plugins(), default.get_assets_zip_provider())
tb.configure(argv=[None, '--logdir', self.dir_path])
url = tb.launch()
sys.stdout.write('TensorBoard at %s \n' % url)
Then to run it just do:
# Tensorboard tool launch
tb_tool = TensorBoardTool(work_dir)
tb_tool.run()
This will allow you to run a Tensorboard server at same time as your main process, without disturbing http request!
For Tensorboard 2.1.0, this works for me:
python -m tensorboard.main --logdir $PWD/logs
You must have your env active first. (In my case, conda install had a fatal error, so I needed to reinstall tf via pip inside conda.)
A full solution for Tensorboard 2 (2019), with automatic opening of the Chrome browser, for Windows and Linux. Works for both environments: conda and virtualenv. This solution suppresses the Tensorboard output so it doesn't (irritatingly) show in stdout
from multiprocessing import Process
import sys
import os
class TensorboardSupervisor:
def __init__(self, log_dp):
self.server = TensorboardServer(log_dp)
self.server.start()
print("Started Tensorboard Server")
self.chrome = ChromeProcess()
print("Started Chrome Browser")
self.chrome.start()
def finalize(self):
if self.server.is_alive():
print('Killing Tensorboard Server')
self.server.terminate()
self.server.join()
# As a preference, we leave chrome open - but this may be amended similar to the method above
class TensorboardServer(Process):
def __init__(self, log_dp):
super().__init__()
self.os_name = os.name
self.log_dp = str(log_dp)
# self.daemon = True
def run(self):
if self.os_name == 'nt': # Windows
os.system(f'{sys.executable} -m tensorboard.main --logdir "{self.log_dp}" 2> NUL')
elif self.os_name == 'posix': # Linux
os.system(f'{sys.executable} -m tensorboard.main --logdir "{self.log_dp}" '
f'--host `hostname -I` >/dev/null 2>&1')
else:
raise NotImplementedError(f'No support for OS : {self.os_name}')
class ChromeProcess(Process):
def __init__(self):
super().__init__()
self.os_name = os.name
self.daemon = True
def run(self):
if self.os_name == 'nt': # Windows
os.system(f'start chrome http://localhost:6006/')
elif self.os_name == 'posix': # Linux
os.system(f'google-chrome http://localhost:6006/')
else:
raise NotImplementedError(f'No support for OS : {self.os_name}')
Initialization:
tb_sup = TensorboardSupervisor('path/to/logs')
After finishing the training/testing:
tb_sup.finalize()
If your python interpreter path is：
/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/bin/python3.6
You can run this command instead of tensorboard
/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/bin/python3.6 /usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorboard/main.py
To run tensorboard from a python script within a specified virtual environment you have to change tensorboard to /path/to/your/environment/bin/tensorboard. It is also recommended to execute the command in a separate thread as suggested by #Dmitry.
Together it looks like this and works for me with tb and tf version 1.14.0:
def run_tensorboard(logdir_absolute):
import os, threading
tb_thread = threading.Thread(
target=lambda: os.system('/home/username/anaconda3/envs/'
'env_name/bin/tensorboard '
'--logdir=' + logdir_absolute),
daemon=True)
tb_thread.start()
As of TensorBoard version 1.9.0, the following works to start TensorBoard with default settings in the same Python process:
import tensorboard as tb
import tensorboard.program
import tensorboard.default
tb.program.FLAGS.logdir = 'path/to/logdir'
tb.program.main(tb.default.get_plugins(),
tb.default.get_assets_zip_provider())
The following will open a Chrome tab and launches TensorBoard. Simply provide the desired directory and your system's name .
import os
os.system(
"cd <directory> \
&& google-chrome http://<your computer name>:6007 \
&& tensorboard --port=6007 --logdir runs"
)
Had the same problem:
As you're working on Windows, you can use batch files to fully-automate opening tensorboard like in the exaple below.
As you probably want to open tensorboard within a visible console window (cmd.exe). Calling one batch-file within your IDE (pycharm) will run it within the IDE, so in the background, which means you can't see the console. Therefore, you can use a workaround: call a batch-file that then calls the batch-file to start tensorboard.
Note: I'm using Anaconda as my virtual-environment for this example
batch_filename = 'start_tb.bat' # set filename for batch file
tb_command = 'tensorboard --logdir=' + log_dir # join strings for tensorflow command
# creates batch file that will call seconds batch file in console window (cmd.exe)
with open(os.path.join('invoke.bat'), "w") as f:
f.writelines('start ' + batch_filename)
# created batch file that activates Anaconda environment and starts tensorboard
with open(os.path.join(batch_filename), "w") as f:
f.writelines('\nconda activate YOURCondaEnvNAME && ' + tb_command) # change to your conda environment, or other virtualenv
# starts tensorboard using the batch files (will open console window)
# calls the 'invoke.bat' that will call 'start_tb.bat'
os.system('invoke.bat')
# starts tensorboard in default browser >> ATTENTION: must be adapted to local host
os.system('start "" http://YOUR-COMPUTER-NAME:6006/') # just copy the URL that tensorboard runs at on your computer
Sometimes you might have to refresh tensorboard within your browser, as it opened already before it was properly set-up.
Try running from python
import os
os.system('python -m tensorflow.tensorboard --logdir=' + path)
works for me in PyCharm (but on linux, so if the shell syntax is different then you have to tweak it)
I want to create a start menu or Desktop shortcut for my Python windows installer package. I am trying to follow https://docs.python.org/3.4/distutils/builtdist.html#the-postinstallation-script
Here is my script;
import sys
from os.path import dirname, join, expanduser
pyw_executable = sys.executable.replace('python.exe','pythonw.exe')
script_file = join(dirname(pyw_executable), 'Scripts', 'tklsystem-script.py')
w_dir = expanduser(join('~','lsf_files'))
print(sys.argv)
if sys.argv[1] == '-install':
print('Creating Shortcut')
create_shortcut(
target=pyw_executable,
description='A program to work with L-System Equations',
filename='L-System Tool',
arguments=script_file,
workdir=wdir
)
I also specified this script in scripts setup option, as indicated by aforementioned docs.
Here is the command I use to create my installer;
python setup.py bdist_wininst --install-script tklsystem-post-install.py
After I install my package using created windows installer, I can't find where my shorcut is created, nor I can confirm whether my script run or not?
How can I make setuptools generated windows installer to create desktop or start menu shortcuts?
Like others have commented here and elsewhere the support functions don't seem to work at all (at least not with setuptools). After a good day's worth of searching through various resources I found a way to create at least the Desktop shortcut. I'm sharing my solution (basically an amalgam of code I found here and here). I should add that my case is slightly different from yasar's, because it creates a shortcut to an installed package (i.e. an .exe file in Python's Scripts directory) instead of a script.
In short, I added a post_install function to my setup.py, and then used the Python extensions for Windows to create the shortcut. The location of the Desktop folder is read from the Windows registry (there are other methods for this, but they can be unreliable if the Desktop is at a non-standard location).
#!/usr/bin/env python
import os
import sys
import sysconfig
if sys.platform == 'win32':
from win32com.client import Dispatch
import winreg
def get_reg(name,path):
# Read variable from Windows Registry
# From https://stackoverflow.com/a/35286642
try:
registry_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, path, 0,
winreg.KEY_READ)
value, regtype = winreg.QueryValueEx(registry_key, name)
winreg.CloseKey(registry_key)
return value
except WindowsError:
return None
def post_install():
# Creates a Desktop shortcut to the installed software
# Package name
packageName = 'mypackage'
# Scripts directory (location of launcher script)
scriptsDir = sysconfig.get_path('scripts')
# Target of shortcut
target = os.path.join(scriptsDir, packageName + '.exe')
# Name of link file
linkName = packageName + '.lnk'
# Read location of Windows desktop folder from registry
regName = 'Desktop'
regPath = r'Software\Microsoft\Windows\CurrentVersion\Explorer\User Shell Folders'
desktopFolder = os.path.normpath(get_reg(regName,regPath))
# Path to location of link file
pathLink = os.path.join(desktopFolder, linkName)
shell = Dispatch('WScript.Shell')
shortcut = shell.CreateShortCut(pathLink)
shortcut.Targetpath = target
shortcut.WorkingDirectory = scriptsDir
shortcut.IconLocation = target
shortcut.save()
setup(name='mypackage',
...,
...)
if sys.argv[1] == 'install' and sys.platform == 'win32':
post_install()
Here's a link to a full setup script in which I used this:
https://github.com/KBNLresearch/iromlab/blob/master/setup.py
If you want to confirm whether the script is running or not, you can print to a file instead of the console. Looks like text you print to console in the post-install script won't show up.
Try this:
import sys
from os.path import expanduser, join
pyw_executable = join(sys.prefix, "pythonw.exe")
shortcut_filename = "L-System Toolsss.lnk"
working_dir = expanduser(join('~','lsf_files'))
script_path = join(sys.prefix, "Scripts", "tklsystem-script.py")
if sys.argv[1] == '-install':
# Log output to a file (for test)
f = open(r"C:\test.txt",'w')
print('Creating Shortcut', file=f)
# Get paths to the desktop and start menu
desktop_path = get_special_folder_path("CSIDL_COMMON_DESKTOPDIRECTORY")
startmenu_path = get_special_folder_path("CSIDL_COMMON_STARTMENU")
# Create shortcuts.
for path in [desktop_path, startmenu_path]:
create_shortcut(pyw_executable,
"A program to work with L-System Equations",
join(path, shortcut_filename),
script_path,
working_dir)
At least with Python 3.6.5, 32bit on Windows, setuptools does work for this. But based on the accepted answer, by trial and error I found some issues that may have caused your script to fail to do what you wanted.
create_shortcut does not accept keyword arguments, only positional, so its usage in your code is invalid
You must add a .lnk extension for Windows to recognise the shortcut
I found sys.executable will be the name of the installer executable, not the python executable
As mentioned, you can't see stdout or stderr so you might want to log to a text file. I would suggest also redirecting sys.stdout and sys.stderr to the log file.
(Maybe not relevant) as mentioned in this question there appears to be a bug with the version string generated by bdist_wininst. I used the hexediting hack from an answer there to work around this. The location in the answer is not the same, you have to find the -32 yourself.
Full example script:
import sys
import os
import datetime
global datadir
datadir = os.path.join(get_special_folder_path("CSIDL_APPDATA"), "mymodule")
def main(argv):
if "-install" in argv:
desktop = get_special_folder_path("CSIDL_DESKTOPDIRECTORY")
print("Desktop path: %s" % repr(desktop))
if not os.path.exists(datadir):
os.makedirs(datadir)
dir_created(datadir)
print("Created data directory: %s" % repr(datadir))
else:
print("Data directory already existed at %s" % repr(datadir))
shortcut = os.path.join(desktop, "MyModule.lnk")
if os.path.exists(shortcut):
print("Remove existing shortcut at %s" % repr(shortcut))
os.unlink(shortcut)
print("Creating shortcut at %s...\n" % shortcut)
create_shortcut(
r'C:\Python36\python.exe',
"MyModuleScript",
shortcut,
"",
datadir)
file_created(shortcut)
print("Successfull!")
elif "-remove" in sys.argv:
print("Removing...")
pass
if __name__ == "__main__":
logfile = r'C:\mymodule_install.log' # Fallback location
if os.path.exists(datadir):
logfile = os.path.join(datadir, "install.log")
elif os.environ.get("TEMP") and os.path.exists(os.environ.get("TEMP"),""):
logfile = os.path.join(os.environ.get("TEMP"), "mymodule_install.log")
with open(logfile, 'a+') as f:
f.write("Opened\r\n")
f.write("Ran %s %s at %s" % (sys.executable, " ".join(sys.argv), datetime.datetime.now().isoformat()))
sys.stdout = f
sys.stderr = f
try:
main(sys.argv)
except Exception as e:
raise
f.close()
sys.exit(0)
UPD: on an off chance that the client machine has pywin32 installed, we try in-process creation first. Somewhat cleaner that way.
Here is another take. This assumes the package is called myapp, and that also becomes the executable that you want a shortcut to. Substitute your own package name and your own shortcut text.
Uses a Windows Scripting Host COM class - in process if possible, inside a Powershell command line as a subprocess if not. Tested on Python 3.6+.
from setuptools import setup
from setuptools.command.install import install
import platform, sys, os, site
from os import path, environ
def create_shortcut_under(root, exepath):
# Root is an env variable name -
# either ALLUSERSPROFILE for the all users' Start menu,
# or APPDATA for the current user specific one
profile = environ[root]
linkpath = path.join(profile, "Microsoft", "Windows", "Start Menu", "Programs", "My Python app.lnk")
try:
from win32com.client import Dispatch
from pywintypes import com_error
try:
sh = Dispatch('WScript.Shell')
link = sh.CreateShortcut(linkpath)
link.TargetPath = exepath
link.Save()
return True
except com_error:
return False
except ImportError:
import subprocess
s = "$s=(New-Object -COM WScript.Shell).CreateShortcut('" + linkpath + "');$s.TargetPath='" + exepath + "';$s.Save()"
return subprocess.call(['powershell', s], stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL) == 0
def create_shortcut(inst):
try:
exepath = path.join(path.dirname(sys.executable), "Scripts", "myapp.exe")
if not path.exists(exepath):
# Support for "pip install --user"
exepath = path.join(path.dirname(site.getusersitepackages()), "Scripts", "myapp.exe")
# If can't modify the global menu, fall back to the
# current user's one
if not create_shortcut_under('ALLUSERSPROFILE', exepath):
create_shortcut_under('APPDATA', exepath)
except:
pass
class my_install(install):
def run(self):
install.run(self)
if platform.system() == 'Windows':
create_shortcut(self)
#...
setup(
#...
cmdclass={'install': my_install},
entry_points={"gui_scripts": ["myapp = myapp.__main__:main"]},