I am trying to understand a script found on the Python documentation page.
I do not understand exactly, how to introduce the arguments, I know that I must put the name of the directory where the virtual environment will be installed so that the argument dirs stores the tuple of directories, however I do not understand how to modify the parameters of the other arguments such as nopip.
Code:
import os
import os.path
from subprocess import Popen, PIPE
import sys
from threading import Thread
from urllib.parse import urlparse
from urllib.request import urlretrieve
import venv
class ExtendedEnvBuilder(venv.EnvBuilder):
"""
This builder installs setuptools and pip so that you can pip or
easy_install other packages into the created virtual environment.
:param nodist: If true, setuptools and pip are not installed into the
created virtual environment.
:param nopip: If true, pip is not installed into the created
virtual environment.
:param progress: If setuptools or pip are installed, the progress of the
installation can be monitored by passing a progress
callable. If specified, it is called with two
arguments: a string indicating some progress, and a
context indicating where the string is coming from.
The context argument can have one of three values:
'main', indicating that it is called from virtualize()
itself, and 'stdout' and 'stderr', which are obtained
by reading lines from the output streams of a subprocess
which is used to install the app.
If a callable is not specified, default progress
information is output to sys.stderr.
"""
def __init__(self, *args, **kwargs):
self.nodist = kwargs.pop('nodist', False)
self.nopip = kwargs.pop('nopip', False)
self.progress = kwargs.pop('progress', None)
self.verbose = kwargs.pop('verbose', False)
super().__init__(*args, **kwargs)
def post_setup(self, context):
"""
Set up any packages which need to be pre-installed into the
virtual environment being created.
:param context: The information for the virtual environment
creation request being processed.
"""
os.environ['VIRTUAL_ENV'] = context.env_dir
if not self.nodist:
self.install_setuptools(context)
# Can't install pip without setuptools
if not self.nopip and not self.nodist:
self.install_pip(context)
def reader(self, stream, context):
"""
Read lines from a subprocess' output stream and either pass to a progress
callable (if specified) or write progress information to sys.stderr.
"""
progress = self.progress
while True:
s = stream.readline()
if not s:
break
if progress is not None:
progress(s, context)
else:
if not self.verbose:
sys.stderr.write('.')
else:
sys.stderr.write(s.decode('utf-8'))
sys.stderr.flush()
stream.close()
def install_script(self, context, name, url):
_, _, path, _, _, _ = urlparse(url)
fn = os.path.split(path)[-1]
binpath = context.bin_path
distpath = os.path.join(binpath, fn)
# Download script into the virtual environment's binaries folder
urlretrieve(url, distpath)
progress = self.progress
if self.verbose:
term = '\n'
else:
term = ''
if progress is not None:
progress('Installing %s ...%s' % (name, term), 'main')
else:
sys.stderr.write('Installing %s ...%s' % (name, term))
sys.stderr.flush()
# Install in the virtual environment
args = [context.env_exe, fn]
p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath)
t1 = Thread(target=self.reader, args=(p.stdout, 'stdout'))
t1.start()
t2 = Thread(target=self.reader, args=(p.stderr, 'stderr'))
t2.start()
p.wait()
t1.join()
t2.join()
if progress is not None:
progress('done.', 'main')
else:
sys.stderr.write('done.\n')
# Clean up - no longer needed
os.unlink(distpath)
def install_setuptools(self, context):
"""
Install setuptools in the virtual environment.
:param context: The information for the virtual environment
creation request being processed.
"""
url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py'
self.install_script(context, 'setuptools', url)
# clear up the setuptools archive which gets downloaded
pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz')
files = filter(pred, os.listdir(context.bin_path))
for f in files:
f = os.path.join(context.bin_path, f)
os.unlink(f)
def install_pip(self, context):
"""
Install pip in the virtual environment.
:param context: The information for the virtual environment
creation request being processed.
"""
url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py'
self.install_script(context, 'pip', url)
def main(args=None):
compatible = True
if sys.version_info < (3, 3):
compatible = False
elif not hasattr(sys, 'base_prefix'):
compatible = False
if not compatible:
raise ValueError('This script is only for use with '
'Python 3.3 or later')
else:
import argparse
parser = argparse.ArgumentParser(prog=__name__,
description='Creates virtual Python '
'environments in one or '
'more target '
'directories.')
parser.add_argument('dirs', metavar='ENV_DIR', nargs='+ Pruebas',
help='A directory in which to create the'
'virtual environment.')
parser.add_argument('--no-setuptools', default=False,
action='store_true', dest='nodist',
help="Don't install setuptools or pip in the "
"virtual environment.")
parser.add_argument('--no-pip', default=False,
action='store_true', dest='nopip',
help="Don't install pip in the virtual "
"environment.")
parser.add_argument('--system-site-packages', default=False,
action='store_true', dest='system_site',
help='Give the virtual environment access to the '
'system site-packages dir.')
if os.name == 'nt':
use_symlinks = False
else:
use_symlinks = True
parser.add_argument('--symlinks', default=use_symlinks,
action='store_true', dest='symlinks',
help='Try to use symlinks rather than copies, '
'when symlinks are not the default for '
'the platform.')
parser.add_argument('--clear', default=False, action='store_true',
dest='clear', help='Delete the contents of the '
'virtual environment '
'directory if it already '
'exists, before virtual '
'environment creation.')
parser.add_argument('--upgrade', default=False, action='store_true',
dest='upgrade', help='Upgrade the virtual '
'environment directory to '
'use this version of '
'Python, assuming Python '
'has been upgraded '
'in-place.')
parser.add_argument('--verbose', default=False, action='store_true',
dest='verbose', help='Display the output '
'from the scripts which '
'install setuptools and pip.')
options = parser.parse_args(args)
if options.upgrade and options.clear:
raise ValueError('you cannot supply --upgrade and --clear together.')
builder = ExtendedEnvBuilder(system_site_packages=options.system_site,
clear=options.clear,
symlinks=options.symlinks,
upgrade=options.upgrade,
nodist=options.nodist,
nopip=options.nopip,
verbose=options.verbose)
for d in options.dirs:
builder.create(d)
if __name__ == '__main__':
rc = 1
try:
main()
rc = 0
except Exception as e:
print('Error: %s' % e, file=sys.stderr)
sys.exit(rc)
All the other options could be modified through command line arguments. You could see available options if run script with --help flag:
python3 pyvenvex.py --help
Output:
Usage: __main__ [-h] [--no-setuptools] [--no-pip] [--system-site-packages]
[--symlinks] [--clear] [--upgrade] [--verbose]
ENV_DIR [ENV_DIR ...]
Creates virtual Python environments in one or more target directories.
positional arguments:
ENV_DIR A directory to create the environment in.
optional arguments:
-h, --help show this help message and exit
--no-setuptools Don't install setuptools or pip in the virtual
environment.
--no-pip Don't install pip in the virtual environment.
--system-site-packages
Give the virtual environment access to the system
site-packages dir.
--symlinks Try to use symlinks rather than copies, when symlinks
are not the default for the platform.
--clear Delete the contents of the environment directory if it
already exists, before environment creation.
--upgrade Upgrade the environment directory to use this version
of Python, assuming Python has been upgraded in-place.
--verbose Display the output from the scripts which install
setuptools and pip.
This arguments handled with argparse module from standard library. You could see a lot of boilerplate code like this in the script:
parser.add_argument('--no-pip', default=False,
action='store_true', dest='nopip',
help="Don't install pip in the virtual "
"environment.")
The only tricky thing here is the keyword argument dest='nopip', which means "parse --no-pip argument and put result into nopip attribute".
Trying to run git-filter-repo command from python. I can't figure out how this should be done.
git filter-repo --path README.md --path guides/ --path tools/releases
So far I have:
filtering_options = git_filter_repo.FilteringOptions.default_options()
filtering_options.source = _fragment_repo_git_directory(workspace, "svn-import")
filtering_options.target = _main_repo_git_directory(workspace).encode()
filtering_options.force = True
filtering_options.replace_refs = "update no add"
repo_filter = git_filter_repo.RepoFilter(
filtering_options,
# ?????????
)
repo_filter.run()
You can take inspiration from the newren/git-filter-repo/contrib/filter-repo-demos/lint-history script, which does use a callback function as your # ????????? argument:
try:
import git_filter_repo as fr
except ImportError:
raise SystemExit("Error: Couldn't find git_filter_repo.py. Did you forget to make a symlink to git-filter-repo named git_filter_repo.py or did you forget to put the latter in your PYTHONPATH?")
def lint_non_binary_blobs(blob, metadata):
...
filter = fr.RepoFilter(args, blob_callback=lint_non_binary_blobs)
filter.run()
Given that my library with foobar.py is setup as such:
\foobar.py
\foobar
\__init__.py
\setup.py
Hierarchy of CLI in the console script:
foobar.py
\cli
\foo
\kungfu
\kungpow
\bar
\blacksheep
\haveyouanywool
[code]:
import click
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
#click.group()
#click.version_option()
def cli():
pass
#cli.group(context_settings=CONTEXT_SETTINGS)
def foo():
pass
#cli.group(context_settings=CONTEXT_SETTINGS)
def bar():
pass
#foo.command('kungfu')
def kungfu():
print('bruise lee')
#foo.command('kungpow')
def kungpow():
print('chosen one')
#bar.command('blacksheep')
def blacksheep():
print('bah bah blacksheep')
#bar.command('haveyouanywool')
def haveyouanywool():
print('have you any wool?')
How should I set my entry in setup.py?
There are many examples but they only show a single command for a single entry point, e.g. Entry Points in setup.py
But is it even possible to setup the console script with how the my foobar.py click script is structured?
If not, how should I restructure the commands in foobar.py?
For context, I have this script for the sacremoses library: https://github.com/alvations/sacremoses/blob/cli/sacremoses.py
But I couldn't figure how to configure the setup.py to install the sacremoses.py script properly: https://github.com/alvations/sacremoses/blob/cli/setup.py
To make the entry points work in your example you need:
entry_points='''
[console_scripts]
command_line_name=foobar:cli
''',
What you are missing is an understanding of the meaning of:
command_line_name=foobar:cli
[console_scripts]
There are three things in command_line_name=foobar:cli:
Name of the script from the command line (command_line_name)
Module where the click command handler is located (foobar)
Name of the click command/group in that module (cli)
setup.py
For your github example, I would suggest:
from distutils.core import setup
import setuptools
console_scripts = """
[console_scripts]
sacremoses=sacremoses.cli:cli
"""
setup(
name='sacremoses',
packages=['sacremoses'],
version='0.0.7',
description='SacreMoses',
long_description='LGPL MosesTokenizer in Python',
author='',
license='',
package_data={'sacremoses': [
'data/perluniprops/*.txt',
'data/nonbreaking_prefixes/nonbreaking_prefix.*'
]},
url='https://github.com/alvations/sacremoses',
keywords=[],
classifiers=[],
install_requires=['six', 'click', 'joblib', 'tqdm'],
entry_points=console_scripts,
)
Command Handler
In the referenced branch of your github repo, there is NO cli.py file. The [code] from your question needs to be saved in sacremoses/cli.py, and then combined with the suggested changes to your setup.py, everything should work fine.
I'm trying to configure setuptools and Click module for multiple functions.
Click documentation instructs in Nesting Commands section to use click.group().
How do you write the entry_points for multiple CLick CLI functions?
I was toying with they syntax, and I managed to get something working but I can't recreate it. I was something like this,
entry_points='''
[console_scripts]
somefunc=yourscript:somefunc
morefunc=yourscript:morefunc
'''
Following the sample given below, I converted the syntax to a dictionary:
entry_points= {'console_scripts':
['somefunc = yourscript:somefunc',
'morefunc = yourscript:morefunc'
]},
After I reinstalled, calling the script raised this error:
(clickenv) > somefunc
Traceback (most recent call last):
[...]
raise TypeError('Attempted to convert a callback into a '
TypeError: Attempted to convert a callback into a command twice.
The way I made this work the first time, was I installed the script, and then gradually changed the code through the various examples. At one point, just as described in the docs, I called the script with $ yourscript somefunc. However, when I recreated the pattern in my project I got that error.
Here I've uninstalled and reinstalled (even though its advertised as unnecessary, pip install -e .) and removed the second entrypoint. Here's my testing example. The function morefunc requires a .txt input file.
# yourscript.py
import click
#click.command()
#click.group()
def cli():
pass
#cli.command()
def somefunc():
click.echo('Hello World!')
#cli.command()
#click.argument('input', type=click.File('rb'))
#click.option('--saveas', default='HelloWorld.txt', type=click.File('wb'))
def morefunc(input, saveas):
while True:
chunk = input.read(1024)
if not chunk:
break
saveas.write(chunk)
# setup.py
from setuptools import setup
setup(
name='ClickCLITest',
version='0.1',
py_modules=['yourscript'],
install_requires=[
'Click',
],
entry_points= {'console_scripts':
['somefunc = yourscript:somefunc']},
)
https://setuptools.readthedocs.io/en/latest/setuptools.html#automatic-script-creation
setup(
…
entry_points={
'console_scripts': [
'somefunc=yourscript:somefunc',
'morefunc=yourscript:morefunc',
],
},
…
)
I would like to have a target (say docs) that runs epydoc for my package. I assume that I need to create a new command but I am not having much luck.
Has anyone done this before?
The Babel project provides several commands for use in setup.py files.
You need to define a distutils.commands entry point with commands; example from the Babel setup.py file:
entry_points = """
[distutils.commands]
compile_catalog = babel.messages.frontend:compile_catalog
extract_messages = babel.messages.frontend:extract_messages
init_catalog = babel.messages.frontend:init_catalog
update_catalog = babel.messages.frontend:update_catalog
"""
where the extra commands are then available as python setup.py commandname.
The entry points point to subclasses of from distutils.cmd import Command. Example again from Babel, from the babel.messages.frontend module:
from distutils.cmd import Command
from distutils.errors import DistutilsOptionError
class compile_catalog(Command):
"""Catalog compilation command for use in ``setup.py`` scripts."""
# Description shown in setup.py --help-commands
description = 'compile message catalogs to binary MO files'
# Options available for this command, tuples of ('longoption', 'shortoption', 'help')
# If the longoption name ends in a `=` it takes an argument
user_options = [
('domain=', 'D',
"domain of PO file (default 'messages')"),
('directory=', 'd',
'path to base directory containing the catalogs'),
# etc.
]
# Options that don't take arguments, simple true or false options.
# These *must* be included in user_options too, but without a = equals sign
boolean_options = ['use-fuzzy', 'statistics']
def initialize_options(self):
# Set a default for each of your user_options (long option name)
def finalize_options(self):
# verify the arguments and raise DistutilOptionError if needed
def run(self):
# Do your thing here.