I'm working on a bazel rule (using version 5.2.0) that uses SWIG (version 4.0.1) to make a python library from C++ code, adapted from a rule in the tensorflow library. The problem I've run into is that, depending on the contents of ctx.file.source.path, the swig invocation might produce a necessary .h file. If it does, the rule below works great. If it doesn't, I get:
ERROR: BUILD:31:11: output 'foo_swig_h.h' was not created
ERROR: BUILD:31:11: SWIGing foo.i. failed: not all outputs were created or valid
If the h_out stuff is removed from _py_swig_gen_impl, the rule below works great when swig doesn't produce the .h file. But, if swig does produce one, bazel seems to ignore it and it isn't available for native.cc_binary to compile, resulting in gcc failing with a 'no such file or directory' error on an #include <foo_swig_cc.h> line in foo_swig_cc.cc.
(The presence or absence of the .h file in the output is determined by whether the .i file at ctx.file.source.path uses SWIG's "directors" feature.)
def _include_dirs(deps):
return depset(transitive = [dep[CcInfo].compilation_context.includes for dep in deps]).to_list()
def _headers(deps):
return depset(transitive = [dep[CcInfo].compilation_context.headers for dep in deps]).to_list()
# Bazel rules for building swig files.
def _py_swig_gen_impl(ctx):
module_name = ctx.attr.module_name
cc_out = ctx.actions.declare_file(module_name + "_swig_cc.cc")
h_out = ctx.actions.declare_file(module_name + "_swig_h.h")
py_out = ctx.actions.declare_file(module_name + ".py")
args = ["-c++", "-python", "-py3"]
args += ["-module", module_name]
args += ["-I" + x for x in _include_dirs(ctx.attr.deps)]
args += ["-I" + x.dirname for x in ctx.files.swig_includes]
args += ["-o", cc_out.path]
args += ["-outdir", py_out.dirname]
args += ["-oh", h_out.path]
args.append(ctx.file.source.path)
outputs = [cc_out, h_out, py_out]
ctx.actions.run(
executable = "swig",
arguments = args,
mnemonic = "Swig",
inputs = [ctx.file.source] + _headers(ctx.attr.deps) + ctx.files.swig_includes,
outputs = outputs,
progress_message = "SWIGing %{input}.",
)
return [DefaultInfo(files = depset(direct = [cc_out, py_out]))]
_py_swig_gen = rule(
attrs = {
"source": attr.label(
mandatory = True,
allow_single_file = True,
),
"swig_includes": attr.label_list(
allow_files = [".i"],
),
"deps": attr.label_list(
allow_files = True,
providers = [CcInfo],
),
"module_name": attr.string(mandatory = True),
},
implementation = _py_swig_gen_impl,
)
def py_wrap_cc(name, source, module_name = None, deps = [], copts = [], **kwargs):
if module_name == None:
module_name = name
python_deps = [
"#local_config_python//:python_headers",
"#local_config_python//:python_lib",
]
# First, invoke the _py_wrap_cc rule, which runs swig. This outputs:
# `module_name.cc`, `module_name.py`, and, sometimes, `module_name.h` files.
swig_rule_name = "swig_gen_" + name
_py_swig_gen(
name = swig_rule_name,
source = source,
swig_includes = ["//third_party/swig_rules:swig_includes"],
deps = deps + python_deps,
module_name = module_name,
)
# Next, we need to compile the `module_name.cc` and `module_name.h` files
# from the previous rule. The `module_name.py` file already generated
# expects there to be a `_module_name.so` file, so we name the cc_binary
# rule this way to make sure that's the resulting file name.
cc_lib_name = "_" + module_name + ".so"
native.cc_binary(
name = cc_lib_name,
srcs = [":" + swig_rule_name],
linkopts = ["-dynamic", "-L/usr/local/lib/"],
linkshared = True,
deps = deps + python_deps,
)
# Finally, package everything up as a python library that can be depended
# on. Note that this rule uses the user-given `name`.
native.py_library(
name = name,
srcs = [":" + swig_rule_name],
srcs_version = "PY3",
data = [":" + cc_lib_name],
imports = ["./"],
)
My question, broadly, how I might best handle this with a single rule. I've tried adding a ctx.actions.write before the ctx.actions.run, thinking that I could generate a dummy '.h' file that would be overwritten if needed. That gives me:
ERROR: BUILD:41:11: for foo_swig_h.h, previous action: action 'Writing file foo_swig_h.h', attempted action: action 'SWIGing foo.i.'
My next idea is to remove the h_out stuff and then try to capture the h file for the cc_binary rule with some kind of glob invocation.
I've seen two approaches: add an attribute to indicate whether it applies, or write a wrapper script to generate it unconditionally.
Adding an attribute means something like "has_h": attr.bool(), and then use that in _py_swig_gen_impl to make the ctx.actions.declare_file(module_name + "_swig_h.h") conditional.
The wrapper script option means using something like this for the executable:
#!/bin/bash
set -e
touch the_path_of_the_header
exec swig "$#"
That will unconditionally create the output, and then swig will overwrite it if applicable. If it's not applicable, then passing around an empty header file in the Bazel rules should be harmless.
For posterity, this is what my _py_swig_gen_impl looks like after implementing #Brian's suggestion above:
def _py_swig_gen_impl(ctx):
module_name = ctx.attr.module_name
cc_out = ctx.actions.declare_file(module_name + "_swig_cc.cc")
h_out = ctx.actions.declare_file(module_name + "_swig_h.h")
py_out = ctx.actions.declare_file(module_name + ".py")
include_dirs = _include_dirs(ctx.attr.deps)
headers = _headers(ctx.attr.deps)
args = ["-c++", "-python", "-py3"]
args += ["-module", module_name]
args += ["-I" + x for x in include_dirs]
args += ["-I" + x.dirname for x in ctx.files.swig_includes]
args += ["-o", cc_out.path]
args += ["-outdir", py_out.dirname]
args += ["-oh", h_out.path]
args.append(ctx.file.source.path)
outputs = [cc_out, h_out, py_out]
# Depending on the contents of `ctx.file.source`, swig may or may not
# output a .h file needed by subsequent rules. Bazel doesn't like optional
# outputs, so instead of invoking swig directly we're going to make a
# lightweight executable script that first `touch`es the .h file that may
# get generated, and then execute that. This means we may be propagating
# an empty .h file around as a "dependency" sometimes, but that's okay.
swig_script_file = ctx.actions.declare_file("swig_exec.sh")
ctx.actions.write(
output = swig_script_file,
is_executable = True,
content = "#!/bin/bash\n\nset -e\ntouch " + h_out.path + "\nexec swig \"$#\"",
)
ctx.actions.run(
executable = swig_script_file,
arguments = args,
mnemonic = "Swig",
inputs = [ctx.file.source] + headers + ctx.files.swig_includes,
outputs = outputs,
progress_message = "SWIGing %{input}.",
)
return [
DefaultInfo(files = depset(direct = outputs)),
]
The ctx.actions.write generates the suggested bash script:
#!/bin/bash
set -e
touch %{h_out.path}
exec swig "$#"
Which guarantees that the expected h_out will always be output by ctx.actions.run, whether or not swig generates it.
Related
I've a monorepo that contains a set of Python AWS lambdas and I'm using Bazel for building and packaging the lambdas. I'm now trying to use Bazel to create a zip file that follows the expected AWS Lambdas packaging and that I can upload to Lambda. Wondering what's the best way to do this with Bazel?
Below are a few different things I've tried thus far:
Attempt 1: py_binary
BUILD.bazel
py_binary(
name = "main_binary",
srcs = glob(["*.py"]),
main = "main.py",
visibility = ["//appcode/api/transaction_details:__subpackages__"],
deps = [
requirement("Faker"),
],
)
Problem:
This generates the following:
main_binary (python executable)
main_binary.runfiles
main_binary.runfiles_manifest
Lambda expects the handler to be in the format of lambda_function.lambda_handler. Since main_binary is an executable vs. a python file, it doesn't expose the actual handler method and the lambda blows up because it can't find it. I tried updating the handler configuration to simply point to the main_binary but it blows up because it expects two arguments(i.e. lambda_function.lambda_handler).
Attempt 2: py_library + pkg_zip
BUILD.bazel
py_library(
name = "main",
srcs = glob(["*.py"]),
visibility = ["//appcode/api/transaction_details:__subpackages__"],
deps = [
requirement("Faker"),
],
)
pkg_zip(
name = "main_zip",
srcs =["//appcode/api/transaction_details/src:main" ],
)
Problem:
This generates a zip file with:
main.py
__init__.py
The zip file now includes the main.py but none of its runtime dependencies. Thus the lambda blows up because it can't find Faker.
Other Attempts:
I've also tried using the --build_python_zip flag as well as the #bazel_tools//tools/zip:zipper with a generic rule but they both lead to similar outcomes as the two previous attempts.
We use #bazel_tools//tools/zip:zipper with a custom rule. We also pull serverless in using rules_nodejs and run it through bazel, which causes the package building to happen prior to running sls deploy.
We use pip_parse from rules_python. I'm not sure whether the _short_path function below will work with pip_install or other mechanisms.
File filtering is supported, although it's awkward. Ideally the zip generation would be handled by a separate binary (i.e., a Python script) which would allow filtering using regular expressions/globs/etc. Bazel doesn't support regular expressions in Starlark, so we use our own thing.
I've included an excerpt:
lambda.bzl
"""
Support for serverless deployments.
"""
def contains(pattern):
return "contains:" + pattern
def startswith(pattern):
return "startswith:" + pattern
def endswith(pattern):
return "endswith:" + pattern
def _is_ignored(path, patterns):
for p in patterns:
if p.startswith("contains:"):
if p[len("contains:"):] in path:
return True
elif p.startswith("startswith:"):
if path.startswith(p[len("startswith:"):]):
return True
elif p.startswith("endswith:"):
if path.endswith(p[len("endswith:"):]):
return True
else:
fail("Invalid pattern: " + p)
return False
def _short_path(file_):
# Remove prefixes for external and generated files.
# E.g.,
# ../py_deps_pypi__pydantic/pydantic/__init__.py -> pydantic/__init__.py
short_path = file_.short_path
if short_path.startswith("../"):
second_slash = short_path.index("/", 3)
short_path = short_path[second_slash + 1:]
return short_path
def _py_lambda_zip_impl(ctx):
deps = ctx.attr.target[DefaultInfo].default_runfiles.files
f = ctx.outputs.output
args = []
for dep in deps.to_list():
short_path = _short_path(dep)
# Skip ignored patterns
if _is_ignored(short_path, ctx.attr.ignore):
continue
args.append(short_path + "=" + dep.path)
ctx.actions.run(
outputs = [f],
inputs = deps,
executable = ctx.executable._zipper,
arguments = ["cC", f.path] + args,
progress_message = "Creating archive...",
mnemonic = "archiver",
)
out = depset(direct = [f])
return [
DefaultInfo(
files = out,
),
OutputGroupInfo(
all_files = out,
),
]
_py_lambda_zip = rule(
implementation = _py_lambda_zip_impl,
attrs = {
"target": attr.label(),
"ignore": attr.string_list(),
"_zipper": attr.label(
default = Label("#bazel_tools//tools/zip:zipper"),
cfg = "host",
executable = True,
),
"output": attr.output(),
},
executable = False,
test = False,
)
def py_lambda_zip(name, target, ignore, **kwargs):
_py_lambda_zip(
name = name,
target = target,
ignore = ignore,
output = name + ".zip",
**kwargs
)
BUILD.bazel
load("#npm_serverless//serverless:index.bzl", "serverless")
load(":lambda.bzl", "contains", "endswith", "py_lambda_zip", "startswith")
py_binary(
name = "my_lambda_app",
...
)
py_lambda_zip(
name = "lambda_archive",
ignore = [
contains("/__pycache__/"),
endswith(".pyc"),
endswith(".pyo"),
# Ignore boto since it's provided by Lambda.
startswith("boto3/"),
startswith("botocore/"),
# With the move to hermetic toolchains, the zip gets a lib/ directory containing the
# python runtime. We don't need that.
startswith("lib/"),
],
target = ":my_lambda_app",
# Only allow building on linux, since we don't want to upload a lambda zip file
# with e.g. macos compiled binaries.
target_compatible_with = [
"#platforms//os:linux",
],
)
# The sls command requires that serverless.yml be in its working directory, and that the yaml file
# NOT be a symlink. So this target builds a directory containing a copy of serverless.yml, and also
# symlinks the generated lambda_archive.zip in the same directory.
#
# It also generates a chdir.js script that we instruct node to execute to change to the proper working directory.
genrule(
name = "sls_files",
srcs = [
"lambda_archive.zip",
"serverless.yml",
],
outs = [
"sls_files/lambda_archive.zip",
"sls_files/serverless.yml",
"sls_files/chdir.js",
],
cmd = """
mkdir -p $(#D)/sls_files
cp $(location serverless.yml) $(#D)/sls_files/serverless.yml
cp -P $(location lambda_archive.zip) $(#D)/sls_files/lambda_archive.zip
echo "const fs = require('fs');" \
"const path = require('path');" \
"process.chdir(path.dirname(fs.realpathSync(__filename)));" > $(#D)/sls_files/chdir.js
""",
)
# Usage:
# bazel run //:sls -- deploy <more args>
serverless(
name = "sls",
args = ["""--node_options=--require=./$(location sls_files/chdir.js)"""],
data = [
"sls_files/chdir.js",
"sls_files/serverless.yml",
"sls_files/lambda_archive.zip",
],
)
serverless.yml
service: my-app
package:
artifact: lambda_archive.zip
# ... other config ...
Below are the changes I made to the previous answer to generate the lambda zip. Thanks #jvolkman for the original suggestion.
project/BUILD.bazel: Added rule to generate requirements_lock.txt from project/requirements.txt
load("#rules_python//python:pip.bzl", "compile_pip_requirements")
compile_pip_requirements(
name = "requirements",
extra_args = ["--allow-unsafe"],
requirements_in = "requirements.txt",
requirements_txt = "requirements_lock.txt",
)
project/WORKSPACE.bazel: swap pip_install with pip_parse
workspace(name = "mdc-eligibility")
load("#bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
name = "rules_python",
sha256 = "9fcf91dbcc31fde6d1edb15f117246d912c33c36f44cf681976bd886538deba6",
strip_prefix = "rules_python-0.8.0",
url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.8.0.tar.gz",
)
load("#rules_python//python:repositories.bzl", "python_register_toolchains")
python_register_toolchains(
name = "python3_9",
python_version = "3.9",
)
load("#rules_python//python:pip.bzl", "pip_parse")
load("#python3_9//:defs.bzl", "interpreter")
pip_parse(
name = "mndc-eligibility-deps",
requirements_lock = "//:requirements_lock.txt",
python_interpreter_target = interpreter,
quiet = False
)
load("#mndc-eligibility-deps//:requirements.bzl", "install_deps")
install_deps()
project/build_rules/lambda_packaging/lambda.bzl: Modified custom rule provided by #jvolkman to include source code in the resulting zip code.
def contains(pattern):
return "contains:" + pattern
def startswith(pattern):
return "startswith:" + pattern
def endswith(pattern):
return "endswith:" + pattern
def _is_ignored(path, patterns):
for p in patterns:
if p.startswith("contains:"):
if p[len("contains:"):] in path:
return True
elif p.startswith("startswith:"):
if path.startswith(p[len("startswith:"):]):
return True
elif p.startswith("endswith:"):
if path.endswith(p[len("endswith:"):]):
return True
else:
fail("Invalid pattern: " + p)
return False
def _short_path(file_):
# Remove prefixes for external and generated files.
# E.g.,
# ../py_deps_pypi__pydantic/pydantic/__init__.py -> pydantic/__init__.py
short_path = file_.short_path
if short_path.startswith("../"):
second_slash = short_path.index("/", 3)
short_path = short_path[second_slash + 1:]
return short_path
# steven chambers
def _py_lambda_zip_impl(ctx):
deps = ctx.attr.target[DefaultInfo].default_runfiles.files
f = ctx.outputs.output
args = []
for dep in deps.to_list():
short_path = _short_path(dep)
# Skip ignored patterns
if _is_ignored(short_path, ctx.attr.ignore):
continue
args.append(short_path + "=" + dep.path)
# MODIFICATION: Added source files to the map of files to zip
source_files = ctx.attr.target[DefaultInfo].files
for source_file in source_files.to_list():
args.append(source_file.basename+"="+source_file.path)
ctx.actions.run(
outputs = [f],
inputs = deps,
executable = ctx.executable._zipper,
arguments = ["cC", f.path] + args,
progress_message = "Creating archive...",
mnemonic = "archiver",
)
out = depset(direct = [f])
return [
DefaultInfo(
files = out,
),
OutputGroupInfo(
all_files = out,
),
]
_py_lambda_zip = rule(
implementation = _py_lambda_zip_impl,
attrs = {
"target": attr.label(),
"ignore": attr.string_list(),
"_zipper": attr.label(
default = Label("#bazel_tools//tools/zip:zipper"),
cfg = "host",
executable = True,
),
"output": attr.output(),
},
executable = False,
test = False,
)
def py_lambda_zip(name, target, ignore, **kwargs):
_py_lambda_zip(
name = name,
target = target,
ignore = ignore,
output = name + ".zip",
**kwargs
)
project/appcode/api/transaction_details/src/BUILD.bazel: Used custom py_lambda_zip rule to zip up py_library
load("#mndc-eligibility-deps//:requirements.bzl", "requirement")
load("#python3_9//:defs.bzl", "interpreter")
load("//build_rules/lambda_packaging:lambda.bzl", "contains", "endswith", "py_lambda_zip", "startswith")
py_library(
name = "main",
srcs = glob(["*.py"]),
visibility = ["//appcode/api/transaction_details:__subpackages__"],
deps = [
requirement("Faker"),
],
)
py_lambda_zip(
name = "lambda_archive",
ignore = [
contains("/__pycache__/"),
endswith(".pyc"),
endswith(".pyo"),
# Ignore boto since it's provided by Lambda.
startswith("boto3/"),
startswith("botocore/"),
# With the move to hermetic toolchains, the zip gets a lib/ directory containing the
# python runtime. We don't need that.
startswith("lib/"),
],
target = ":main",
)
I have a VBA script in Microsoft Access. The VBA script is part of a large project with multiple people, and so it is not possible to leave the VBA environment.
In a section of my script, I need to do complicated linear algebra on a table quickly. So, I move the VBA tables written as recordsets) into Python to do linear algebra, and back into VBA. The matrices in python are represented as numpy arrays.
Some of the linear algebra is proprietary and so we are compiling the proprietary scripts with pyinstaller.
The details of the process are as follows:
The VBA script creates a csv file representing the table input.csv.
The VBA script runs the python script through the command line
The python script loads the csv file input.csv as a numpy matrix, does linear algebra on it, and creates an output csv file output.csv.
VBA waits until python is done, then loads output.csv.
VBA deletes the no-longer-needed input.csv file and output.csv file.
This process is inefficient.
Is there a way to load VBA matrices into Python (and back) without the csv clutter? Do these methods work with compiled python code through pyinstaller?
I have found the following examples on stackoverflow that are relevant. However, they do not address my problem specifically.
Return result from Python to Vba
How to pass Variable from Python to VBA Sub
Solution 1
Either retrieve the COM running instance of Access and get/set the data directly with the python script via the COM API:
VBA:
Private Cache
Public Function GetData()
GetData = Cache
Cache = Empty
End Function
Public Sub SetData(data)
Cache = data
End Sub
Sub Usage()
Dim wshell
Set wshell = VBA.CreateObject("WScript.Shell")
' Make the data available via GetData()'
Cache = Array(4, 6, 8, 9)
' Launch the python script compiled with pylauncher '
Debug.Assert 0 = wshell.Run("C:\dev\myapp.exe", 0, True)
' Handle the returned data '
Debug.Assert Cache(3) = 2
End Sub
Python (myapp.exe):
import win32com.client
if __name__ == "__main__":
# get the running instance of Access
app = win32com.client.GetObject(Class="Access.Application")
# get some data from Access
data = app.run("GetData")
# return some data to Access
app.run("SetData", [1, 2, 3, 4])
Solution 2
Or create a COM server to expose some functions to Access :
VBA:
Sub Usage()
Dim Py As Object
Set Py = CreateObject("Python.MyModule")
Dim result
result = Py.MyFunction(Array(5, 6, 7, 8))
End Sub
Python (myserver.exe or myserver.py):
import sys, os, win32api, win32com.server.localserver, win32com.server.register
class MyModule(object):
_reg_clsid_ = "{5B4A4174-EE23-4B70-99F9-E57958CFE3DF}"
_reg_desc_ = "My Python COM Server"
_reg_progid_ = "Python.MyModule"
_public_methods_ = ['MyFunction']
def MyFunction(self, data) :
return [(1,2), (3, 4)]
def register(*classes) :
regsz = lambda key, val: win32api.RegSetValue(-2147483647, key, 1, val)
isPy = not sys.argv[0].lower().endswith('.exe')
python_path = isPy and win32com.server.register._find_localserver_exe(1)
server_path = isPy and win32com.server.register._find_localserver_module()
for cls in classes :
if isPy :
file_path = sys.modules[cls.__module__].__file__
class_name = '%s.%s' % (os.path.splitext(os.path.basename(file_path))[0], cls.__name__)
command = '"%s" "%s" %s' % (python_path, server_path, cls._reg_clsid_)
else :
file_path = sys.argv[0]
class_name = '%s.%s' % (cls.__module__, cls.__name__)
command = '"%s" %s' % (file_path, cls._reg_clsid_)
regsz("SOFTWARE\\Classes\\" + cls._reg_progid_ + '\\CLSID', cls._reg_clsid_)
regsz("SOFTWARE\\Classes\\AppID\\" + cls._reg_clsid_, cls._reg_progid_)
regsz("SOFTWARE\\Classes\\CLSID\\" + cls._reg_clsid_, cls._reg_desc_)
regsz("SOFTWARE\\Classes\\CLSID\\" + cls._reg_clsid_ + '\\LocalServer32', command)
regsz("SOFTWARE\\Classes\\CLSID\\" + cls._reg_clsid_ + '\\ProgID', cls._reg_progid_)
regsz("SOFTWARE\\Classes\\CLSID\\" + cls._reg_clsid_ + '\\PythonCOM', class_name)
regsz("SOFTWARE\\Classes\\CLSID\\" + cls._reg_clsid_ + '\\PythonCOMPath', os.path.dirname(file_path))
regsz("SOFTWARE\\Classes\\CLSID\\" + cls._reg_clsid_ + '\\Debugging', "0")
print('Registered ' + cls._reg_progid_)
if __name__ == "__main__":
if len(sys.argv) > 1 :
win32com.server.localserver.serve(set([v for v in sys.argv if v[0] == '{']))
else :
register(MyModule)
Note that you'll have to run the script once without any argument to register the class and to make it available to VBA.CreateObject.
Both solutions work with pylauncher and the array received in python can be converted with numpy.array(data).
Dependency :
https://pypi.python.org/pypi/pywin32
You can try loading your record set into an array, dim'ed as Double
Dim arr(1 to 100, 1 to 100) as Double
by looping, then pass the pointer to the first element ptr = VarPtr(arr(1, 1)) to Python, where
arr = numpy.ctypeslib.as_array(ptr, (100 * 100,)) ?
But VBA will still own the array memory
There is a very simple way of doing this with xlwings. See xlwings.org and make sure to follow the instructions to enable macro settings, tick xlwings in VBA references, etc. etc.
The code would then look as simple as the following (a slightly silly block of code that just returns the same dataframe back, but you get the picture):
import xlwings as xw
import numpy as np
import pandas as pd
# the #xw.decorator is to tell xlwings to create an Excel VBA wrapper for this function.
# It has no effect on how the function behaves in python
#xw.func
#xw.arg('pensioner_data', pd.DataFrame, index=False, header=True)
#xw.ret(expand='table', index=False)
def pensioner_CF(pensioner_data, mortality_table = "PA(90)", male_age_adj = 0, male_improv = 0, female_age_adj = 0, female_improv = 0,
years_improv = 0, arrears_advance = 0, discount_rate = 0, qxy_tables=0):
pensioner_data = pensioner_data.replace(np.nan, '', regex=True)
cashflows_df = pd.DataFrame()
return cashflows_df
I'd be interested to hear if this answers the question. It certainly made my VBA / python experience a lot easier.
This command works fine on my personal computer but keeps giving me this error on my work PC. What could be going on? I can run the Char_Limits.py script directly in Powershell without a problem.
error: compiling 'C:\ProgramData\Anaconda2\lib\site-packages\jinja2\asyncsupport.py' failed
SyntaxError: invalid syntax (asyncsupport.py, line 22)
My setup.py file looks like:
from distutils.core import setup
import py2exe
setup (console=['Char_Limits.py'])
My file looks like:
import xlwings as xw
from win32com.client import constants as c
import win32api
"""
Important Notes: Header row has to be the first row. No columns without a header row. If you need/want a blank column, just place a random placeholder
header value in the first row.
Product_Article_Number column is used to determine the number of rows. It must be populated for every row.
"""
#functions, hooray!
def setRange(columnDict, columnHeader):
column = columnDict[columnHeader]
rngForFormatting = xw.Range((2,column), (bttm, column))
cellReference = xw.Range((2,column)).get_address(False, False)
return rngForFormatting, cellReference
def msg_box(message):
win32api.MessageBox(wb.app.hwnd, message)
#Character limits for fields in Hybris
CharLimits_Fields = {"alerts":500, "certifications":255, "productTitle":300,
"teaserText":450 , "includes":1000, "compliance":255, "disclaimers":9000,
"ecommDescription100":100, "ecommDescription240":240,
"internalKeyword":1000, "metaKeywords":1000, "metaDescription":1000,
"productFeatures":7500, "productLongDescription":1500,"requires":500,
"servicePlan":255, "skuDifferentiatorText":255, "storage":255,
"techDetailsAndRefs":12000, "warranty":1000}
# Fields for which a break tag is problematic.
BreakTagNotAllowed = ["ecommDescription100", "ecommDescription240", "productTitle",
"skuDifferentiatorText"]
app = xw.apps.active
wb = xw.Book(r'C:\Users\XXXX\Documents\Import File.xlsx')
#identifies the blanket range of interest
firstCell = xw.Range('A1')
lstcolumn = firstCell.end("right").column
headers_Row = xw.Range((1,1), (1, lstcolumn)).value
columnDict = {}
for column in range(1, len(headers_Row) + 1):
header = headers_Row[column - 1]
columnDict[header] = column
try:
articleColumn = columnDict["Product_Article_Number"]
except:
articleColumn = columnDict["Family_Article_Number"]
firstCell = xw.Range((1,articleColumn))
bttm = firstCell.end("down").row
wholeRange = xw.Range((1,1),(bttm, lstcolumn))
wholeRangeVal = wholeRange.value
#Sets the font and deletes previous conditional formatting
wholeRange.api.Font.Name = "Arial Unicode MS"
wholeRange.api.FormatConditions.Delete()
for columnHeader in columnDict.keys():
if columnHeader in CharLimits_Fields.keys():
rng, cellRef = setRange(columnDict, columnHeader)
rng.api.FormatConditions.Add(2,3, "=len(" + cellRef + ") >=" + str(CharLimits_Fields[columnHeader]))
rng.api.FormatConditions(1).Interior.ColorIndex = 3
if columnHeader in BreakTagNotAllowed:
rng, cellRef = setRange(columnDict, columnHeader)
rng.api.FormatConditions.Add(2,3, '=OR(ISNUMBER(SEARCH("<br>",' + cellRef + ')), ISNUMBER(SEARCH("<br/>",' + cellRef + ")))")
rng.api.FormatConditions(2).Interior.ColorIndex = 6
searchResults = wholeRange.api.Find("~\"")
if searchResults is not None:
msg_box("There's a double quote in this spreadsheet")
else:
msg_box("There are no double quotes in this spreadsheet")
# app.api.FindFormat.Clear
# app.api.FindFormat.Interior.ColorIndex = 3
# foundRed = wholeRange.api.Find("*", SearchFormat=True)
# if foundRed is None:
# msg_box("There are no values exceeding character limits")
# else:
# msg_box("There are values exceeding character limits")
# app.api.FindFormat.Clear
# app.api.FindFormat.Interior.ColorIndex = 6
# foundYellow = wholeRange.api.Find("*", SearchFormat=True)
# if foundYellow is None:
# msg_box("There are no break tags in this spreadsheet")
# else:
# msg_box("There are break tags in this spreadsheet")
Note:
If you are reading this, I would try Santiago's solution first.
The issue:
Looking at what is likely at line 22 on the github package:
async def concat_async(async_gen):
This is making use of the async keyword which was added in python 3.5, however py2exe only supports up to python 3.4. Now jinja looks to be extending the python language in some way (perhaps during runtime?) to support this async keyword in earlier versions of python. py2exe cannot account for this language extension.
The Fix:
async support was added in jinja2 version 2.9 according to the documentation. So I tried installing an earlier version of jinja (version 2.8) which I downloaded here.
I made a backup of my current jinja installation by moving the contents of %PYTHONHOME%\Lib\site-packages\jinja2 to some other place.
extract the previously downloaded tar.gz file and install the package via pip:
cd .\Downloads\dist\Jinja2-2.8 # or wherever you extracted jinja2.8
python setup.py install
As a side note, I also had to increase my recursion limit because py2exe was reaching the default limit.
from distutils.core import setup
import py2exe
import sys
sys.setrecursionlimit(5000)
setup (console=['test.py'])
Warning:
If whatever it is you are using relies on the latest version of jinja2, then this might fail or have unintended side effects when actually running your code. I was compiling a very simple script.
I had the same trouble coding in python3.7. I fixed that adding the excludes part to my py2exe file:
a = Analysis(['pyinst_test.py'],
#...
excludes=['jinja2.asyncsupport','jinja2.asyncfilters'],
#...)
I took that from: https://github.com/pyinstaller/pyinstaller/issues/2393
I call the python code from a Powershell script in order to loop over some arguments. Calling the python script from a Powershell is straight forward and works without a hitch:
PS C:\Windows\system32> C:\Users\Administrator\AppData\Local\Programs\Python\Python35-32\python.exe C:\Users\Administrator\AppData\Local\Programs\youtube-upload-master\bin\youtube-upload C:\Users\Administrator\Documents\timelapse\videos\timelapse_10.0.0.51-2016-06-21.mp4 --client-secrets=C:\Users\Administrator\Documents\timelapse\credentials\.yt-ul-ioa-secr.json --credentials-file=C:\Users\Administrator\Documents\timelapse\credentials\.yt-ul-ioa-cred.json --title="Timelapse 21.06.2016" --playlist "Timelapses June 2016"
Then within a script I am changing the parameters inserting variables into the argument strings, and finally calling the whole thing with Invoke-Command:
# yt-ul.ps1
param(
#[switch]$all_cams = $false,
[int]$days = -1,
[string]$cam = "ioa"
)
$cam_ip_hash = #{
"ioa" = "10.0.0.51";
"pam" = "10.0.0.52";
"biz" = "10.0.0.56";
"prz" = "10.160.58.25";
"igu" = "10.160.38.35"}
$cam_ip = $cam_ip_hash[$cam]
$date = (Get-Date).AddDays($days).ToString("yyyy-MM-dd")
$py = "C:\Users\Administrator\AppData\Local\Programs\Python\Python35-32\python.exe"
$yt_ul = "C:\Users\Administrator\AppData\Local\Programs\youtube-upload-master\bin\youtube-upload"
$title_date = (Get-Date).AddDays($days).ToString("dd.MM.yyyy")
$us = New-Object System.Globalization.CultureInfo("en-US")
$playlist_date = (Get-Date).AddDays($days).ToString("Y", $us)
$vid = "C:\Users\Administrator\Documents\timelapse\videos\timelapse_$cam_ip-$date.mp4"
$secr = "--client-secrets=C:\Users\Administrator\Documents\timelapse\credentials\.yt-ul-igu-secr.json"
$cred = "--credentials-file=C:\Users\Administrator\Documents\timelapse\credentials\.yt-ul-igu-cred.json"
$title = "--title=`"Timelapse $title_date`""
$playlist_date = "--playlist `"Timelapses $playlist_date`""
$arg_list = "$yt_ul $vid $secr $cred $title $playlist_date"
Invoke-Command "$py $arg_list"
But actually calling the script fails as follows:
PS C:\Users\Administrator\Documents\scripts> .\yt-ul.ps1
Invoke-Command : Parameter set cannot be resolved using the specified named parameters.
At C:\Users\Administrator\Documents\scripts\yt-ul.ps1:34 char:1
+ Invoke-Command "$py $arg_list"
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : InvalidArgument: (:) [Invoke-Command], ParameterBindingException
+ FullyQualifiedErrorId : AmbiguousParameterSet,Microsoft.PowerShell.Commands.InvokeCommandCommand
I assume I am doing something really stupid with the single and double quotes, but I am not sure.
Thanks to JosefZ this works:
& $py $yt_ul $vid $secr $cred --title "Timelapse $title_date" --playlist "Timelapses $playlist_date"
Normally you can concatenate options like '-abbb', which will expand to '-a -b -b -b'. Counts would be 1 for a, abd 3 for b.
However when mixing prefix_chars I see something different ...
import argparse
parser = argparse.ArgumentParser( prefix_chars='-+' )
parser.add_argument( '-x', action='count', dest='counter1' )
parser.add_argument( '+x', action='count', dest='counter2' )
args = parser.parse_args( '-xxx +xxx -xxx'.split() )
print( 'counter1 = ' + str(args.counter1) )
print( 'counter2 = ' + str(args.counter2) )
Running this results in:
counter1 = 8
counter2 = 1
Apparently '+xxx' doesn't expand to '+x +x +x', but to '+x -x -x'.
Changing the prefix_chars to '+-' results in:
counter1 = 2
counter2 = 7
Now '-xxx' expands to '-x +x +x'.
Is this defined behaviour, or am I missing something?
This was patched in late 2010, in early 2.7
http://bugs.python.org/issue9352
================
I'm not aware of bug/issues or code changes that would affect this, but I could dig into it.
For a start, strings of single prefix options are handled rather deeply in the parsing. In the current argparse.py the relevant code is:
class ArgumentParser
def _parse_known_args
# function to convert arg_strings into an optional action
def consume_optional(start_index):
match_argument = self._match_argument
action_tuples = []
while True:
...
chars = self.prefix_chars # e.g. the `-+` parameter
if arg_count == 0 and option_string[1] not in chars:
action_tuples.append((action, [], option_string))
char = option_string[0]
option_string = char + explicit_arg[0]
new_explicit_arg = explicit_arg[1:] or None
optionals_map = self._option_string_actions
if option_string in optionals_map:
action = optionals_map[option_string]
explicit_arg = new_explicit_arg
else:
msg = _('ignored explicit argument %r')
raise ArgumentError(action, msg % explicit_arg)
It's the pair of lines:
char = option_string[0]
option_string = char + explicit_arg[0]
that preserves the initial -/+ when handling the repeated characters (in the unparsed explicit_arg string.
I can imagine the case where the code split +xyz into +x,-y,-z, and was corrected to use +x,+y,+z. But it will require some digging into bug/issues and/or the Python repository to find out if and when that change was made.
What does your problem argparse.py have at this point?