We Keep Coding

Creating a python package (deb/rpm) from cmake

I am trying to create a python package (deb & rpm) from cmake, ideally using cpack. I did read
https://cmake.org/cmake/help/latest/cpack_gen/rpm.html and,
https://cmake.org/cmake/help/latest/cpack_gen/deb.html
The installation works just fine (using component install) for my shared library. However I cannot make sense of the documentation to install the python binding (glue) code. Using the standard cmake install mechanism, I tried:
install(
FILES __init__.py library.py
DESTINATION ${ACME_PYTHON_PACKAGE_DIR}/project_name
COMPONENT python)
And then using brute-force approach ended-up with:
# debian based package (relative path)
set(ACME_PYTHON_PACKAGE_DIR lib/python3/dist-packages)
and
# rpm based package (full path required)
set(ACME_PYTHON_PACKAGE_DIR /var/lang/lib/python3.8/site-packages)
The above is derived from:
debian % python -c 'import site; print(site.getsitepackages())'
['/usr/local/lib/python3.9/dist-packages', '/usr/lib/python3/dist-packages', '/usr/lib/python3.9/dist-packages']
while:
rpm % python -c 'import site; print(site.getsitepackages())'
['/var/lang/lib/python3.8/site-packages']
It is pretty clear that the brute-force approach will not be portable, and is doomed to fail on the next release of python. The only possible solution that I can think of is generating a temporary setup.py python script (using setuptools), that will do the install. Typically cmake would call the following process:
% python setup.py install --root ${ACME_PYTHON_INSTALL_ROOT}
My questions are:
Did I understand the cmake/cpack documentation correctly for python package ? If so this means I need to generate an intermediate setup.py script.
I have been searching through the cmake/cpack codebase (git grep setuptools) but did not find helper functions to handle generation of setup.py and passing the result files back to cpack. Is there an existing cmake module which I could re-use ?
I did read, some alternative solution, such as:
How to build debian package with CPack to execute setup.py?
Which seems overly complex, and geared toward Debian-only based system. I need to handle RPM in my case.

As mentionned in my other solution, the ugly part is dealing with absolute path in cmake install() commands. I was able to refactor the code to avoid usage of absolute path in install(). I simply changed the installation into:
install(
# trailing slash is important:
DIRECTORY ${SETUP_OUTPUT}/
# "." syntax is a reliable mechanism, see:
# https://gitlab.kitware.com/cmake/cmake/-/issues/22616
DESTINATION "."
COMPONENT python)
And then one simply needs to:
set(CMAKE_INSTALL_PREFIX "/")
set(CPACK_PACKAGING_INSTALL_PREFIX "/")
include(CPack)
At this point all install path now need to include explicitely /usr since we've cleared the value for CMAKE_INSTALL_PREFIX.
The above has been tested for deb and rpm packages. CPACK_BINARY_TGZ does properly run with the above solution:
https://gitlab.kitware.com/cmake/cmake/-/issues/22925

I am going to post the temporary solution I am using at the moment, until someone provide something more robust.
So I eventually manage to stumble upon:
https://alioth-lists.debian.net/pipermail/libkdtree-devel/2012-October/000366.html and,
Using CMake with setup.py
Re-using the above to do an install step instead of a build step can be done as follow:
find_package(Python COMPONENTS Interpreter)
set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in")
set(SETUP_PY "${CMAKE_CURRENT_BINARY_DIR}/setup.py")
set(SETUP_DEPS "${CMAKE_CURRENT_SOURCE_DIR}/project_name/__init__.py")
set(SETUP_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/build-python")
configure_file(${SETUP_PY_IN} ${SETUP_PY})
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/setup_timestamp
COMMAND ${Python_EXECUTABLE} ARGS ${SETUP_PY} install --root ${SETUP_OUTPUT}
COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/setup_timestamp
DEPENDS ${SETUP_DEPS})
add_custom_target(target ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/setup_timestamp)
And then the ugly part is:
install(
# trailing slash is important:
DIRECTORY ${SETUP_OUTPUT}/
DESTINATION "/" # FIXME may cause issues with other cpack generators
COMPONENT python)
Turns out that the documentation for install() is pretty clear about absolute paths:
https://cmake.org/cmake/help/latest/command/install.html#introduction
DESTINATION
[...]
As absolute paths are not supported by cpack installer generators,
it is preferable to use relative paths throughout.
For reference, here is my setup.py.in:
from setuptools import setup
if __name__ == '__main__':
setup(name='project_name_python',
version='${PROJECT_VERSION}',
package_dir={'': '${CMAKE_CURRENT_SOURCE_DIR}'},
packages=['project_name'])
You can be fancy and remove the __pycache__ folder using the -B flag:
COMMAND ${Python_EXECUTABLE} ARGS -B ${SETUP_PY} install --root ${SETUP_OUTPUT}
You can be extra fancy and add debian option such as:
if(CPACK_BINARY_DEB)
set(EXTRA_ARG "--install-layout" "deb")
endif()
use as:
COMMAND ${Python_EXECUTABLE} ARGS -B ${SETUP_PY} install --root ${SETUP_OUTPUT} ${EXTRA_ARG}

Cmake seems not to use the Python interpreter it confirms it uses

I have the following CMakeLists.txt file, which is instructed to use Python 3.4
cmake_minimum_required(VERSION 3.2 FATAL_ERROR)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../cmake/")
project(aConfigd VERSION 1.0)
string(TOLOWER aConfigd project_id)
find_package(PythonInterp 3.4 REQUIRED)
include(FindPythonInterp)
set(PYTHON ${PYTHON_EXECUTABLE})
message(STATUS "\${PYTHON_EXECUTABLE} == ${PYTHON_EXECUTABLE}")
set(pkgdatadir /usr/share/configd)
set(configdir /etc/amy)
set(SONARCONFIGID_SOURCE_DIR etc/configd)
set(SRC_DIR configd/src/)
include(common)
# "${SRC_DIR}/systemd_client.py"
# "${SRC_DIR}/amyconfig_service.py"
"${SRC_DIR}/__init__.py"
"${SRC_DIR}/main.py"
"${SRC_DIR}/application.py"
DESTINATION ${pkgdatadir}/configd/
)
#general
set(CPACK_PACKAGE_NAME "a-config")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "a-config-manager")
set(CPACK_PACKAGE_DESCRIPTION "a-config-manager")
# redhat
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/etc/amy
)
include(cpack)
Indeed, it confirms that ${PYTHON_EXECUTABLE} == /usr/bin/python3.4 (see 4th line below):
$ make clean ; cmake -DCMAKE_BUILD_TYPE=Release -DSHORT_VERSION=NO -DCUSTOMER=NO .. ; make -j12 ; make package
-- Found PythonInterp: /usr/bin/python3.4 (found suitable version "3.4.5", minimum required is "3.4")
-- Found PythonInterp: /usr/bin/python3.4 (found version "3.4.5")
-- ${PYTHON_EXECUTABLE} == /usr/bin/python3.4
-- Build Type: Release
-- Detected distribution: rhel fedora
-- Detected aConfigd version: 2.3.0-3030-gf7733cf659
-- Detected distribution: rhel fedora
-- Configuring done
-- Generating done
-- Build files have been written to: /local/raid0/git/amy/aConfig/build
Run CPack packaging tool...
CPack: Create package using RPM
CPack: Install projects
CPack: - Run preinstall target for: aConfigd
CPack: - Install project: aConfigd
CPack: Create package
CPackRPM:Warning: CPACK_SET_DESTDIR is set (=ON) while requesting a relocatable package (CPACK_RPM_PACKAGE_RELOCATABLE is set): this is not supported, the package won't be relocatable.
CPackRPM: Will use GENERATED spec file: /local/raid0/git/my/aConfig/build/_CPack_Packages/Linux/RPM/SPECS/a-config.spec
CPack: - package: /local/raid0/git/my/aConfig/build/a-config-2.3.0-3030-gf7733cf659.el7.my.x86_64.rpm generated.
$
However, if I uncomment the "${SRC_DIR}/systemd_client.py" line, I get the error:
Compiling /local/raid0/git/my/aConfig/build/_CPack_Packages/Linux/RPM/a-config-2.3.0-3030-gf7733cf659.el7.my.x86_64/usr/share/configd/configd/systemd_client.py ...
File "/usr/share/configd/configd/systemd_client.py", line 21
def __init__(self, systemd_proxy:Gio.DBusProxy):
^
SyntaxError: invalid syntax
Isn't def __init__(self, systemd_proxy:Gio.DBusProxy): a valid Python 3.4 syntax?
If yes, why does Cmake complains?

The root-cause occurs in the rpmbuild step.
RPM is trying to be extra-helpful, and tries to (byte-code) compile .py files it encounters.
Alas, it wrongly uses the python2 interpreter to create a file's byte-code (even though find_package(PythonInterp 3.4 REQUIRED) is declared in the CMakeLists.txt file).
The fix that worked for me was:
set(CPACK_RPM_BUILDREQUIRES python34-devel)
set(CPACK_RPM_SPEC_MORE_DEFINE "%define __python ${PYTHON_EXECUTABLE}")

When you just run "${SRC_DIR}/systemd_client.py", you're telling it to run that script the same way it would be run by the shell: by looking at the #! line and running it with whatever interpreter is specified there. Which is probably something like #! /usr/bin/python or #! /usr/bin/env python.
If you want to run your script with a particular interpreter, you have to run that interpreter and pass it the script—just as you would at the shell. I'm pretty rusty with CMake, but I'd assume you do that like this:
"${PYTHON_EXECUTABLE}" "${SRC_DIR}/amyconfig_service.py"
Alternatively, since this is your code, maybe you want to use setuptools to programmatically generate scripts for your entry-points, which means it would create a #! line for them that runs whichever Python version was used to run setup.py.

Installing megam for NLTK on Windows

I need to install megam for nltk classification routines in Python.
I followed the instruction by Milk Magic from this post:
0. Downloaded megam source from http://www.umiacs.umd.edu/~hal/megam/index.html
1. Installed cygwin with gcc, make and ocaml packages
2. changed the makefile
3. when trying to compile megam with a makefile I receive an error with the following content
make
ocamldep *.mli *.ml > .depend
ocamlc -g -custom -o megam str.cma -cclib -lcamlstr bigarray.cma -cclib -lbigarray unix.cma -cclib -lunix -I /lib/ocaml/caml fastdot_c.c fastdot.cmo intHashtbl.cmo arry.cmo util.cmo data.cmo bitvec.cmo cg.cmo wsemlm.cmo bfgs.cmo pa.cmo perceptron.cmo radapt.cmo kernelmap.cmo abffs.cmo main.cmo
sh: flexlink: command not found
File "fastdot_c.c", line 1:
Error: Error while building custom runtime system
make: *** [Makefile:101: megam] Error 2
Do you know what the problem might be?
Maybe somebody has solved the same problem recently and could help.

As the error is
sh: flexlink: command not found
you need to find the package that contains it
$ cygcheck -p flexlink
Found 5 matches for flexlink
flexdll-0.34-1 - flexdll: Creates DLLs with runtime symbol resolution (installed binaries and support files)
flexdll-0.35-1 - flexdll: Creates DLLs with runtime symbol resolution (installed binaries and support files)
flexdll-0.35-2 - flexdll: Creates DLLs with runtime symbol resolution
...
So you need to install the flexdll package.
$ cygcheck -l flexdll |grep bin
/usr/bin/flexlink

How to package a linked DLL and a pyd file into one self contained pyd file?

I am building a python module with Cython that links against a DLL file. In order to succesfully import my module I need to have the DLL in the Windows search path. Otherwise, the typical error message is:
ImportError: DLL load failed: The specified module could not be found.
Is there a way to packaged the DLL directly into the produced pyd file to make the distribution easier?
One example of this is with the OpenCV distribution, where a (huge) pyd file is distributed and is the only file needed for the Python bindings to work.

Python's packaging & deployment is still a pain point for many of us. There is just not a silver bullet. Here are several methods:
1. OpenCV build method
The method is decribed here : https://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.html#bindings-basics
OpenCV generates these wrapper functions automatically from the C++
headers using some Python scripts which are located in
modules/python/src2.
Basically it parse the header files and generate the static PyObject keywords whenever it's needed. Once the header are created appropriately, it just call python setup. Honestly, it might work, but I would not advise this method.
2. Makefiles
If you already use Makefile, just create a rule to place your lib accordinlgy. Example, from my own code :
setup.py
from distutils.core import setup, Extension
setup(name='sha1_hmac', version='1.0', \
ext_modules=[Extension('sha1_hmac',
library_dirs=['C:\MinGW\lib'],
sources= ['../tools/sha1.c','sha1_hmac.c'])])
Makefile
# The hmac generation used by the webserver is done
# using the sha1.c implementation. There is a binding needed to
# glue the C code with the python script
libsha1_hmac:
ifeq ($(OS), Windows_NT)
$(PYTHON) setup.py build --compiler=mingw32
else
$(PYTHON) setup.py install --home=$(CURDIR)
endif
.PHONY: webserver
webserver: libsha1_hmac
ifeq ($(OS), Windows_NT)
mv $(shell find build -type f -name "sha1*.pyd") $(LIB)
else
mv -f $(shell find $(LIB)/python -type f -name "sha1*.so") $(LIB)
endif
$(PYTHON) hmac_server.py
3. Modern deployement tools
There are several new tools to deploy python applications, namely wheels which seem to gain traction. I don't use it, but it look like it can ease up your bundling problem :
How can I make a Python Wheel from an existing native library?
Once it wheeled, you can install it like this : pip install some-package.whl

CMake not linking Python

Sorry if I'm duplicating a question, but I just cannot find the solution to what I'm looking for anywhere on the internet, yet I believe that this is a very simple problem.
I'm trying to extend python with some custom C++ libraries, and building my C++ libraries with CMake. I'm following the instructions on https://docs.python.org/2/extending/extending.html, but it's not compiling correctly.
When I try to build it, I get these messages:
"C:\Program Files (x86)\JetBrains\CLion 140.2310.6\bin\cmake\bin\cmake.exe" --build C:\Users\pkim2\.clion10\system\cmake\generated\76c451cd\76c451cd\Debug --target parsers -- -j 8
Linking CXX executable parsers.exe
CMakeFiles\parsers.dir/objects.a(main.cpp.obj): In function `spam_system':
C:/code/ground-trac/ground/launch/trunk/Software Support/Data Analysis Scripts/data_review_automation/parsers/main.cpp:9: undefined reference to `_imp__PyArg_ParseTuple'
C:/code/ground-trac/ground/launch/trunk/Software Support/Data Analysis Scripts/data_review_automation/parsers/main.cpp:12: undefined reference to `_imp__Py_BuildValue'
c:/mingw/bin/../lib/gcc/mingw32/4.8.1/../../../libmingw32.a(main.o):(.text.startup+0xa7): undefined reference to `WinMain#16'
collect2.exe: error: ld returned 1 exit status
CMakeFiles\parsers.dir\build.make:87: recipe for target 'parsers.exe' failed
CMakeFiles\Makefile2:59: recipe for target 'CMakeFiles/parsers.dir/all' failed
CMakeFiles\Makefile2:71: recipe for target 'CMakeFiles/parsers.dir/rule' failed
mingw32-make.exe[3]: *** [parsers.exe] Error 1
mingw32-make.exe[2]: *** [CMakeFiles/parsers.dir/all] Error 2
mingw32-make.exe[1]: *** [CMakeFiles/parsers.dir/rule] Error 2
mingw32-make.exe: *** [parsers] Error 2
Makefile:109: recipe for target 'parsers' failed
Based on this, I suspect that this is a problem with the way I'm linking things in the CMakeLists.txt file, but I have no idea how to do it properly. This is what my CMakeLists.txt looks like right now:
cmake_minimum_required(VERSION 2.8.4)
project(parsers)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
set(SOURCE_FILES main.cpp)
include_directories(C:\\Python27\\include)
link_directories(C:\\Python27\\)
target_link_libraries(python2.7)
add_executable(parsers ${SOURCE_FILES})
How in the world do I get this thing to compile correctly? I am running Windows 7 64-bit, and using CLion as my IDE.

Your first problem is that you are using target_link_libraries wrong: you should pass it the target to which to add a link and then the library you want to link in:
target_link_libraries(parsers python2.7)
Your second problem is that you are building an executable, instead of a shared library. If you want to make your extension accessible from python it needs to be a library.
add_library(parsers SHARED ${SOURCE_FILES})
But now comes the good news: your life becomes much simpler (and more portable) if you use the built in CMake module FindPythonLibs.cmake. To build a python module you would only need to do the following:
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
find_package(PythonLibs REQUIRED)
add_library(parsers SHARED ${SOURCE_FILES})
include_directories(${PYTHON_INCLUDE_DIRS})
target_link_libraries(parsers ${PYTHON_LIBRARIES})

if you use windows, try this CMakeLists.txt
cmake_minimum_required(VERSION 3.8)
project(CSample)
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g -ftest-coverage -fprofile-arcs" )
link_directories(D:/Programs/mingw/mingw64/lib/)
include_directories(D:/Programs/Python/Python37/include/)
link_libraries(D:/Programs/Python/Python37/libs/python37.lib)
# Python
add_executable(CSample main.cpp)

You are using target_link_libraries() wrong. Check the docs; you probably want something like:
add_executable(parsers ${SOURCE_FILES})
target_link_libraries(parsers python2.7)
Note that the output from CMake should already tell you that something is wrong. On my machine:
CMake Error at CMakeLists.txt:8 (target_link_libraries):
Cannot specify link libraries for target "python2.7" which is not built by
this project.