I'm testing a scenario that when C++ set a function pointer to a python class variable, and then use PyObject_CallMethod to run another python method, which contain that class variable.
whole process would like this.
(1). PyCFunction_NewEx() make a py function -> (2). PyDict_SetItemString() assign to class variable under __dict__ -> (3). PyObject_CallMethod() call python method witch contain (1).
When I put all the code inside main() function (whitout void setCallback() and all code inside void setCallback() were placed in main()), It runs perfectly fine. However, after I put some code into a function, sometimes get seg fault, sometimes doesn't call function pointer in python and sometimes get correct answer.
How do I resolve this problem?
C++ Code: main.cpp
#include <python3.7/Python.h>
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <python3.7/methodobject.h>
// func ptr.
PyObject *myCallFunc(PyObject *self,PyObject *args) {
printf(" aaaaaaaaaaaaaaaaaaaaaaa\n");
return NULL;
}
// def func ptr
typedef PyObject *(*PyCallFunc)(PyObject *self,PyObject *arg);
// set func ptr into python member var
void setCallback(PyObject *ClassObj){
PyCallFunc pyCallFunc = myCallFunc;
PyMethodDef methd = {"methd",pyCallFunc,METH_VARARGS,"py call func"};
PyObject *fName = PyUnicode_FromString(methd.ml_name);
if(fName == NULL) {
printf(" fName\n");
exit(0);
}
PyObject *pyRunFunc = PyCFunction_NewEx(&methd,NULL,fName);
if(pyRunFunc == NULL){
printf(" can not create py function. exit.");
exit(0);
}
Py_DECREF(fName);
PyObject* classAttrDict = PyObject_GetAttrString(ClassObj, "__dict__"); // extract instance Dictionary.
if(classAttrDict == NULL) {
printf(" classAttrDict\n");
exit(0);
}
int pRetSetCurrPrice = PyDict_SetItemString(classAttrDict, "callFunc", pyRunFunc);
if(pRetSetCurrPrice != 0){
printf(" set error. exit.");
exit(0);
}
}
int main(int argc,char **argv){
Py_SetProgramName((wchar_t *)argv[0]);
void *pyMem = PyMem_Malloc(sizeof(wchar_t*)*argc);
wchar_t** _argv = (wchar_t**)&pyMem;
for (int i=0; i<argc; i++) {
wchar_t* arg = Py_DecodeLocale(argv[i], NULL);
_argv[i] = arg;
}
Py_Initialize();
PySys_SetArgv(argc, _argv);
PyObject* programName = PyUnicode_FromString("test");
if(programName == NULL) {
printf(" programName\n");
exit(0);
}
PyObject* pCustomFunc = PyImport_Import(programName); // import test
Py_DECREF(programName);
if(pCustomFunc == NULL) {
printf(" pCustomFunc\n");
exit(0);
}
PyObject* pClass = PyObject_GetAttrString(pCustomFunc, "Test"); // pClass = test.Test
if(pClass == NULL) {
printf(" pClass\n");
exit(0);
}
PyObject* pNewInstance = PyObject_CallObject(pClass,NULL); // pNewInstance = test.Test()
if(pNewInstance == NULL) {
printf(" pNewInstance\n");
exit(0);
}
setCallback(pNewInstance);
PyObject* pCallRet = PyObject_CallMethod(pNewInstance, "runCustomFunc",NULL); // pCallRet = pNewInstance.callFunc()
if(pCallRet == NULL) {
printf(" pCallRet\n");
//exit(0);
}
sleep(2);
printf(" \n\nend\n\n");
Py_Finalize();
return 0;
}
Python code: test.py
import sys
def dummyFunc():
pass
class Test:
def __init__(self):
self.aaa = 0
self.callFunc = dummyFunc
def runCustomFunc(self):
print(" print from python.")
print(" ref count of self.callFunc 1 is %d" %(sys.getrefcount(self.callFunc)))
self.callFunc()
print(" ref count of self.callFunc 2 is %d" %(sys.getrefcount(self.callFunc)))
return 1
cmake for this test project: CMakeLists.txt
# set cmake and compiler.
cmake_minimum_required(VERSION 3.12...3.15)
set(CMAKE_CXX_FLAGS -std=c++17)
# set variable
set(CMAKE_POSITION_INDEPENDENT_CODE ON) # test if this can resolve the problem
set(THREADS_PREFER_PTHREAD_FLAG ON)
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release)
endif()
set(CMAKE_CXX_FLAGS "-Wall -Wextra") # test if optimize cause the problem
set(CMAKE_CXX_FLAGS_DEBUG "-g") # test if optimize cause the problem
set(CMAKE_CXX_FLAGS_RELEASE "-O0") # test if optimize cause the problem
set(LINK_LIB "/usr/local/lib")
set(PYTHON3_LINKER "-lpython3.7")
#set(PTHREAD "-lpthread")
set(PYTHON3_HEADER "/usr/include/python3.7")
set(PYTHON3_LIB "/usr/lib/python3.7/config-3.7m-x86_64-linux-gnu")
set(CPP_FILE_LIST "main.cpp")
include_directories( ${PYTHON3_HEADER})
link_directories( ${PYTHON3_LIB} ${LINK_LIB})
add_executable(pyEmbedFunPtrTest ${CPP_FILE_LIST})
target_link_libraries(pyEmbedFunPtrTest ${PYTHON3_LINKER})
find_package(Threads REQUIRED)
target_link_libraries(pyEmbedFunPtrTest Threads::Threads)
#target_compile_options(pyEmbedFunPtrTest PUBLIC "-pthread")
It could be because the PyMethodDef is created on the stack of the setCallback
You can verify it in the source code of cpython here.
the PyMethodDef is not copied, it is referenced instead.
Related
I have a linked list code:
#include <iostream>
class Node
{
private:
/* data */
public:
int value;
Node* next;
Node(int value){
this->value = value;
this->next = nullptr;
}
};
class LinkedList
{
private:
/* data */
public:
Node *start;
LinkedList(){
this->start = nullptr;
}
void insert(int value){
Node *node = new Node(value);
if (this->start == nullptr)
{
this->start = node;
}else
{
Node* temp = this->start;
while (temp->next != nullptr)
{
temp = temp->next;
}
temp->next = node;
}
}
void print(){
Node* temp = this->start;
while (temp != nullptr)
{
std::cout<<temp->value<<std::endl;
temp = temp->next;
}
}
void __iter__(){
Node* node = this->start;
while (node)
{
yield node;
node = node->next;
}
}
};
int main(int argc, char const *argv[])
{
LinkedList listed;
listed.insert(4);
listed.insert(7);
listed.insert(9);
listed.insert(6);
listed.print();
return 0;
}
As you can see I have __iter__ method in my LinkedList class, but I have found that yield is not define to use in C++, so my console just shows:
info.cpp: In member function 'void LinkedList::__iter__()':
info.cpp:59:13: error: 'yield' was not declared in this scope
59 | yield node;
| ^~~~~
I am taking as reference the next piece of python code:
def __iter__(self):
node = self.start
while node:
yield node
node = node.next
I hope you can help me to solve that, thanks.
Take a look at this source: https://en.cppreference.com/w/cpp/iterator/iterator
It says that
std::iterator is the base class provided to simplify definitions of the required types for iterators.
Furthermore:
This source https://cplusplus.com/reference/iterator/iterator/
gives you the following example:
// std::iterator example
#include <iostream> // std::cout
#include <iterator> // std::iterator, std::input_iterator_tag
class MyIterator : public std::iterator<std::input_iterator_tag, int>
{
int* p;
public:
MyIterator(int* x) :p(x) {}
MyIterator(const MyIterator& mit) : p(mit.p) {}
MyIterator& operator++() {++p;return *this;}
MyIterator operator++(int) {MyIterator tmp(*this); operator++(); return tmp;}
bool operator==(const MyIterator& rhs) const {return p==rhs.p;}
bool operator!=(const MyIterator& rhs) const {return p!=rhs.p;}
int& operator*() {return *p;}
};
int main () {
int numbers[]={10,20,30,40,50};
MyIterator from(numbers);
MyIterator until(numbers+5);
for (MyIterator it=from; it!=until; it++)
std::cout << *it << ' ';
std::cout << '\n';
return 0;
}
It includes the iterator via #include <iterator>, defines its operations as public and then inside the main function effectively uses it. You will need to apply something similar, but you will need to step between pointers for your operator++ methods.
C PART
#include <stdlib.h>
#include <iostream>
#include "Python.h"
using namespace std;
char* PyCall(const char* a, const char* b, const char* c) {
Py_Initialize();
if (!Py_IsInitialized())
{
}
PyRun_SimpleString("import sys");
PyRun_SimpleString("sys.path.append('./')");
PyObject* pFunc1 = NULL;
PyObject* pFunc2 = NULL;
PyObject* pFunc3 = NULL;
PyObject* pModule = PyImport_ImportModule(a);
if (pModule == NULL)
{
cout << "notfind";
}
pFunc3 = PyObject_GetAttrString(pModule, b);
PyObject* args3 = PyTuple_New(1);
PyObject* args2 = PyBytes_FromString(c);
PyTuple_SetItem(args3, 0, args2);
PyObject* pRet = PyObject_CallObject(pFunc3, args3);
char* result = NULL;
if (pRet)
{
result = PyBytes_AsString(pRet);
}
return result;
}
int main()
{
char* res = PyCall("mytest", "codetest", "{'title':'Task Manager'}");
cout << res;
}
Python Part
def codetest(title):
import win32gui
import win32api
import json
dic = json.loads(title)
a = win32gui.FindWindow(None,dic["title"])
return str(a)
The basic Python library was imported successfully, but a runtime error occurred
enter image description here
Exception thrown at 0x00007ff680271103 (in pycode. Exe): 0xc0000005: an access violation occurred while reading location 0x000000000000000.
This problem has been resolved. The problem of the return value during the call resulted in cout error, and the parameter passed in python is bytes that needs to be decoded and transcoded. I didn’t read the C-API documentation carefully.
I am writing a simple extension module using the C API. Here is a test example:
#include <Python.h>
#include <structmember.h>
typedef struct {
PyObject_HEAD
int i;
} MyObject;
static PyMemberDef my_members[] = {
{"i", T_INT, offsetof(MyObject, i), READONLY, "Some integer"},
{NULL} /* Sentinel */
};
static int MyType_init(MyObject *self, PyObject *args, PyObject *kwds)
{
char *keywords[] = {"i", NULL};
int i = 0;
if(!PyArg_ParseTupleAndKeywords(args, kwds, "|i", keywords, &i)) {
return -1;
}
self->i = i;
return 0;
}
static PyTypeObject MyType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "_my.MyType",
.tp_doc = "Pointless placeholder class",
.tp_basicsize = sizeof(MyObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.tp_init = (initproc)MyType_init,
.tp_members = my_members,
};
static PyModuleDef my_module = {
PyModuleDef_HEAD_INIT,
.m_name = "_my",
.m_doc = "My module is undocumented!",
.m_size = -1,
};
/* Global entry point */
PyMODINIT_FUNC PyInit__my(void)
{
PyObject *m = NULL;
PyExc_MyException = PyErr_NewExceptionWithDoc("_my.MyException",
"Indicates that something went horribly wrong.",
NULL, NULL);
if(PyExc_MyException == NULL) {
return NULL;
}
if(PyType_Ready(&MyType) < 0) {
goto err;
}
if((m = PyModule_Create(&my_module)) == NULL) {
goto err;
}
Py_INCREF(&MyType);
PyModule_AddObject(m, "MyType", (PyObject *)&MyType);
PyModule_AddObject(m, "MyException", (PyObject *)PyExc_MyException);
return m;
err:
Py_CLEAR(PyExc_MyException);
return NULL;
}
This module contains a class MyType and an exception MyException. The purpose of this module is to provide the base functionality for some additional code written in Python. I would like to be able to do something like:
my.py
from _my import *
class MyType(MyType):
def __init__(i=0):
raise MyException
Obviously this example is highly contrived. I just want to illustrate how I would like to use the names in my extension. Right now the import is working fine according to this rule:
If __all__ is not defined, the set of public names includes all names found in the module’s namespace which do not begin with an underscore character ('_').
However, I would like to have a finer level of control. So far, all I have been able to come up with is manually adding an __all__ attribute to the module:
PyObject *all = NULL;
...
all = Py_BuildValue("[s, s]", "MyType", "MyException");
if(all == NULL) {
goto err;
}
PyModule_AddObject(m, "__all__", all);
...
err:
...
Py_CLEAR(all);
This seems a bit clunky. Is there a function in the C API for defining an internal equivalent to __all__ for a module?
Part of the reason that I think there may be an internal equivalent is that __all__ is after all a dunder attribute.
My problem is as follows:
I would like to call a C function from my Python file and return a value back to that Python file.
I have tried the following method of using embedded C in Python (the following code is the C code called "mod1.c). I am using Python3.4 so the format follows that given in the documentation guidelines. The problem comes when I call my setup file (second code below).
#include
#include "sum.h"
static PyObject*
mod_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
/* DECLARATION OF METHODS */
static PyMethodDef ModMethods[] = {
{"sum", mod_sum, METH_VARARGS, "Descirption"}, // {"methName", modName_methName, METH_VARARGS, "Description.."}, modName is name of module and methName is name of method
{NULL,NULL,0,NULL}
};
// Module Definition Structure
static struct PyModuleDef summodule = {
PyModuleDef_HEAD_INIT,
"sum",
NULL,
-1,
ModMethods
};
/* INITIALIZATION FUNCTION */
PyMODINIT_FUNC initmod(void)
{
PyObject *m;
m = PyModule_Create(&summodule);
if (m == NULL)
return m;
}
Setup.py
from distutils.core import setup, Extension
setup(name='buildsum', version='1.0', \
ext_modules=[Extension('buildsum', ['mod1.c'])])
The result that I get when I compile my code using gcc is the following error: Cannot export PyInit_buildsum: symbol not defined
I would greatly appreciate any insight or help on this problem, or any suggestion in how to call C from Python. Thank you!
---------------------------------------EDIT ---------------------------------
Thank you for the comments:
I have tried the following now:
static PyObject*
PyInit_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
For the first function; however, I still get the same error of PyInit_sum: symbol not defined
The working code from above in case anyone runs into the same error: the answer from #dclarke is correct. The initialization function in python 3 must have PyInit_(name) as its name.
#include <Python.h>
#include "sum.h"
static PyObject* mod_sum(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args,"ii",&a,&b))
return NULL;
s = sum(a,b);
return Py_BuildValue("i",s);
}
/* DECLARATION OF METHODS*/
static PyMethodDef ModMethods[] = {
{"modsum", mod_sum, METH_VARARGS, "Descirption"},
{NULL,NULL,0,NULL}
};
// Module Definition Structure
static struct PyModuleDef summodule = {
PyModuleDef_HEAD_INIT,"modsum", NULL, -1, ModMethods
};
/* INITIALIZATION FUNCTION*/
PyMODINIT_FUNC PyInit_sum(void)
{
PyObject *m;
m = PyModule_Create(&summodule);
return m;
}
I have written a program in c using libcurl to load url and send the return value to Python (I am passing 2 integer value from Python to C. i am yet to enhance the code, currently trying the logic and variable accessibility between Python and C.). I am able to compile the program successfully. When i load the module in Python i am getting error saying "undefined symbol: curl_easy_getinfo". Please let me know how to fix the issue.
Code:
#include <Python.h>
#include <stdio.h>
#include <time.h>
#include <stdio.h>
#include <pthread.h>
#include <curl/curl.h>
#define NUMT 4
/*
List of URLs to fetch.
If you intend to use a SSL-based protocol here you MUST setup the OpenSSL
callback functions as described here:
http://www.openssl.org/docs/crypto/threads.html#DESCRIPTION
*/
const char * const urls[NUMT]= {
"http://www.google.com",
"http://www.yahoo.com/",
"http://www.haxx.se/done.html",
"http://www.haxx.se/"
};
#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL 3
struct myprogress {
double lastruntime;
curl_off_t totdnld;
void *url;
CURL *curl;
};
static PyObject *foo1_add(PyObject *self, PyObject *args)
{
int a;
int b;
int s;
if (!PyArg_ParseTuple(args, "ii", &a, &b))
{
return NULL;
}
s = sum (a, b);
return Py_BuildValue("i", s);
// return Py_BuildValue("i", a + b);
}
static PyMethodDef foo1_methods[] = {
{ "add", (PyCFunction)foo1_add, METH_VARARGS, NULL },
{ NULL, NULL, 0, NULL }
};
PyMODINIT_FUNC initfoo1()
{
Py_InitModule3("foo1", foo1_methods, "My first extension module.");
}
int sum(int x, int y) {
int z;
z = x + y;
z = geturl (x, y);
return (z);
}
/* this is how the CURLOPT_XFERINFOFUNCTION callback works */
#ifdef 0
static int xferinfo(void *p,
curl_off_t dltotal, curl_off_t dlnow,
curl_off_t ultotal, curl_off_t ulnow)
{
struct myprogress *myp = (struct myprogress *)p;
CURL *curl = myp->curl;
double curtime = 0;
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &curtime);
/* under certain circumstances it may be desirable for certain functionality
to only run every N seconds, in order to do this the transaction time can
be used */
if((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) {
myp->lastruntime = curtime;
fprintf(stderr, "TOTAL TIME: %f \r\n", curtime);
}
if (dlnow > 0) {
fprintf(stderr, "UP: %" CURL_FORMAT_CURL_OFF_T " of %" CURL_FORMAT_CURL_OFF_T
" DOWN: %" CURL_FORMAT_CURL_OFF_T " of %" CURL_FORMAT_CURL_OFF_T
"\r\n",
ulnow, ultotal, dlnow, dltotal);
}
myp->totdnld = myp->totdnld + dlnow;
if (dlnow > 0) {
fprintf(stderr, "TOTAL Download: %" CURL_FORMAT_CURL_OFF_T " url is: %s \r\n", myp->totdnld, myp->url);
}
// if(dlnow > STOP_DOWNLOAD_AFTER_THIS_MANY_BYTES)
// return 1;
return 0;
}
#endif
/* for libcurl older than 7.32.0 (CURLOPT_PROGRESSFUNCTION) */
static int older_progress(void *p,
double dltotal, double dlnow,
double ultotal, double ulnow)
{
return xferinfo(p,
(curl_off_t)dltotal,
(curl_off_t)dlnow,
(curl_off_t)ultotal,
(curl_off_t)ulnow);
}
static void *pull_one_url(void *url)
{
CURL *curl;
CURLcode res = CURLE_OK;
struct myprogress prog;
curl = curl_easy_init();
if(curl) {
prog.lastruntime = 0;
prog.curl = curl;
prog.url = url;
prog.totdnld = (curl_off_t) 0;
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, older_progress);
/* pass the struct pointer into the progress function */
curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &prog);
#ifdef 0
#if LIBCURL_VERSION_NUM >= 0x072000
/* xferinfo was introduced in 7.32.0, no earlier libcurl versions will
compile as they won't have the symbols around.
If built with a newer libcurl, but running with an older libcurl:
curl_easy_setopt() will fail in run-time trying to set the new
callback, making the older callback get used.
New libcurls will prefer the new callback and instead use that one even
if both callbacks are set. */
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, xferinfo);
/* pass the struct pointer into the xferinfo function, note that this is
an alias to CURLOPT_PROGRESSDATA */
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &prog);
#endif
#endif
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
res = curl_easy_perform(curl);
if(res != CURLE_OK)
fprintf(stderr, "%s\n", curl_easy_strerror(res));
curl_easy_cleanup(curl);
}
return NULL;
}
/*
int pthread_create(pthread_t *new_thread_ID,
const pthread_attr_t *attr,
void * (*start_func)(void *), void *arg);
*/
int geturl(int x, int y)
{
pthread_t tid[NUMT];
int i;
int error;
/* Must initialize libcurl before any threads are started */
curl_global_init(CURL_GLOBAL_ALL);
for(i=0; i< NUMT; i++) {
error = pthread_create(&tid[i],
NULL, /* default attributes please */
pull_one_url,
(void *)urls[i]);
if(0 != error)
fprintf(stderr, "Couldn't run thread number %d, errno %d\n", i, error);
else
fprintf(stderr, "Thread %d, gets %s\n", i, urls[i]);
}
/* now wait for all threads to terminate */
for(i=0; i< NUMT; i++) {
error = pthread_join(tid[i], NULL);
fprintf(stderr, "Thread %d terminated\n", i);
}
return (x * y);
}
Command used for compilation:
gcc -lcurl -lpthread -shared -I/usr/include/python2.7 -fPIC sample.c –o add.so
Error:
>>> import foo1
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ImportError: ./foo1.so: undefined symbol: curl_easy_perform
>>>
Try moving -lcurl and -lpthread to after sample.c in your compilation command. The linker resolves symbols in left-to-right order, so references from sample.c (e.g., curl_easy_getinfo) will be resolved from libraries specified after it.
It's better to use -pthread than -lpthread by the way. It sets preprocessor flags to make some functions reentrant for example.