C_Python not releasing buffer memory - python

I'm writing C code for python (Python C API), and I noticed that python is not releasing the memory of the file, I'm wondering if the issue is in my code.
I want to simplify as much as passable, but I hope that no details will be missing.
The file is a binary file with buffers, first 4 bytes is the buffer size, then the buffer.
The binary file (big_file.comp):
du ~/Desktop/TEST_FILES/big_file.comp
4175416 ~/Desktop/TEST_FILES/big_file.comp
The python code (test.py):
#!/usr/bin/env python3
from struct import unpack_from
from psutil import Process
from os import getpid
import decomplib
def file_handler(file_name):
with open(file_name, 'rb') as reader:
while True:
next_4_bytes = reader.read(4)
if next_4_bytes == b'':
break
next_size, *_ = unpack_from("I", next_4_bytes)
buffer = reader.read(next_size)
yield buffer, next_size
def main():
args = _parse_args()
decompress = decomplib.Decompress()
for buf, buf_size in file_handler(args.file):
for msg in decompress.decompress_buffer(buf, buf_size):
print(msg)
if __name__ == "__main__":
pid = getpid()
ps = Process(pid)
main()
print(ps.memory_info())
Some of the C code simplified:
#include <Python.h>
#include "structmember.h"
typedef struct {
PyObject_HEAD
uint32_t arr_size;
} DecompressObject;
static int Decompress_init(DecompressObject *self, PyObject *args, PyObject *kwds){
return 0;
}
static PyObject* Decompress_handle_buffer(DecompressObject* self, PyObject* args){
uint32_t buf_size = 0;
uint8_t *buf = NULL;
// get buffer and buffer length from python function
if(!PyArg_ParseTuple(args, "y*i", &buf, &buf_size)){
PyErr_SetString(PyExc_Exception, "Failed to parse function arguments");
return NULL;
}
self->arr_size = 10;
Py_XINCREF(self);
return (PyObject *) self;
}
static PyObject* Decompress_next(DecompressObject *self, PyObject *Py_UNUSED(ignored)){
static uint32_t seq_index = 0;
if (seq_index < self->arr_size) {
seq_index++;
Py_RETURN_NONE;
}
seq_index = 0;
return NULL;
}
static void Decompress_dealloc(DecompressObject *self){
Py_TYPE(self)->tp_free((PyObject *) self);
}
static PyMethodDef Decompress_methods[] = {
{"decompress_buffer", (PyCFunction) Decompress_handle_buffer, METH_VARARGS, "Decompress a buffer to asc data."},
{NULL} /* Sentinel */
};
static PyTypeObject DecompressType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "decomplib.Decompress",
.tp_doc = "Decompress object",
.tp_basicsize = sizeof(DecompressObject),
.tp_itemsize = 0,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.tp_alloc = PyType_GenericAlloc,
.tp_new = PyType_GenericNew,
.tp_iter = PyObject_SelfIter,
.tp_init = (initproc) Decompress_init,
.tp_dealloc = (destructor) Decompress_dealloc,
.tp_iternext = (iternextfunc) Decompress_next,
.tp_methods = Decompress_methods,
};
static PyModuleDef Decompressmodule = {
PyModuleDef_HEAD_INIT,
.m_name = "decomplib",
.m_doc = "Decompress an compressed file.",
.m_size = -1,
};
PyMODINIT_FUNC PyInit_decomplib(void){
PyObject *d;
if (PyType_Ready(&DecompressType) < 0)
return NULL;
d = PyModule_Create(&Decompressmodule);
if (d == NULL)
return NULL;
Py_INCREF(&DecompressType);
if (PyModule_AddObject(d, "Decompress", (PyObject *) &DecompressType) < 0) {
Py_DECREF(&DecompressType);
Py_DECREF(d);
return NULL;
}
return d;
}
As a result, I got the following output:
./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=4349915136, vms=4412583936, shared=6270976, text=2867200, lib=0, data=4344135680, dirty=0)
While playing around I noticed that if I change in the C function Decompress_handle_buffer the call to the function PyArg_ParseTuple the second argument from "y*i" to "Si", Python do cleanup the memory...
./test.py -f ~/Desktop/TEST_CAN_OPT/big_fie.comp
None
None
None
...
None
None
None
pmem(rss=22577152, vms=84869120, shared=6361088, text=2867200, lib=0, data=16420864, dirty=0)
However, The buffer is NOT correctly read.
Any ideas?!
Extra Info:
I'm using a virtual machine (VMware Workstation 15)
OS Ubuntu 18.4
Python 3.6.9

y* does not correspond to uint8_t like you're using it. As stated in the documentation, it fills a Py_buffer struct that you're supposed to provide.
You need to actually provide a Py_buffer, and when you're done with it, you need to release the buffer with PyBuffer_Release.

Related

How does ctypes define the class for C structure with function pointer only and init the variable in python?

I'm working on the python with ctypes to call the c so file, but the c file define the structure with function pointer
// mem ==================================================================
typedef struct StdMemFunc
{
void* (*const malloc) (unsigned long size);
void (*const free) (void* ptr);
void* (*const realloc) (void* ptr, unsigned long size);
void* (*const calloc) (unsigned long count, unsigned long size);
void* (*const set) (void* ptr, int value, unsigned long num);
void* (*const copy) (void* dest, const void* src, unsigned long num);
}*StdMemFunc;
typedef struct StdLib
{
const uint32_t version;
bool (*const is_version_compatible) (uint32_t version, uint32_t func_mask);
void (*const delay) (int32_t milli_sec);
const StdMemFunc mem;
}*StdLib;
and mock the function in another file as below
void *std_malloc(unsigned long size)
{
return malloc(size);
}
void std_free(void *ptr)
{
free(ptr);
}
void *std_realloc(void *ptr, unsigned long size)
{
return realloc(ptr, size);
}
void *std_calloc(unsigned long count, unsigned long size)
{
return calloc(count, size);
}
void *std_memset(void *ptr, int value, unsigned long num)
{
return memset(ptr, value, num);
}
void *std_memcopy(void *dest, const void *src, unsigned long num)
{
return memcpy(dest, src, num);
}
struct StdMemFunc mem_func =
{
.malloc = std_malloc,
.free = std_free,
.realloc = std_realloc,
.calloc = std_calloc,
.set = std_memset,
.copy = std_memcopy
};
then the python need to call another method with std_lib as paramater, the std_lib with call mem->malloc() method in C part, so how to define the class in the python with ctypes?
I have tried the below one, but it was not work
class StdMemFunc(Structure):
_fields_ = [
("malloc", ctypes.CFUNCTYPE(c_void_p, c_ulong)),
("free", ctypes.CFUNCTYPE(None, c_void_p)),
("realloc", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_ulong)),
("calloc", ctypes.CFUNCTYPE(c_void_p, c_ulong, c_ulong)),
("set", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_int, c_ulong)),
("copy", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_ulong))
]
class StdLib(Structure):
_fields_ = [
("version", c_uint32),
("is_version_compatible", c_bool),
("delay", c_void_p),
("mem", POINTER(StdMemFunc)),
]
libc_std_lib = CDLL('/home/linus/code/galileo/mock_std_lib.so')
std_lib = StdLib()
std_lib.mem.malloc = libc_std_lib.std_malloc
libc_modbus.modbus_create_server_station.argtypes = [POINTER(ModbusNodeDef), c_int, StdLib, PlcDrvAccessor]
libc_modbus.modbus_create_server_station.restype = POINTER(ModbusStation)
libc_modbus.modbus_create_server_station(node_def, node_num, std_lib, plc_drv_accessor)
It looks like there are two problems here:
The is_version_compatible and delay fields in the StdLib struct are functions, but you are defining them as constants.
You are not instantiating all the fields in the struct, meaning that the program might be trying to dereference a null pointer, as null pointers are the default value for pointer types.
The StdLib struct definition should look something like this:
class StdLib(Structure):
_fields_ = [
("version", c_uint32),
("is_version_compatible", CFUNCTYPE(POINTER(c_bool), c_uint32, _uint32)),
("delay", CFUNCTYPE(c_void_p, c_int32)),
("mem", POINTER(StdMemFunc)),
]
For the instantiation, I would do something like this:
libc_std_lib = CDLL('/home/linus/code/galileo/mock_std_lib.so')
std_mem_func = StdMemFunc(
libc_std_lib.std_malloc,
libc_std_lib.std_free,
libc_std_lib.std_realloc,
libc_std_lib.std_calloc,
libc_std_lib.std_set,
libc_std_lib.std_copy
)
std_lib = StdLib(
1,
reference_to_is_version_compatible_func,
reference_to_delay_func,
std_mem_func
)
Of course, you need to pass the correct params/function references to StdLib. Maybe you will need to mock the is_version_compatible and delay functions as well.
Disclaimer: this is entirely untested, so I don't guarantee it will work.
The OP's code isn't quite reproducible, but I was able to get the same error message on the following line:
std_lib.mem.malloc = libc_std_lib.std_malloc
If I am following correctly, the OP wants to initialize the C structure with functions that are provided in C, but libc.std_lib.std_malloc isn't wrapped properly to do that. It is a function that wraps a C function that is callable from Python, not C.
ctypes function prototypes can be instantiated a number of ways, and the one that works is:
prototype(func_spec[, paramflags])
Returns a foreign function exported by a shared library. func_spec must be a 2-tuple
(name_or_ordinal, library). The first item is the name of the exported
function as string, or the ordinal of the exported function as small
integer. The second item is the shared library instance.
For example:
std_lib.mem.malloc = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_ulong)(('std_malloc',libc_std_lib))
Here's a working set of files:
test.cpp
#include <stdlib.h>
#include <stdint.h>
#include <memory.h>
#include <stdio.h>
#ifdef _WIN32
# define API __declspec(dllexport)
#else
# define API
#endif
extern "C" {
typedef struct StdMemFunc {
void* (*const malloc)(unsigned long size);
void (*const free)(void* ptr);
void* (*const realloc)(void* ptr, unsigned long size);
void* (*const calloc)(unsigned long count, unsigned long size);
void* (*const set)(void* ptr, int value, unsigned long num);
void* (*const copy)(void* dest, const void* src, unsigned long num);
} *StdMemFuncPtr;
typedef struct StdLib {
const uint32_t version;
bool (*const is_version_compatible)(uint32_t version, uint32_t func_mask);
void (*const delay)(int32_t milli_sec);
const StdMemFunc mem;
} *StdLibPtr;
API void* std_malloc(unsigned long size) {
return malloc(size);
}
API void std_free(void* ptr) {
free(ptr);
}
API void* std_realloc(void* ptr, unsigned long size) {
return realloc(ptr, size);
}
API void* std_calloc(unsigned long count, unsigned long size) {
return calloc(count, size);
}
API void* std_memset(void* ptr, int value, unsigned long num) {
return memset(ptr, value, num);
}
API void* std_memcopy(void* dest, const void* src, unsigned long num) {
return memcpy(dest, src, num);
}
// A couple of test functions that accepts the initialized structure
// and calls sum of the function pointers.
API char* testit(StdLib* test) {
// This is how I debugged this, by comparing the *actual*
// function pointer value to the one received from Python.
// Once they matched the code worked.
printf("%p %p\n", std_malloc, test->mem.malloc);
char* p = static_cast<char*>(test->mem.malloc(10));
test->mem.set(p, 'A', 9);
p[9] = 0;
return p;
}
API void freeit(StdLib* test, char* p) {
test->mem.free(p);
}
}
test.py
import ctypes as ct
# prototypes
MALLOC = ct.CFUNCTYPE(ct.c_void_p,ct.c_ulong)
FREE = ct.CFUNCTYPE(None,ct.c_void_p)
REALLOC = ct.CFUNCTYPE(ct.c_void_p, ct.c_void_p, ct.c_ulong)
CALLOC = ct.CFUNCTYPE(ct.c_void_p, ct.c_ulong, ct.c_ulong)
SET = ct.CFUNCTYPE(ct.c_void_p,ct.c_void_p,ct.c_int,ct.c_ulong)
COPY = ct.CFUNCTYPE(ct.c_void_p, ct.c_void_p, ct.c_ulong)
class StdMemFunc(ct.Structure):
_fields_ = [("malloc", MALLOC),
("free", FREE),
("realloc", REALLOC),
("calloc", CALLOC),
("set", SET),
("copy", COPY)]
class StdLib(ct.Structure):
_fields_ = [("version", ct.c_uint32),
# Note these two fields were function pointers as well.
# Declared correctly now.
("is_version_compatible", ct.CFUNCTYPE(ct.c_bool, ct.c_uint32, ct.c_uint32)),
("delay", ct.CFUNCTYPE(None, ct.c_int32)),
("mem", StdMemFunc)]
dll = ct.CDLL('./test')
dll.testit.argtypes = ct.POINTER(StdLib),
dll.testit.restype = ct.POINTER(ct.c_char)
dll.freeit.argtypes = ct.POINTER(StdLib), ct.c_char_p
dll.freeit.restype = None
lib = StdLib()
lib.mem.malloc = MALLOC(('std_malloc', dll))
lib.mem.realloc = REALLOC(('std_realloc', dll))
lib.mem.calloc = CALLOC(('std_calloc', dll))
lib.mem.free = FREE(('std_free', dll))
lib.mem.set = SET(('std_memset', dll))
lib.mem.copy = COPY(('std_memcopy', dll))
p = dll.testit(lib)
# One way to access the data in the returned pointer is to slice it to the known length
print(p[:10])
# If known to be null-terminated, can also cast to c_char_p, which expects
# null-terminated data, and extract the value.
print(ct.cast(p,ct.c_char_p).value)
dll.freeit(lib,p)
Output:
b'AAAAAAAAA\x00'
b'AAAAAAAAA'

Python Method Call in a C++ Code through Python C-API

I am working on my Project which implies the use of Empirical Mode Decomposition in C++ for EEG Signals. The input Data is Eigen::MatrixXd, where the rows are the Channels and the columns are the samples.
I did not found a good C++ library for EMD so I want to use a Python one (dsatools). I have downloaded the package through Pip installer from the setup.py file on Xubuntu... so it's a system package now.
the problem is that the program can't read the module.
this is the code:
std::vector <Eigen::MatrixXd> DataAquisition::EMD (Eigen::MatrixXd array, int order, int iterations, int locality) {
std::vector <Eigen::MatrixXd> IMFs;
for (int i = 0; i < array.rows(); i++) {
Eigen::MatrixXd Kanals = array.row(i);
Eigen::MatrixXd IMFs_Cpp;
Py_Initialize();
//PyRun_SimpleString("from dsatools._base._imf_decomposition import * ");
PyObject* sys_path = PySys_GetObject("path");
PyObject* ProgrammName = PyUnicode_FromString("/home/user/Schreibtisch/mne-cpp-main/applications/mne_bci/MNE-BCI-QT/dsatools-master/dsatools/_base/_imf_decomposition/_emd.py");
PyList_Append(sys_path, ProgrammName);
PyObject* pModuleString = PyUnicode_FromString ((char*)"_emd.py");
PyObject* pModule = PyImport_Import(pModuleString);
PyObject* pFunction = PyObject_GetAttrString(pModule,(char*)"emd");
//PyObject* pDict = PyModule_GetDict(pModule);
//PyObject* pFunc = PyDict_GetItemString(pDict, (char*)"emd");
if (PyCallable_Check(pFunction))
{
PyObject* Signal = Py_BuildValue("(d)",(double*)Kanals.data());
PyObject* Order = Py_BuildValue("(i)",order);
PyObject* method = Py_BuildValue("(z)",(char*)"cubic");
PyObject* max_itter = Py_BuildValue("(i)",iterations);
PyObject* args = PyTuple_Pack(4,Signal,Order,method,max_itter);
PyErr_Print();
PyObject* IMFs_Py = PyObject_CallObject(pFunction,args);
PyErr_Print();
if (PyArray_Check(IMFs_Py))
std::cout << "EMD Output is NOT Array \n";
PyArrayObject *np_ret = reinterpret_cast <PyArrayObject*> (IMFs_Py);
int Rows = PyArray_SHAPE(np_ret)[0];
int Cols = PyArray_SHAPE(np_ret)[1];
double* c_out = reinterpret_cast<double*>(PyArray_DATA(np_ret));
Eigen::MatrixXd IMFs_Cpp = Eigen::Map <Eigen::MatrixXd> (c_out,Rows,Cols);
IMFs.push_back(IMFs_Cpp);
}
else
std::cout << "Python did not call the function \n";
Py_Finalize();
}
return IMFs;}
this is how the code in Python should look like and I just want to call the emd function:

Numpy Python/C API - PyArray_SimpleNewFromData hangs

I'm figuring out the Python/C API for a more complex task. Initially, I wrote a simple example of adding two ndarrays of shape = (2,3) and type = float32.
I am able to pass two numpy arrays into c functions, read their dimensions and data and perform custom addion on data. But when I try to wrap the resulting data using PyArray_SimpleNewFromData, code hangs (returns NULL?)
To replicate the issue, create three files: mymath.c, setup.py, test.py in a folder as follows and run test.py (it runs setup.py to compile and install the module and then runs a simple test).
I'm using python in windows, inside an anaconda environment. I'm new to the Python/C API. So, any help would be much appreciated.
​
// mymath.c
#include <Python.h>
#include <stdio.h>
#include "numpy/arrayobject.h"
#include "numpy/npy_math.h"
#include <math.h>
#include <omp.h>
/*
C functions
*/
float* arr_add(float* d1, float* d2, int M, int N){
float * result = (float *) malloc(sizeof(float)*M*N);
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
result [m*N+ n] = d1[m*N+ n] + d2[m*N+ n];
return result;
}
/*
Unwrap apply and wrap pyObjects
*/
void capsule_cleanup(PyObject *capsule) {
void *memory = PyCapsule_GetPointer(capsule, NULL);
free(memory);
}
// add two 2d arrays (float32)
static PyObject *arr_add_fn(PyObject *self, PyObject *args)
{
PyArrayObject *arr1, *arr2;
if (!PyArg_ParseTuple(args, "OO", &arr1, &arr2))
return NULL;
// get data as flat list
float *d1, *d2;
d1 = (float *) arr1->data;
d2 = (float *) arr2->data;
int M, N;
M = (int)arr1->dimensions[0];
N = (int)arr1->dimensions[1];
printf("Dimensions, %d, %d \n\n", M,N);
PyObject *result, *capsule;
npy_intp dim[2];
dim[0] = M;
dim[1] = N;
float * d3 = arr_add(d1, d2, M, N);
result = PyArray_SimpleNewFromData(2, dim, NPY_FLOAT, (void *)d3);
if (result == NULL)
return NULL;
// -----------This is not executed. code hangs--------------------
for (int m=0; m<M; m++)
for (int n=0; n<N; n++)
printf("%f \n", d3[m*N+n]);
capsule = PyCapsule_New(d3, NULL, capsule_cleanup);
PyArray_SetBaseObject((PyArrayObject *) result, capsule);
return result;
}
/*
Bundle functions into module
*/
static PyMethodDef MyMethods [] ={
{"arr_add", arr_add_fn, METH_VARARGS, "Array Add two numbers"},
{NULL,NULL,0,NULL}
};
/*
Create module
*/
static struct PyModuleDef mymathmodule = {
PyModuleDef_HEAD_INIT,
"mymath", "My doc of mymath", -1, MyMethods
};
PyMODINIT_FUNC PyInit_mymath(void){
return PyModule_Create(&mymathmodule);
}
​
# setup.py
from distutils.core import setup, Extension
import numpy
module1 = Extension('mymath',
sources = ['mymath.c'],
# define_macros = [('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')],
include_dirs=[numpy.get_include()],
extra_compile_args = ['-fopenmp'],
extra_link_args = ['-lgomp'])
setup (name = 'mymath',
version = '1.0',
description = 'My math',
ext_modules = [module1])
​
# test.py
import os
os.system("python .\setup.py install")
import numpy as np
import mymath
a = np.arange(6,dtype=np.float32).reshape(2,3)
b = np.arange(6,dtype=np.float32).reshape(2,3)
c = mymath.arr_add(a,b)
print(c)

Pass Variable from Python to C

I have Python Script embedded in C which I run in a thread. I need to pass the variable 'a' from the Python-Class 'Detect Motion' to my C program continuously. (Not as a return value)
I know I could do this with a fifo or something like that, but is there a way to pass it directly to C, maybe by calling a C function?
C:
#include <Python.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <pthread.h>
pthread_t mythread;
void *ThreadProc();
PyObject *pName, *pModule, *pDict, *pFunc, *pFunc2;
int main(int argc, char *argv[])
{
py_callback = PyCFunction_New(&callback_descr, NULL);
char *script = "motion";
char *functionUse = "get_values";
Py_Initialize();
pName = PyString_FromString(script);
pModule = PyImport_Import(pName);
// pDict and pFunc are borrowed references
pDict = PyModule_GetDict(pModule);
pFunc = PyDict_GetItemString(pDict, functionUse);
// POSIX code
pthread_create( &mythread, NULL, ThreadProc, NULL);
// Random testing code
for(int i = 0; i < 10; i++)
{
printf("Printed from the main thread.\n");
sleep(1);
}
printf("Main Thread waiting for My Thread to complete...\n");
// Join and wait for the created thread to complete...
// POSIX code
pthread_join(mythread, NULL);
printf("Main thread finished gracefully.\n");
return 0;
}
void *ThreadProc()
{
if (PyCallable_Check(pFunc))
{
PyObject_CallObject(pFunc, NULL);
}
else {
PyErr_Print();
}
// Clean up
Py_DECREF(pModule);
Py_DECREF(pName);
Py_Finalize();
printf("My thread is finishing...\n");
}
Python:
import numpy as np
import picamera
import picamera.array
class DetectMotion(picamera.array.PiMotionAnalysis):
def analyse(self, a):
a = np.sqrt(
np.square(a['x'].astype(np.float)) +
np.square(a['y'].astype(np.float))
).clip(0, 255).astype(np.uint8)
# If there're more than 10 vectors with a magnitude greater
# than 60, then say we've detected motion
print a
if (a > 60).sum() > 10:
print 'Motion detected!'
def get_values():
with picamera.PiCamera() as camera:
with DetectMotion(camera) as output:
camera.resolution = (640, 480)
camera.start_preview()
camera.start_recording(
'/dev/null', format='h264', motion_output=output)
camera.wait_recording(10)
camera.stop_recording()
camera.stop_preview()

How to execute Python script from CreateProcess in C on Windows?

I have managed to get C code calling Python scripts happily on Unix using PIPES within the C code. I now need to do the same on Windows.
Essentially I would like to write scripts in different scripting languages like Python / Lua etc on Windows and be able to execute them using STDIN / STDOUT etc.
I have been looking at the "CreateProcess" call at:
http://msdn.microsoft.com/en-us/library/ms682425(VS.85).aspx
and although I can get it to work with a "child written in C", I cannot get it to call a Python script.
Below is the "parent / sender code" on my windows box:
#include<windows.h>
#include <stdio.h>
#include <stdlib.h>
#pragma comment(lib, "User32.lib")
void DisplayError(char *pszAPI);
void readFromPipe(HANDLE hPipeRead);
void createChildProcess(char *commandLine,
HANDLE hChildStdOut,
HANDLE hChildStdIn,
HANDLE hChildStdErr);
DWORD WINAPI writeToPipe(LPVOID lpvThreadParam);
HANDLE hChildProcess = NULL;
HANDLE hStdIn = NULL;
BOOL bRunThread = TRUE;
char *inputStream;
int main(int argc, char *argv[]){
HANDLE hOutputReadTmp,hOutputRead,hOutputWrite;
HANDLE hInputWriteTmp,hInputRead,hInputWrite;
HANDLE hErrorWrite;
HANDLE hThread;
DWORD ThreadId;
SECURITY_ATTRIBUTES sa;
int streamLen;
sa.nLength= sizeof(SECURITY_ATTRIBUTES);
sa.lpSecurityDescriptor = NULL;
sa.bInheritHandle = TRUE;
if (!CreatePipe(&hOutputReadTmp,&hOutputWrite,&sa,0))
return 1;
if (!DuplicateHandle(GetCurrentProcess(),hOutputWrite,
GetCurrentProcess(),&hErrorWrite,0,
TRUE,DUPLICATE_SAME_ACCESS))
return 1;
if (!CreatePipe(&hInputRead,&hInputWriteTmp,&sa,0))
return 1;
if (!DuplicateHandle(GetCurrentProcess(),hOutputReadTmp,
GetCurrentProcess(),
&hOutputRead,
0,FALSE,
DUPLICATE_SAME_ACCESS))
return 1;
if (!DuplicateHandle(GetCurrentProcess(),hInputWriteTmp,
GetCurrentProcess(),
&hInputWrite,
0,FALSE,
DUPLICATE_SAME_ACCESS))
return 1;
if (!CloseHandle(hOutputReadTmp)) return 1;;
if (!CloseHandle(hInputWriteTmp)) return 1;;
if ( (hStdIn = GetStdHandle(STD_INPUT_HANDLE)) == INVALID_HANDLE_VALUE )
return 1;
if (argc == 2){
createChildProcess(argv[1], hOutputWrite,hInputRead,hErrorWrite);
}else{
puts("No process name / input stream specified\n");
return 1;
}
if (!CloseHandle(hOutputWrite)) return 1;;
if (!CloseHandle(hInputRead )) return 1;;
if (!CloseHandle(hErrorWrite)) return 1;;
hThread = CreateThread(NULL,0,writeToPipe,
(LPVOID)hInputWrite,0,&ThreadId);
if (hThread == NULL)
return 1;;
readFromPipe(hOutputRead);
if (!CloseHandle(hStdIn))
return 1;
bRunThread = FALSE;
if (WaitForSingleObject(hThread,INFINITE) == WAIT_FAILED)
return 1;;
if (!CloseHandle(hOutputRead)) return 1;;
if (!CloseHandle(hInputWrite)) return 1;;
}
void createChildProcess(char *commandLine,
HANDLE hChildStdOut,
HANDLE hChildStdIn,
HANDLE hChildStdErr){
PROCESS_INFORMATION pi;
STARTUPINFO si;
ZeroMemory(&si,sizeof(STARTUPINFO));
si.cb = sizeof(STARTUPINFO);
si.dwFlags = STARTF_USESTDHANDLES;
si.hStdOutput = hChildStdOut;
si.hStdInput = hChildStdIn;
si.hStdError = hChildStdErr;
if (!CreateProcess(NULL,commandLine,NULL,NULL,TRUE,
NULL,NULL,NULL,&si,&pi))
hChildProcess = pi.hProcess;
if (!CloseHandle(pi.hThread)) return 1;;
}
void readFromPipe(HANDLE hPipeRead)
{
CHAR lpBuffer[256];
DWORD nBytesRead;
DWORD nCharsWritten;
while(TRUE)
{
if (!ReadFile(hPipeRead,lpBuffer,sizeof(lpBuffer),
&nBytesRead,NULL) || !nBytesRead)
{
if (GetLastError() == ERROR_BROKEN_PIPE)
break; // pipe done - normal exit path.
else
return 1; // Something bad happened.
}
if (!WriteConsole(GetStdHandle(STD_OUTPUT_HANDLE),lpBuffer,
nBytesRead,&nCharsWritten,NULL))
return 1;;
}
}
DWORD WINAPI writeToPipe(LPVOID lpvThreadParam)
{
CHAR read_buff[256];
DWORD nBytesRead,nBytesWrote;
HANDLE hPipeWrite = (HANDLE)lpvThreadParam;
while (bRunThread){
nBytesRead = 21;
strncpy(read_buff, "hello from the paren\n",21);
read_buff[nBytesRead] = '\0';
if (!WriteFile(hPipeWrite,read_buff,nBytesRead,&nBytesWrote,NULL)){
if (GetLastError() == ERROR_NO_DATA)
break; //Pipe was closed (normal exit path).
else
return 1;;
}
}
return 1;
}
Quite a bit of the above code is "hardcoded" just for testing purposes...essentially I passing some text like "hello from the paren" to be sent to a "child.exe"....
Here is the code for the child.c...a simple ECHO of what is sent to it
#include<windows.h>
#include<stdio.h>
#include<string.h>
void main (){
CHAR szInput[1024];
ZeroMemory(szInput,1024);
gets(szInput);
puts(szInput);
fflush(NULL);
}
To run the app I send "CallSubProcess.exe Child.exe" and it works 100%
Next I want to change "child.c" to be a PYTHON SCRIPT...
import sys
if __name__ == "__main__":
inStream = sys.stdin.read()
outStream = inStream
sys.stdout.write(outStream)
sys.stdout.flush()
So how can I change the CreateProcess call to execute this script?
if (!CreateProcess("C:\\Python26\\python.exe", "echo.py",NULL, NULL,FALSE, 0,NULL,NULL,&si,&pi)){
But it never works.
Any ideas how I can get this to work? Any help will be greatly appreciated.
My application posts a string to a python script, and the python script posts the string back to the c
application. It works well.
//c code
#pragma comment(lib, "json_vc71_libmtd.lib")
#include <windows.h>
#include <iostream>
#include <io.h>
#include "./json/json.h"
using namespace std;
DWORD WINAPI threadproc(PVOID pParam);
HANDLE hRead, hWrite, hRead1, hWrite1;
int main()
{
SECURITY_ATTRIBUTES sa;
sa.nLength = sizeof(SECURITY_ATTRIBUTES);
sa.lpSecurityDescriptor = NULL;
sa.bInheritHandle = TRUE;
if (!CreatePipe(&hRead, &hWrite, &sa, 0)){
::MessageBox(NULL, L"can't create pipe", L"error", MB_OK);
return -1;
}
if (!CreatePipe(&hRead1, &hWrite1, &sa, 0)){
::MessageBox(NULL, L"can't create pipe1", L"error", MB_OK);
return -1;
}
STARTUPINFO si;
PROCESS_INFORMATION pi;
GetStartupInfo(&si);
si.cb = sizeof(STARTUPINFO);
si.hStdError = hWrite;
si.hStdOutput = hWrite;
si.hStdInput = hRead1;
si.wShowWindow = SW_SHOW;
si.dwFlags = STARTF_USESHOWWINDOW | STARTF_USESTDHANDLES;
WCHAR szCmdLine[] = L"\"D:\\tools\\python\\python.exe\" D:\\code\\test\\pipeCallCore\\pipeCallCore\\json_wraper.py";
if (!CreateProcess(NULL, szCmdLine, NULL, NULL, TRUE, CREATE_NEW_CONSOLE, NULL, NULL, &si, &pi)){
::MessageBox(NULL, L"can't create process", L"error", MB_OK);
return -1;
}
CloseHandle(hWrite);
CloseHandle(hRead1);
const int cBufferSize = 4096;
char buffer[cBufferSize] = {0};
DWORD bytes;
int i = 0;
while (true){
cout << "come !" << endl;
ZeroMemory(buffer, sizeof(buffer));
sprintf(buffer, "{\"write\":%d}\n", i ++);
if (NULL == WriteFile(hWrite1, buffer, strlen(buffer), &bytes, NULL)){
::MessageBox(NULL, L"write file failed!", L"error", MB_OK);
break;
}
ZeroMemory(buffer, sizeof(buffer));
if (NULL == ReadFile(hRead, buffer, cBufferSize - 1, &bytes, NULL)){
::MessageBox(NULL, L"readfile failed", L"error", MB_OK);
return -1;
}
cout <<"yes " << buffer << endl;
Sleep(2000);
}
return 0;
}
//python code
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
while True:
try:
s = sys.stdin.readline()
sys.stdout.write(s)
sys.stdout.flush()
except EOFError, KeyboardInterrupt:
break
Maybe
if (!CreateProcess("C:\\Python26\\python.exe",
"echo.py 'hello from parent'",
NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi)) {
CreateProcess is kind of tricky to use.
From the MSDN documentation:
If both lpApplicationName and lpCommandLine are non-NULL, ... lpApplicationName specifies the module to execute, and ... lpCommandLine specifies the command line.... Console processes written in C can use the argc and argv arguments to parse the command line. Because argv[0] is the module name, C programmers generally repeat the module name as the first token in the command line.
To avoid the weirdness, I recommend always passing NULL for the first argument and to pass the full command-line as the second:
CreateProcess(NULL, "\"C:\\Python26\\python.exe\" echo.py", ...);

Categories