I created a kernel module and I want to communicate using a /proc file between the module and a script in python.
I am using the Ubuntu 22.04 kernel version v5.15. I tried to create the /proc file in my module below:
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kdev_t.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/slab.h> // kmalloc()
#include <linux/uaccess.h> // copy_to/from_user()
#include <linux/proc_fs.h>
static struct proc_dir_entry *parent;
char etx_array[] = "hello how are you?";
int len = sizeof(etx_array) / sizeof(char);
static int open_proc(struct inode *inode, struct file *file)
{
pr_info("proc file opend.....\t");
return 0;
}
/*
* This function will be called when we close the procfs file
*/
static int release_proc(struct inode *inode, struct file *file)
{
pr_info("proc file released.....\n");
return 0;
}
/*
* This function will be called when we read the procfs file
*/
static ssize_t read_proc(struct file *filp, char __user *buffer, size_t length, loff_t * offset)
{
pr_info("proc file read.....\n");
if (copy_to_user(buffer, etx_array, len))
pr_err("Data Send : Err!\n");
return length;;
}
/*
* This function will be called when we write the procfs file
*/
static ssize_t write_proc(struct file *filp, const char *buff, size_t len, loff_t * off)
{
pr_info("proc file wrote.....\n");
if (copy_from_user(etx_array, buff, len))
pr_err("Data Write : Err!\n");
return len;
}
static struct proc_ops proc_fops = {
.proc_open = open_proc,
.proc_read = read_proc,
.proc_write = write_proc,
.proc_release = release_proc,
};
static int etx_driver_init(void)
{
proc_create("etx_proc", 0666, parent, &proc_fops);
return 0;
}
static void etx_driver_exit(void)
{
proc_remove(parent);
}
module_init(etx_driver_init);
module_exit(etx_driver_exit);
MODULE_LICENSE("GPL");
And if I try to use /proc file by python3 in user space like this:
import os
pf = open("/proc/etx_proc","r")
print(pf.read())
So I get this (running the python program):
The problem here is that pf.read() in Python reads the entire file. How does the kernel know when the content of your proc file is over? When you return 0 from your .proc_read function. Since you are always copying etx_array out and you are always returning length, this means that to any application that tries to read the file (and also to the kernel itself) it will look like the file has infinite length. Python will keep reading and reading until it eventually runs out of memory and gets killed by the kernel OOM killer (or until your system freezes).
Always returning the requested size (length) is wrong as it does not match the actual length of the data copied to userspace. You have to check the requested size, copy no more than that and also no more than your buffer size (len), then return a meaningful value. You can keep track of how much you read with the loff_t * pointer that gets passed to your functions.
The same reasoning goes for your .proc_write function: it doesn't make sense to always return len. Check the requested size, cap it as needed, then perform the operation and return a meaningful value.
Copying data to/from userspace is a lot trickier than you might think, and there are a lot of pitfalls, you need to check everything that could possibly go wrong, specially when dealing with sizes. Here's an example of how this would work:
// Side note: this is the correct "len" you want.
size_t len = sizeof(etx_array) / sizeof(char) - 1;
// ...
static ssize_t read_proc(struct file *filp, char __user *buf, size_t size, loff_t *off)
{
loff_t offset = *off;
size_t remaining;
pr_info("proc file read\n");
if (offset < 0)
return -EINVAL;
if (offset >= len || size == 0)
return 0;
if (size > len - offset)
size = len - offset;
remaining = copy_to_user(buf, etx_array + offset, size);
if (remaining == size) {
pr_err("copy_to_user failed\n");
return -EFAULT;
}
size -= remaining;
*off = offset + size;
return size;
}
static ssize_t write_proc(struct file *filp, const char *buf, size_t size, loff_t *off)
{
loff_t offset = *off;
size_t remaining;
pr_info("proc file write\n");
if (offset < 0)
return -EINVAL;
if (offset >= len || size == 0)
return 0;
if (size > len - offset)
size = len - offset;
remaining = copy_from_user(etx_array + offset, buf, size);
if (remaining == size) {
pr_err("copy_from_user failed\n");
return -EFAULT;
}
size -= remaining;
*off = offset + size;
return size;
}
Check out this other answer of mine talking about simple_read_from_buffer() / simple_write_to_buffer(), which are simpler kernel interfaces that accomplish exactly the same as the above code for you automatically and (most importantly) correctly.
I'm working on the python with ctypes to call the c so file, but the c file define the structure with function pointer
// mem ==================================================================
typedef struct StdMemFunc
{
void* (*const malloc) (unsigned long size);
void (*const free) (void* ptr);
void* (*const realloc) (void* ptr, unsigned long size);
void* (*const calloc) (unsigned long count, unsigned long size);
void* (*const set) (void* ptr, int value, unsigned long num);
void* (*const copy) (void* dest, const void* src, unsigned long num);
}*StdMemFunc;
typedef struct StdLib
{
const uint32_t version;
bool (*const is_version_compatible) (uint32_t version, uint32_t func_mask);
void (*const delay) (int32_t milli_sec);
const StdMemFunc mem;
}*StdLib;
and mock the function in another file as below
void *std_malloc(unsigned long size)
{
return malloc(size);
}
void std_free(void *ptr)
{
free(ptr);
}
void *std_realloc(void *ptr, unsigned long size)
{
return realloc(ptr, size);
}
void *std_calloc(unsigned long count, unsigned long size)
{
return calloc(count, size);
}
void *std_memset(void *ptr, int value, unsigned long num)
{
return memset(ptr, value, num);
}
void *std_memcopy(void *dest, const void *src, unsigned long num)
{
return memcpy(dest, src, num);
}
struct StdMemFunc mem_func =
{
.malloc = std_malloc,
.free = std_free,
.realloc = std_realloc,
.calloc = std_calloc,
.set = std_memset,
.copy = std_memcopy
};
then the python need to call another method with std_lib as paramater, the std_lib with call mem->malloc() method in C part, so how to define the class in the python with ctypes?
I have tried the below one, but it was not work
class StdMemFunc(Structure):
_fields_ = [
("malloc", ctypes.CFUNCTYPE(c_void_p, c_ulong)),
("free", ctypes.CFUNCTYPE(None, c_void_p)),
("realloc", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_ulong)),
("calloc", ctypes.CFUNCTYPE(c_void_p, c_ulong, c_ulong)),
("set", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_int, c_ulong)),
("copy", ctypes.CFUNCTYPE(c_void_p, c_void_p, c_ulong))
]
class StdLib(Structure):
_fields_ = [
("version", c_uint32),
("is_version_compatible", c_bool),
("delay", c_void_p),
("mem", POINTER(StdMemFunc)),
]
libc_std_lib = CDLL('/home/linus/code/galileo/mock_std_lib.so')
std_lib = StdLib()
std_lib.mem.malloc = libc_std_lib.std_malloc
libc_modbus.modbus_create_server_station.argtypes = [POINTER(ModbusNodeDef), c_int, StdLib, PlcDrvAccessor]
libc_modbus.modbus_create_server_station.restype = POINTER(ModbusStation)
libc_modbus.modbus_create_server_station(node_def, node_num, std_lib, plc_drv_accessor)
It looks like there are two problems here:
The is_version_compatible and delay fields in the StdLib struct are functions, but you are defining them as constants.
You are not instantiating all the fields in the struct, meaning that the program might be trying to dereference a null pointer, as null pointers are the default value for pointer types.
The StdLib struct definition should look something like this:
class StdLib(Structure):
_fields_ = [
("version", c_uint32),
("is_version_compatible", CFUNCTYPE(POINTER(c_bool), c_uint32, _uint32)),
("delay", CFUNCTYPE(c_void_p, c_int32)),
("mem", POINTER(StdMemFunc)),
]
For the instantiation, I would do something like this:
libc_std_lib = CDLL('/home/linus/code/galileo/mock_std_lib.so')
std_mem_func = StdMemFunc(
libc_std_lib.std_malloc,
libc_std_lib.std_free,
libc_std_lib.std_realloc,
libc_std_lib.std_calloc,
libc_std_lib.std_set,
libc_std_lib.std_copy
)
std_lib = StdLib(
1,
reference_to_is_version_compatible_func,
reference_to_delay_func,
std_mem_func
)
Of course, you need to pass the correct params/function references to StdLib. Maybe you will need to mock the is_version_compatible and delay functions as well.
Disclaimer: this is entirely untested, so I don't guarantee it will work.
The OP's code isn't quite reproducible, but I was able to get the same error message on the following line:
std_lib.mem.malloc = libc_std_lib.std_malloc
If I am following correctly, the OP wants to initialize the C structure with functions that are provided in C, but libc.std_lib.std_malloc isn't wrapped properly to do that. It is a function that wraps a C function that is callable from Python, not C.
ctypes function prototypes can be instantiated a number of ways, and the one that works is:
prototype(func_spec[, paramflags])
Returns a foreign function exported by a shared library. func_spec must be a 2-tuple
(name_or_ordinal, library). The first item is the name of the exported
function as string, or the ordinal of the exported function as small
integer. The second item is the shared library instance.
For example:
std_lib.mem.malloc = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_ulong)(('std_malloc',libc_std_lib))
Here's a working set of files:
test.cpp
#include <stdlib.h>
#include <stdint.h>
#include <memory.h>
#include <stdio.h>
#ifdef _WIN32
# define API __declspec(dllexport)
#else
# define API
#endif
extern "C" {
typedef struct StdMemFunc {
void* (*const malloc)(unsigned long size);
void (*const free)(void* ptr);
void* (*const realloc)(void* ptr, unsigned long size);
void* (*const calloc)(unsigned long count, unsigned long size);
void* (*const set)(void* ptr, int value, unsigned long num);
void* (*const copy)(void* dest, const void* src, unsigned long num);
} *StdMemFuncPtr;
typedef struct StdLib {
const uint32_t version;
bool (*const is_version_compatible)(uint32_t version, uint32_t func_mask);
void (*const delay)(int32_t milli_sec);
const StdMemFunc mem;
} *StdLibPtr;
API void* std_malloc(unsigned long size) {
return malloc(size);
}
API void std_free(void* ptr) {
free(ptr);
}
API void* std_realloc(void* ptr, unsigned long size) {
return realloc(ptr, size);
}
API void* std_calloc(unsigned long count, unsigned long size) {
return calloc(count, size);
}
API void* std_memset(void* ptr, int value, unsigned long num) {
return memset(ptr, value, num);
}
API void* std_memcopy(void* dest, const void* src, unsigned long num) {
return memcpy(dest, src, num);
}
// A couple of test functions that accepts the initialized structure
// and calls sum of the function pointers.
API char* testit(StdLib* test) {
// This is how I debugged this, by comparing the *actual*
// function pointer value to the one received from Python.
// Once they matched the code worked.
printf("%p %p\n", std_malloc, test->mem.malloc);
char* p = static_cast<char*>(test->mem.malloc(10));
test->mem.set(p, 'A', 9);
p[9] = 0;
return p;
}
API void freeit(StdLib* test, char* p) {
test->mem.free(p);
}
}
test.py
import ctypes as ct
# prototypes
MALLOC = ct.CFUNCTYPE(ct.c_void_p,ct.c_ulong)
FREE = ct.CFUNCTYPE(None,ct.c_void_p)
REALLOC = ct.CFUNCTYPE(ct.c_void_p, ct.c_void_p, ct.c_ulong)
CALLOC = ct.CFUNCTYPE(ct.c_void_p, ct.c_ulong, ct.c_ulong)
SET = ct.CFUNCTYPE(ct.c_void_p,ct.c_void_p,ct.c_int,ct.c_ulong)
COPY = ct.CFUNCTYPE(ct.c_void_p, ct.c_void_p, ct.c_ulong)
class StdMemFunc(ct.Structure):
_fields_ = [("malloc", MALLOC),
("free", FREE),
("realloc", REALLOC),
("calloc", CALLOC),
("set", SET),
("copy", COPY)]
class StdLib(ct.Structure):
_fields_ = [("version", ct.c_uint32),
# Note these two fields were function pointers as well.
# Declared correctly now.
("is_version_compatible", ct.CFUNCTYPE(ct.c_bool, ct.c_uint32, ct.c_uint32)),
("delay", ct.CFUNCTYPE(None, ct.c_int32)),
("mem", StdMemFunc)]
dll = ct.CDLL('./test')
dll.testit.argtypes = ct.POINTER(StdLib),
dll.testit.restype = ct.POINTER(ct.c_char)
dll.freeit.argtypes = ct.POINTER(StdLib), ct.c_char_p
dll.freeit.restype = None
lib = StdLib()
lib.mem.malloc = MALLOC(('std_malloc', dll))
lib.mem.realloc = REALLOC(('std_realloc', dll))
lib.mem.calloc = CALLOC(('std_calloc', dll))
lib.mem.free = FREE(('std_free', dll))
lib.mem.set = SET(('std_memset', dll))
lib.mem.copy = COPY(('std_memcopy', dll))
p = dll.testit(lib)
# One way to access the data in the returned pointer is to slice it to the known length
print(p[:10])
# If known to be null-terminated, can also cast to c_char_p, which expects
# null-terminated data, and extract the value.
print(ct.cast(p,ct.c_char_p).value)
dll.freeit(lib,p)
Output:
b'AAAAAAAAA\x00'
b'AAAAAAAAA'
I am mapping integers to memory in C++ (Process 1) and trying to read them in Python (Process 2) ..
Current Results:
1) map integer 3 in C++ ==> Python (b'\x03\x00\x00\x00')
2) map integer 4 in C++ ==> Python (b'\x04\x00\x00\x00'), and so on ..
code:
Process 1
#include <windows.h>
#include <iostream>
using namespace std;
void main()
{
auto name = "new";
auto size = 4;
HANDLE hSharedMemory = CreateFileMapping(NULL, NULL, PAGE_READWRITE, NULL, size, name);
auto pMemory = (int*)MapViewOfFile(hSharedMemory, FILE_MAP_ALL_ACCESS, NULL, NULL, size);
for (int i = 0; i < 10; i++)
{
* pMemory = i;
cout << i << endl;
Sleep(1000);
}
UnmapViewOfFile(pMemory);
CloseHandle(hSharedMemory);
}
Process 2
import time
import mmap
bufSize = 4
FILENAME = 'new'
for i in range(10):
data = mmap.mmap(0, bufSize, tagname=FILENAME, access=mmap.ACCESS_READ)
dataRead = data.read(bufSize)
print(dataRead)
time.sleep(1)
However, my goal is to map an array that is 320*240 in size but when I try a simple array as below
int arr[4] = {1,2,3,4};
and attempt to map to memory by * pMemory = arr;
I am getting the error "a value of type int* cannot be assigned to an entity of type int" and error code "0x80070002" ..
Any ideas on how to solve this problem??
P.S for some reason integer 9 is mapped as b'\t\x00\x00\x00' in python ==> what am I missing?
Use memcpy to copy the array to shared memory.
#include <cstring>
#include <windows.h>
int main() {
int array[320*240];
const int size = sizeof(array);
const char *name = "new";
HANDLE hSharedMemory = CreateFileMapping(NULL, NULL, PAGE_READWRITE, NULL, size, name);
void *pMemory = MapViewOfFile(hSharedMemory, FILE_MAP_ALL_ACCESS, NULL, NULL, size);
std::memcpy(pMemory, array, size);
UnmapViewOfFile(pMemory);
CloseHandle(hSharedMemory);
}
I would like to integrate C modules in the Python, so my choice fell on the interface Python.h. Everything compiled without errors and warnings, so I can not understand what the problem is.
C side:
#include <python3.5m/Python.h>
...
#define PyInt_AsLong(x) (PyLong_AsLong((x)))
typedef PyObject* Py;
static Py getSumma(Py self, Py args){
Py nums;
if (!PyArg_ParseTuple(args, "O", &nums)){
return NULL;
}
size_t numsAmount = PyList_Size(args);
int32_t summa = 0;
for (size_t i = 0; i < numsAmount; i++){
Py temp = PyList_GetItem(nums, i);
int32_t num = PyInt_AsLong(temp);
summa += num;
}
return Py_BuildValue("l", summa);
}
static PyMethodDef moduleMethods[] = {
{"getSumma", (PyCFunction)getSumma, METH_VARARGS, NULL},
{NULL, NULL, 0, NULL}
};
static PyModuleDef SummaLogic = {
PyModuleDef_HEAD_INIT,
"SummaLogic",
"",
-1,
moduleMethods
};
PyMODINIT_FUNC PyInit_SummaLogic(void){
return PyModule_Create(&SummaLogic);
}
setup.py:
from distutils.core import setup, Extension
SummaLogic = Extension("SummaLogic", sources=['SummaLogic.c'])
setup(ext_modules=[SummaLogic])
Python side:
from SummaLogic import getSumma
if __name__ == "__main__":
a = [1, 2, 3]
b = getSumma(a)
print(b)
It seems right, but when I start it in terminal - nothing happens, just hanging without any activity. What could I miss?
It boils down to PyList_Size and that you don't check for errors there.
You probably wanted to use it on nums, not args as argument. However you used on args and a very interesting thing happened:
args is a tuple,
therefore PyList_Size failed and returned -1
that -1 which was cast to an unsigned size_t which probably resulted in a very huge number, probably 2**64-1
therefore your iteration runs a "very long time" because it takes quite a while to iterate over 2**64-1 items (apart from all the out-of-bound memory accesses).
The quick fix would be to use:
Py_ssize_t listlength = PyList_Size(nums); /* nums instead of args */
if (listlength == -1) { /* check for errors */
return NULL;
}
size_t numsAmount = (size_t)listlength /* cast to size_t only after you checked for errors */
However you should check what the error conditions are and test for them after every python C API function call otherwise you'll get a lot of undefined behaviours. Also I probably would stick to the defined return types instead of int32_t (PyInt_AsLong returns long so you might get weird casting errors there as well!), size_t, ... and the typedef PyObject* Py; makes things really tricky for someone who regularly writes C extensions.
I have a few values defined as symbolic constants in my header file:
#define NONE 0x00
#define SYM 0x11
#define SEG 0x43
...
The names of these values represent a certain type of data.
Now in my current implementation of my module I put all these symbolic links into an array
static unsigned char TYPES[] = { NONE, SYM, SEG, ...}
And add the positions of the types in the array as int constants in the module.
PyMODINIT_FUNC initShell(void)
{
PyObject *m;
m= Py_InitModule3("Sample", sample_Methods,"Sample Modules");
if (m == NULL)
return;
...
PyModule_AddIntConstant(m, "NONE", 0);
PyModule_AddIntConstant(m, "SYM", 1);
PyModule_AddIntConstant(m, "SEG", 2);
...
}
And when calling functions I have to do something like :
static PyObject *py_samplefunction(PyObject *self, PyObject *args, PyObject *kwargs) {
int type;
if (!PyArg_ParseTuple(args,kwargs,"i",&type)
return NULL;
int retc;
retc = sample_function(TYPES[type]);
return Py_BuildValue("i", retc);
}
I'm not very happy with this hack and I think it is very prone to errors and so I'm basically looking for a solution which eliminates the array and allows for direct use of the constants in a function call. Any tips?
Edit
Using PyModule_AddIntMacro(m, SEG); and calling sample function as such, solves it:
static PyObject *py_samplefunction(PyObject *self, PyObject *args, PyObject *kwargs) {
int type;
if (!PyArg_ParseTuple(args,kwargs,"i",&type)
return NULL;
int retc;
retc = sample_function((unsigned char) type);
return Py_BuildValue("i", retc);
}
Why not just add the constants to the module ?
PyModule_AddIntMacro(m, SYM);