SWIG how to make a typedef function pointer in a struct callable from Python

Question

TL;DR Does anybody know how to instruct SWIG to treat these members of a C-struct as a function pointer and make it callable from Python?

The full story I have C structs which contain pointers to functions. The functions are all typedefed. I have a C function which will allocate memory for this C struct and which will set the function pointers to point to valid C functions. My simplified header file looks like this

// simplified api.h
typedef void *handle_t;
typedef void sample_t;
typedef error_t comp_close_t(handle_t *h);
typedef error_t comp_process_t(handle_t h,
                               sample_t *in_ptr,
                               sample_t *out_ptr,
                               size_t *nr_samples);
typedef struct
{
    comp_close_t *close;
    comp_process_t *process;
} audio_comp_t;

// prototype for init
error_t comp_init(handle_t *h, int size);

And corresponding simplified source file:

// simplified api.c
static comp_close_t my_close;
static comp_process_t my_process;

audio_comp_t comp = {
    my_close,
    my_process
};

error_t comp_init(audio_comp_t **handle) {
    *handle = ∁
    return 0;
}

error_t my_close(handle_t *h) {
    // stuff
    *h = NULL;
    return 0;
}

error_t my_process(handle_t h,
                   sample_t *in_ptr,
                   sample_t *out_ptr,
                   size_t *nr_samples) {
    audio_comp_t *c = (audio_comp_t*) h;
    // stuff 
    printf("doing something useful\n");
}

And the latest version of my interface file:

%module comp_wrapper
%{
#include "api.h"
%}

%include "api.h"

// Take care of the double pointer in comp_init
%ignore comp_init;
%rename(comp_init) comp_init_overload;
%newobject comp_init;

%inline %{
audio_comp_t* comp_init_overload(int size) {
    audio_comp_t *result = NULL;
    error_t err = comp_init(&result, size);

    if (SSS_NO_ERROR == err) {
        ...
    }

    return result;
}
%}

// wrap the process call to verify the process_t * function pointer    
%inline %{
sss_error_t call_process(   audio_comp_t *h, 
                            sample_t *in, 
                            sample_t *out, 
                            size_t nr_samples)
{
    return h->process(h, in, out, &nr_samples);
}
%}

I want to use SWIG to create language bindings so that I can call these object-alike structures with minimal boiler plate code from Python. Ultimately I want to use this like:

h = comp_init(50)
h.process(h, input_data, output_data, block_size)
h.close(h)

However, SWIG treats these function pointers in these structs as Objects, so whenever I want to call them I get

>>> h = comp_init(50)
>>> h.api.process()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: 'SwigPyObject' object is not callable

I can workaround it by means of something like the 'call_process' function which you can find in the interface file:

call_process(h, in, out, 32) 

but it would require me to add an extra wrapper for all struct member functions, while this shouldn't be necessary since [the SWIG documentation states that function pointers are fully supported][1]

I assume I should write some code in the interface file so that SWIG knows that it's dealing with a function rather than a SwigPyObject

There is some information on how to deal with (python)callbacks, but none of it seems to work in this case specifically: SWIG call function pointers stored within struct

or without duplicating more or less all of the information from the header file into the interface file: Using SWIG with pointer to function in C struct

and last but not least there seems to be a difference when you wrap a function pointer in a struct so solution 5 does not work: How to wrap a c++ function which takes in a function pointer in python using SWIG

Does anybody know how to instruct SWIG to treat these members of a C-struct as a function pointer and make it callable from Python?

Answer 1

The simplest solution is if we claim to SWIG that the function pointers are simply member functions then the wrapper it will generate works quite nicely.

To demonstrate that in this instance we need to fix up a few errors in your sample code, so I ended up with api.h looking like this:

// simplified api.h
#include <stdint.h>
#include <stdlib.h>

typedef uint32_t api_error_t;

typedef void *handle_t;
typedef void sample_t;
typedef api_error_t comp_close_t(handle_t h);
typedef api_error_t comp_process_t(handle_t h,
                               sample_t *in_ptr,
                               sample_t *out_ptr,
                               size_t nr_samples);
typedef struct
{
    comp_close_t *close;
    comp_process_t *process;
} audio_comp_t;

// prototype for init
api_error_t comp_init(handle_t *new_h);

and api.c looking like this:

#include "api.h"
#include <stdio.h>

// simplified api.c
static comp_close_t my_close;
static comp_process_t my_process;

audio_comp_t comp = {
    my_close,
    my_process
};

api_error_t comp_init(handle_t *handle) {
    *handle = &comp;
    return 0;
}

api_error_t my_close(handle_t h) {
    (void)h; // stuff
    return 0;
}

api_error_t my_process(handle_t h,
                   sample_t *in_ptr,
                   sample_t *out_ptr,
                   size_t nr_samples) {
    audio_comp_t *c = (audio_comp_t*) h;
    (void)c;(void)in_ptr;(void)out_ptr;// stuff 
    printf("doing something useful\n");
    return 0;
}

With that in place we can write api.i as below:

%module api

%{
#include "api.h"
%}

%include <stdint.i>

%typemap(in,numinputs=0) handle_t *new_h (handle_t tmp) %{
    $1 = &tmp;
%}

%typemap(argout) handle_t *new_h %{
    if (!result) {
        $result = SWIG_NewPointerObj(tmp$argnum, $descriptor(audio_comp_t *), 0 /*| SWIG_POINTER_OWN */);
    }
    else {
        // Do something to make the error a Python exception...
    }
%}

// From my earlier answer: https://stackoverflow.com/a/11029809/168175
%typemap(in,numinputs=0) handle_t self "$1=NULL;"
%typemap(check) handle_t self {
  $1 = arg1;
}

typedef struct {
  api_error_t close(handle_t self);
  api_error_t process(handle_t self,
                      sample_t *in_ptr,
                      sample_t *out_ptr,
                      size_t nr_samples);
} audio_comp_t;

%ignore audio_comp_t;
%include "api.h"

Here we've done a few things besides hiding the original structure and claiming it's full of member functions instead of member pointers:

1. Make SWIG automatically pass the handle in as the 1st argument instead of requiring Python users to be excessively verbose. (In python it becomes h.close() instead of h.close(h))
2. Use argout typemap to wrap the real comp_init function instead of just replacing it. That's purely a matter of preference I just added it to show how it could be used.

This lets my run the following Python:

import api

h=api.comp_init()
print(h)
h.process(None, None, 0)
h.close()

---

We can do something that'll work quite nicely for both Python and C if you're willing to make some cosmetic changes to your API's header to facilitate things.

I introduced a macro into api.h, MAKE_API_FUNC, which wraps the typedef statements you had in it originally. When compiled with a C compiler it still produces the exact same results, however it lets us manipulate things better with SWIG.

So api.h now looks like this:

// simplified api.h
#include <stdint.h>
#include <stdlib.h>

typedef uint32_t api_error_t;

typedef void *handle_t;
typedef void sample_t;
#ifndef MAKE_API_FUNC
#define MAKE_API_FUNC(name, type, ...) typedef api_error_t comp_ ## name ## _t(__VA_ARGS__)
#endif

MAKE_API_FUNC(close, audio_comp_t, handle_t);
MAKE_API_FUNC(process, audio_comp_t, handle_t, sample_t *, sample_t *, size_t);

typedef struct
{
    comp_close_t *close;
    comp_process_t *process;
} audio_comp_t;

// prototype for init
api_error_t comp_init(handle_t *new_h);

So in api.i we now replace that macro, with another one, which claims to SWIG that the function pointer typedef is in fact a struct, with a specially provided __call__ function. By creating this extra function we can proxy all our Python arguments automatically into a call to the real function pointer.

%module api

%{
#include "api.h"
%}

%include <stdint.i>

// From: https://stackoverflow.com/a/2653351
#define xstr(a) str(a)
#define str(a) #a

#define name_arg(num, type) arg_ ## num
#define param_arg(num, type) type name_arg(num, type)

#define FE_0(...)
#define FE_1(action,a1) action(0,a1)
#define FE_2(action,a1,a2) action(0,a1), action(1,a2)
#define FE_3(action,a1,a2,a3) action(0,a1), action(1,a2), action(2,a3)
#define FE_4(action,a1,a2,a3,a4) action(0,a1), action(1,a2), action(2,a3), action(3,a4)
#define FE_5(action,a1,a2,a3,a4,a5) action(0,a1), action(1,a2), action(2,a3), action(3,a4), action(4,a5)

#define GET_MACRO(_1,_2,_3,_4,_5,NAME,...) NAME
%define FOR_EACH(action,...)
  GET_MACRO(__VA_ARGS__, FE_5, FE_4, FE_3, FE_2, FE_1, FE_0)(action,__VA_ARGS__)
%enddef

%define MAKE_API_FUNC(name, api_type, ...)
%nodefaultctor comp_ ## name ## _t;
%nodefaultdtor comp_ ## name ## _t;
typedef struct {
    %extend {
        api_error_t __call__(FOR_EACH(param_arg, __VA_ARGS__)) {
            return $self(FOR_EACH(name_arg, __VA_ARGS__));
        }
    }
} comp_ ## name ## _t;
// Workaround from: https://github.com/swig/swig/issues/609
%rename("%s_fptr", "%$isvariable", "match$ismember"="1", "match$type"=xstr(comp_ ## name ## _t)) name;
%extend api_type {
    %pythoncode %{
        name = lambda self, *args: self.name ## _fptr(self, *args)
    %}
}
%enddef

%ignore comp_init;
%include "api.h"

%extend  audio_comp_t {
    audio_comp_t() {
        handle_t new_h = NULL;
        api_error_t err = comp_init(&new_h);
        if (err) {
            // throw or set Python error directly
        }
        return new_h;
    }

    ~audio_comp_t() {
        (void)$self;
        // Do whatever we need to cleanup properly here, could actually call close
    }
}

This is using the same preprocessor mechanisms I used in my answer on wrapping std::function objects, but applied to the function pointers of this problem. In addition I used %extend to make a constructor/destructor from the perspective of Python, which makes the API nicer to use. I'd probably use %rename too if this were real code.

With that said we can now use the following Python code:

import api

h=api.audio_comp_t()
print(h)
print(h.process)
h.process(None, None, 0)

See SWIG docs for a discussion on how to map the error codes onto exceptions nicely for Python too.

---

We can simplify this further, by removing the need to iterate over the arguments of the variadic macro, with one simple trick. If we change our api.h macro to take 3 arguments, the 3rd of which is all the function pointer's arguments like this:

// simplified api.h
#include <stdint.h>
#include <stdlib.h>

typedef uint32_t api_error_t;

typedef void *handle_t;
typedef void sample_t;
#ifndef MAKE_API_FUNC
#define MAKE_API_FUNC(name, type, args) typedef api_error_t comp_ ## name ## _t args
#endif

MAKE_API_FUNC(close, audio_comp_t, (handle_t self));
MAKE_API_FUNC(process, audio_comp_t, (handle_t self, sample_t *in_ptr, sample_t *out_ptr, size_t nr_samples));

typedef struct
{
    comp_close_t *close;
    comp_process_t *process;
} audio_comp_t;

// prototype for init
api_error_t comp_init(handle_t *new_h);

Then we can now change our SWIG interface to not provide a definition of the __call__ function we added via %extend, and instead write a macro that directly makes the function pointer call we wanted:

%module api

%{
#include "api.h"
%}

%include <stdint.i>

// From: https://stackoverflow.com/a/2653351
#define xstr(a) str(a)
#define str(a) #a

%define MAKE_API_FUNC(name, api_type, arg_types)
%nodefaultctor comp_ ## name ## _t;
%nodefaultdtor comp_ ## name ## _t;
%{
#define comp_ ## name ## _t___call__(fptr, ...) fptr(__VA_ARGS__)
%}
typedef struct {
    %extend {
        api_error_t __call__ arg_types;
    }
} comp_ ## name ## _t;
// Workaround from: https://github.com/swig/swig/issues/609
%rename("%s_fptr", "%$isvariable", "match$ismember"="1", "match$type"=xstr(comp_ ## name ## _t)) name;
%extend api_type {
    %pythoncode %{
        name = lambda self, *args: self.name ## _fptr(self, *args)
    %}
}
%enddef

%ignore comp_init;
%include "api.h"

%extend  audio_comp_t {
    audio_comp_t() {
        handle_t new_h = NULL;
        api_error_t err = comp_init(&new_h);
        if (err) {
            // throw or set Python error directly
        }
        return new_h;
    }

    ~audio_comp_t() {
        (void)$self;
        // Do whatever we need to cleanup properly here, could actually call close
    }
}

The tricky thing here was that the use of typedef struct {...} name; idiom made renaming or hiding the function pointers inside the struct harder. (That was only necessary however to keep the addition of the handle_t argument automatic however).