how to move packet from NF_INET_PRE_ROUTING to NF_INET_POST_ROUTING?

Question

I have a kernel module that utilizes netfilter hooks. The goal is to forward packets to another destination. As I can see by design packets coming from outside with daddr set to my servers IP pass through NF_INET_PRE_ROUTING and then suppose to be queued for local application. On NF_INET_PRE_ROUTING I alter specific packets (detect my own protocol) and replace daddr with remote servers IP and saddr with my servers IP. I would like to do it from within kernel module itself but cannot find a way to either move existing packet to another routing point (either NF_INET_FORWARD or NF_INET_LOCAL_OUT or even NF_INET_POST_ROUTING) or to create new packet and insert it into TCP/IP stack as if it is sent from server itself. Currently the packet simply goes to the blackhole after first hook. I do not see it going to any other hooks somehow. How could I do that?

My current code (testing code where remote server is same as client):

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/netfilter.h>
#include <linux/netfilter_ipv4.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <net/ip.h>
#include <net/tcp.h>
#include <net/route.h>

#define DEBUG 1

static struct nf_hook_ops nfho;

static __be32 srv_addr = 0x620aa8c0;
static __be32 cli_addr = 0x630aa8c0;
static __be32 rem_addr = 0x630aa8c0;

static unsigned int hook_func(unsigned int hooknum, struct sk_buff *skb, const struct net_device *in, const struct net_device *out, int (*okfn)(struct sk_buff *)){
    struct iphdr *ip_header;
    struct tcphdr *tcp_header;

    ip_header = (struct iphdr *)skb_network_header(skb);
    skb_set_transport_header(skb, ip_header->ihl * 4);
    tcp_header = (struct tcphdr *)skb_transport_header(skb);

#if DEBUG > 0
if(tcp_header->dest == ntohs(80) || tcp_header->source == ntohs(80))//(ip_header->saddr == cli_addr || ip_header->saddr == srv_addr || ip_header->saddr == rem_addr) && 
printk(KERN_INFO "[HTTP] Got a packet to %d.%d.%d.%d:%d from %d.%d.%d.%d:%d in hooknum=%d\n", 
    ip_header->daddr & 0x000000FF,
    (ip_header->daddr & 0x0000FF00) >> 8,
    (ip_header->daddr & 0x00FF0000) >> 16,
    (ip_header->daddr & 0xFF000000) >> 24,
    ntohs(tcp_header->dest), 
    ip_header->saddr & 0x000000FF,
    (ip_header->saddr & 0x0000FF00) >> 8,
    (ip_header->saddr & 0x00FF0000) >> 16,
    (ip_header->saddr & 0xFF000000) >> 24,
    ntohs(tcp_header->source),
    hooknum);
#endif

    if(ip_header->saddr == cli_addr && tcp_header->dest == ntohs(80)){
        ip_header->daddr = rem_addr;
        ip_header->saddr = srv_addr;
        ip_header->check = 0;
        ip_send_check(ip_header);
        tcp_header->check = 0;
        tcp_header->check = tcp_v4_check(skb->len - 4*ip_header->ihl, ip_header->saddr, ip_header->daddr, csum_partial((char *)tcp_header, skb->len - 4*ip_header->ihl,0));

        okfn(skb);
        return NF_STOP;
    }
    if(ip_header->saddr == rem_addr && tcp_header->source == ntohs(80)){
        ip_header->daddr = cli_addr;
        ip_header->saddr = srv_addr;
        ip_header->check = 0;
        ip_send_check(ip_header);
        tcp_header->check = 0;
        tcp_header->check = tcp_v4_check(skb->len - 4*ip_header->ihl, ip_header->saddr, ip_header->daddr, csum_partial((char *)tcp_header, skb->len - 4*ip_header->ihl,0));

        okfn(skb);
        return NF_STOP;
    }
    return NF_ACCEPT;
}




static int __init init_main(void) {
    nfho.hook = hook_func;
    nfho.hooknum = 0;
    nfho.pf = PF_INET;
    nfho.priority = NF_IP_PRI_FIRST;
    nf_register_hook(&nfho);
#if DEBUG > 0
    printk(KERN_INFO "[HTTP] Successfully inserted protocol module into kernel.\n");
#endif
    return 0;
}

static void __exit cleanup_main(void) {
    nf_unregister_hook(&nfho);
#if DEBUG > 0
    printk(KERN_INFO "[HTTP] Successfully unloaded protocol module.\n");
#endif
}

module_init(init_main);
module_exit(cleanup_main);

MODULE_LICENSE("GPL v3");
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);

Answer 1

I couldn't find any way to programmatically forward packets in a more or less proper way. The only way I found (seems to be very popular solution) is to manually modify all related fields in skb_buff and transmit altered packet through dev_queue_xmit. This way isn't good, because it does not implement finding a good route for packet. F.e. if neighbouring network includes many nodes that actually could be used for packet routing it doesn't seem to be possible to find a proper route from kernel module (or I am not aware of such way). Also source code for kernels TCP/IP stack shows presence of ip_forward function, which is not available from any part of kernel module and my try to reproduce that function ended up in dragging half of TCP/IP stack into module. This function could be ideal option for programmatical packet forwarding since it only takes a few parameters and all will alter all needed parts of packet by itself.

Anyways. My own fixed code now looks like this:

#include <linux/types.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/netfilter_ipv4.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include "my_mod.h"


#define DRIVER_AUTHOR "AlexKey"
#define DRIVER_DESC "HTTP packets manipulations"

#define DEBUG 1

static struct nf_hook_ops nfho;

static unsigned int hook_func(unsigned int hooknum, struct sk_buff *skb, const struct net_device *in, const struct net_device *out, int (*okfn)(struct sk_buff *)){
    struct iphdr *ip_header;
    struct tcphdr *tcp_header;
    struct ethhdr *eth_header;

    u32 saddr, daddr;
    u16 source, dest;

    /* Get all the headers */
    eth_header = (struct ethhdr *)skb_mac_header(skb);
    ip_header = (struct iphdr *)skb_network_header(skb);
    skb_set_transport_header(skb, ip_header->ihl * 4);
    tcp_header = (struct tcphdr *)skb_transport_header(skb);

    /* If the packet source or dest are not 80 then the packet is not for us :) */
    if(tcp_header->source != ntohs(80) && tcp_header->dest != ntohs(80))
        return NF_ACCEPT;

#if DEBUG > 0
printk(KERN_INFO "[HTTP] Got packet on %d from %d\n", htons(tcp_header->dest), htons(tcp_header->source));
#endif

    saddr = ip_header->saddr;
    daddr = ip_header->daddr;

    source = tcp_header->source;
    dest = tcp_header->dest;

    /* In link layer header change sender mac to our ethernet mac
        and destination mac to sender mac :) ping-pong */
    memcpy(eth_header->h_dest,eth_header->h_source,ETH_ALEN);
    memcpy(eth_header->h_source,skb->dev->dev_addr,ETH_ALEN);

    /* Set new link layer headers to socket buffer */
    skb->data = (unsigned char *)eth_header;
    skb->len += ETH_HLEN;

    /* Setting it as outgoing packet */
    skb->pkt_type = PACKET_OUTGOING;

    /* Swap the IP headers sender and destination addresses */
    memcpy(&ip_header->saddr, &daddr, sizeof(u32));
    memcpy(&ip_header->daddr, &saddr, sizeof(u32));


    /* If transmission suceeds then report it stolen
        if it fails then drop it */
    if(dev_queue_xmit(skb)==NET_XMIT_SUCCESS){
#if DEBUG > 0
printk(KERN_INFO "[HTTP] Successfully sent packet\n");
#endif
        return NF_STOLEN;
    } else {
#if DEBUG > 0
printk(KERN_INFO "[HTTP] Sending failed\n");
#endif
        return NF_DROP;
    }

}


static int __init init_main(void) {
    nfho.hook = hook_func;
    nfho.hooknum = 0;
    nfho.pf = PF_INET;
    nfho.priority = NF_IP_PRI_FIRST;
    nf_register_hook(&nfho);

#if DEBUG > 0
    printk(KERN_INFO "[HTTP] Successfully inserted protocol module into kernel.\n");
#endif
    return 0;
}

static void __exit cleanup_main(void) {
    nf_unregister_hook(&nfho);
#if DEBUG > 0
    printk(KERN_INFO "[HTTP] Successfully unloaded protocol module.\n");
#endif
}

module_init(init_main);
module_exit(cleanup_main);

MODULE_LICENSE("GPL v3");
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);

I would love to hear any modifications to this code.

Answer 2

Alex, I had the exact same problem you had trying to send the mangled skb from kernel. I went through the same thought process, but couldn't find an elegant solution that will properly handle the routing of the outgoing packet. Until I found that I can also use sockets in the kernel.

Create a raw socket in your kernel module using sock_create in socket.h like so:

struct socket *mySock;
if ( sock_create(PF_INET, SOCK_RAW, IPPROTO_RAW, &mySock) != 0 )
{
    /* Error creating socket */
}

Once you modified the IP header, you can then use a function to send your skb using sock_sendmsg:

int sock_send(struct socket *sock, struct sockaddr_in *addr, struct iovec *iov, int iovlen, int totalLen)
{
    struct msghdr msg;
    mm_segment_t oldfs;
    int size = 0;

    if (sock == NULL || sock->sk == NULL)
    {
        return 0;
    }

    msg.msg_flags = 0;
    msg.msg_name = addr;
    msg.msg_namelen = sizeof(struct sockaddr_in);
    msg.msg_control = NULL;
    msg.msg_controllen = 0;
    msg.msg_iov = iov;
    msg.msg_iovlen = iovlen;

    /* Set to kernel data segment since sock_sendmsg expects user space pointers */
    oldfs = get_fs();
    set_fs(KERNEL_DS);

    size = sock_sendmsg(sock, &msg, totalLen);
    set_fs(oldfs);
    return size;
}

Remember with IPPROTO_RAW socket, you must make the IP header yourself, but you already have one in the skb. Now you just have to create and populate the struct iovec array and pass it to sock_send.

For struct sockaddr_in *addr, use the same destination address as the IP header:

struct sockaddr_in addr = { .sin_family = PF_INET,
                            .sin_port   = 0,  /* 0 for RAW socket */
                            .sin_addr   = { .s_addr = dstAddr } };

Remember to return NF_DROP or free the skb and return NF_STOLEN to clean it up once you are done with the skb.

Answer 3

The way to do this using kernel hooks is to manually modify all related fields in skb_buff and transmit altered packet through dev_queue_xmit. As you are trying to create a packet "out of thin air" to a destination, you need to be careful with the routing. Assuming that the routing is properly set up from the userspace point of view, all you need to do to enable the packet to fly, is to use ip_route_output() before dev_queue_xmit(). For example:

struct rtable *rt;
struct net *nt;

// do the packet mangling, headers copying here

skb->dev = new_dev;  // new_dev is the iface through which to reach the dest
nt = dev_net(skb->dev);
rt = ip_route_output(nt, ip_header->daddr, ip_header->saddr, 
    RT_TOS(ip_header->tos), skb->dev_ifindex);

skb_dst_set(skb, &(rt->dst));

return NF_ACCEPT; // pass the mangled packet on, business as usual