head	1.14;
access;
symbols;
locks; strict;
comment	@ * @;


1.14
date	2010.08.09.20.22.31;	author wessels;	state Exp;
branches;
next	1.13;

1.13
date	2010.08.09.17.47.33;	author wessels;	state Exp;
branches;
next	1.12;

1.12
date	2010.08.09.17.46.15;	author wessels;	state Exp;
branches;
next	1.11;

1.11
date	2010.08.09.17.45.52;	author wessels;	state Exp;
branches;
next	1.10;

1.10
date	2010.08.09.17.42.58;	author wessels;	state Exp;
branches;
next	1.9;

1.9
date	2010.08.09.17.42.41;	author wessels;	state Exp;
branches;
next	1.8;

1.8
date	2010.08.09.16.58.56;	author wessels;	state Exp;
branches;
next	1.7;

1.7
date	2010.08.09.16.57.39;	author wessels;	state Exp;
branches;
next	1.6;

1.6
date	2010.08.09.16.55.19;	author wessels;	state Exp;
branches;
next	1.5;

1.5
date	2009.08.07.17.42.40;	author wessels;	state Exp;
branches;
next	1.4;

1.4
date	2009.07.24.20.39.02;	author wessels;	state Exp;
branches;
next	1.3;

1.3
date	2009.04.15.20.38.43;	author wessels;	state Exp;
branches;
next	1.2;

1.2
date	2008.04.28.17.46.07;	author wessels;	state Exp;
branches;
next	1.1;

1.1
date	2007.10.08.16.43.36;	author wessels;	state Exp;
branches;
next	;


desc
@@


1.14
log
@iadd ipv6 support
add -l local option
add -s sorted option
@
text
@#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/errno.h>
#include <sys/time.h>
#include <assert.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
#include <netinet/udp.h>
#include <arpa/inet.h>
#include <arpa/nameser.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <inttypes.h>

#include <pcap.h>
#include "pcap_layers.h"

#define LIMIT_OPEN_FD 8192
#define LIMIT_CLT_PKTS 2048
#define LIMIT_MAXRSS (256<<20)
#define LIMIT_PKTS_IN_MEM (2<<20)
#define QUAD_A(ip) ((ntohl(ip.u.in4.s_addr) >> 24) & 0xFF)
#define QUAD_B(ip) ((ntohl(ip.u.in4.s_addr) >> 16) & 0xFF)
#define QUAD_C(ip) ((ntohl(ip.u.in4.s_addr) >>  8) & 0xFF)
#define QUAD_D(ip) ((ntohl(ip.u.in4.s_addr)      ) & 0xFF)

#define MAX_FILTER_SZ 256*1024

typedef struct _dlink_node dlink_node;
typedef struct _dlink_list dlink_list;

struct _dlink_node {
    void *data;
    dlink_node *prev;
    dlink_node *next;
};

struct _dlink_list {
    dlink_node *head;
    dlink_node *tail;
};

struct _packet {
    struct pcap_pkthdr hdr;
    void *data;
    struct _packet *next;
};

struct inx_addr {
	uint8_t family;
	union {
		struct in_addr in4;
		struct in6_addr in6;
	} u;
};

struct _client {
    struct inx_addr addr;
    struct _packet *pkthead;
    struct _packet **pkttail;
    int npackets;
    pcap_dumper_t *fd;
    struct _client *next;
    dlink_node lru;
};

static struct _client *Hash[256];
static struct _dlink_list *LRU;
static unsigned int nopen = 3;
static unsigned int nclts = 0;
static unsigned int npacketsmem = 0;
static pcap_t *in = NULL;
static int input_sorted = 0;	/* assume input already sorted by sip */
static int use_subdirs = 1;	/* write files into subdirs */

int
inx_addr_hash(struct inx_addr a)
{
	if (AF_INET == a.family)
		return QUAD_A(a);
	return 0;
}

int
inx_addr_equal(struct inx_addr a, struct inx_addr b)
{
	if (a.family != b.family)
		return 0;
	if (AF_INET == a.family)
		return a.u.in4.s_addr == b.u.in4.s_addr;
	return 0 == memcmp(&a.u.in6, &b.u.in6, 16);
}

void
dlinkAdd(void *data, dlink_node * m, dlink_list * list)
{
    m->data = data;
    m->prev = NULL;
    m->next = list->head;
    if (list->head)
	list->head->prev = m;
    list->head = m;
    if (list->tail == NULL)
	list->tail = m;
}

void
dlinkDelete(dlink_node * m, dlink_list * list)
{
    if (m->next)
	m->next->prev = m->prev;
    if (m->prev)
	m->prev->next = m->next;
    if (m == list->head)
	list->head = m->next;
    if (m == list->tail)
	list->tail = m->prev;
    m->next = m->prev = NULL;
}

void
hashDelete(struct _client *f)
{
    int i = inx_addr_hash(f->addr);
    struct _client **F;
    for (F = &Hash[i]; *F; F = &(*F)->next)
	if (f == *F) {
		*F = f->next;
		break;
	}
}

void
mksubdir(const char *path)
{
    char *t;
    for (t = (char *) path; *t; t++) {
	if ('/' == *t) {
		*t = '\0';
		if (mkdir(path, 0755) < 0 && EEXIST != errno) {
			perror(path);
			exit(1);
		}
		*t = '/';
	}
    }
}

const char *
output_fname(const struct _client *f) {
	unsigned int l = 0;
	static char fname[256];
    	static char aname[128];
	inet_ntop(f->addr.family, &f->addr.u, aname, sizeof(aname));
        if (use_subdirs && AF_INET == f->addr.family) {
		l += snprintf(&fname[l], sizeof(fname)-l, "%03d/", QUAD_A(f->addr));
		l += snprintf(&fname[l], sizeof(fname)-l, "%03d/", QUAD_B(f->addr));
	} else
        if (use_subdirs && AF_INET6 == f->addr.family) {
		l += snprintf(&fname[l], sizeof(fname)-l, "%04x/", f->addr.u.in6.__u6_addr.__u6_addr16[0]);
		l += snprintf(&fname[l], sizeof(fname)-l, "%04x/", f->addr.u.in6.__u6_addr.__u6_addr16[1]);
	}
	l += snprintf(&fname[l], sizeof(fname)-l, "%s", aname);
	if (!input_sorted)
		l += snprintf(&fname[l], sizeof(fname)-l, "/%lu.%06lu", 
                        (long unsigned int) f->pkthead->hdr.ts.tv_sec,
                        (long unsigned int) f->pkthead->hdr.ts.tv_usec);
	l += snprintf(&fname[l], sizeof(fname)-l, "%s", ".pcap");
	assert(l < sizeof(fname));
	return fname;
}


void
clt_pcap_open(struct _client *f)
{
    const char *file;
    if (NULL != f->fd)
	return;
    file = output_fname(f);
    if (input_sorted) {
	struct stat sb;
	if (0 == stat(file, &sb)) {
		fprintf(stderr, "%s already exist, perhaps input is not sorted?\n", file);
		exit(1);
	}
    }
    f->fd = pcap_dump_open(in, file);
    if (NULL == f->fd && errno == ENOENT) {
        mksubdir(file);
        f->fd = pcap_dump_open(in, file);
    }
    if (NULL == f->fd) {
        perror(file);
        exit(1);
    }
    nopen++;
}


void
clt_free_packets(struct _client *f)
{
    struct _packet *p;
    struct _packet *n;
    for (p = f->pkthead; p; p = n) {
	n = p->next;
	free(p->data);
	free(p);
    }
    npacketsmem -= f->npackets;
    f->npackets = 0;
    f->pkthead = NULL;
    f->pkttail = &f->pkthead;
}

void
clt_pcap_write(struct _client *f)
{
    struct _packet *p;
    if (0 == f->npackets)
	return;
    for (p = f->pkthead; p; p = p->next)
	pcap_dump((void *)f->fd, &p->hdr, p->data);
    clt_free_packets(f);
}

void
clt_pcap_close(struct _client *f)
{
    if (NULL == f->fd)
	return;
    pcap_dump_close(f->fd);
    nopen--;
    f->fd = NULL;
}

void
clt_free(struct _client *f)
{
    if (f->npackets)
	clt_free_packets(f);
    hashDelete(f);
    dlinkDelete(&f->lru, LRU);
    free(f);
    nclts--;
}

void
close_lru(void)
{
    int nc = 0;
    dlink_node *p = LRU->tail;
    fprintf(stderr, "Closing LRU...");
    while (nopen > (LIMIT_OPEN_FD / 2) && p) {
	struct _client *f = p->data;
	p = p->prev;
	if (NULL == f->fd)
		continue;
	clt_pcap_write(f);
	clt_pcap_close(f);
	/* clt_free(f); */
	nc++;
    }
    fprintf(stderr, "%d\n", nc);
}

void
flush(struct _client *f)
{
    if (nopen >= LIMIT_OPEN_FD)
	close_lru();
    clt_pcap_open(f);
    clt_pcap_write(f);
}

long
getmaxrss(void)
{
    struct rusage ru;
    getrusage(RUSAGE_SELF, &ru);
    return ru.ru_maxrss;
}

void
flushall()
{
    int i;
    int n = 0;
    struct _client *f;
    struct _client *next;
    fprintf(stderr, "Flushing...\n");
    for (i = 0; i < 256; i++) {
	for (f = Hash[i]; f; f = next) {
	    next = f->next;
	    if (0 == f->npackets)
		continue;
	    if (f->fd == NULL)
		clt_pcap_open(f);
	    clt_pcap_write(f);
	    n++;
	    if (0 == (n % 1000))
		fprintf(stderr, "flushed %d clts, open fd: %d\n", n, nopen);
	    if (nopen >= LIMIT_OPEN_FD)
	        close_lru();
	}
    }
    fprintf(stderr, "flushed %d\n", n);
    fprintf(stderr, "open files: %d\n", nopen);
    fprintf(stderr, "max rss: %ld\n", getmaxrss());
}

void
stash2(struct _client *f, struct pcap_pkthdr *hdr, const unsigned char *data)
{
    struct _packet *p = calloc(1, sizeof(*p));
    assert(p);
    p->hdr = *hdr;
    p->data = malloc(hdr->caplen);
    assert(p->data);
    memcpy(p->data, data, hdr->caplen);
    *f->pkttail = p;
    f->pkttail = &p->next;
    f->npackets++;
    npacketsmem++;
#if 0
    if (f->npackets > LIMIT_CLT_PKTS)
	flush(f);
#endif
}

void
stash(struct inx_addr a, struct pcap_pkthdr *hdr, const unsigned char *data)
{
    struct _client **F;
    struct _client *f;
    int i = inx_addr_hash(a);
    for (F = &Hash[i]; (f = *F); F = &(*F)->next) {
	if (inx_addr_equal(f->addr, a))
	    break;
    }
    if (NULL == f) {
	nclts++;
	f = calloc(1, sizeof(*f));
	assert(f);
	f->addr = a;
	f->pkttail = &f->pkthead;
	f->next = Hash[i];
	Hash[i] = f;
    } else if (f != Hash[i]) {
	/* move to top */
	*F = f->next;
	f->next = Hash[i];
	Hash[i] = f;
    }
    stash2(f, hdr, data);
    dlinkDelete(&f->lru, LRU);
    dlinkAdd(f, &f->lru, LRU);
}

void
print_stats(struct timeval ts, uint64_t pkt_count)
{
    struct timeval now;
    gettimeofday(&now, NULL);
    fprintf(stderr, "%ld.%03ld: at %ld, %12"PRIu64" pkts, %9d clts, %4d files\n",
	(long) now.tv_sec,
	(long) now.tv_usec / 1000,
	(long)ts.tv_sec,
	pkt_count,
	nclts,
	nopen);
}

int
my_ip4_handler(const struct ip *ip4, int len, void *userdata)
{
    struct inx_addr *a = userdata;
    a->family = AF_INET;
    a->u.in4 = ip4->ip_src;
    return 0;
}

int
my_ip6_handler(const struct ip6_hdr *ip6, int len, void *userdata)
{
    struct inx_addr *a = userdata;
    a->family = AF_INET6;
    a->u.in6 = ip6->ip6_src;
    return 0;
}

int
is_rfc1918(struct inx_addr a)
{
    unsigned long clt_addr = ntohl(a.u.in4.s_addr);
    if (AF_INET != a.family)
	return 0;
    // 10/8
    if ( ( clt_addr & 0xff000000) == 0x0A000000 )
        return 1;
    // 172.16/12
    if ( ( clt_addr & 0xfff00000) == 0xAC100000 )
        return 1;
    // 192.168/16
    if ( ( clt_addr & 0xffff0000) == 0xC0A80000 )
        return 1;

    return 0;
}


int
main(int argc, char *argv[])
{
    uint64_t pkt_count = 0;
    char errbuf[PCAP_ERRBUF_SIZE + 1];
    struct pcap_pkthdr hdr;
    const u_char *data;
    char *filterstr = NULL;
    char *filterfile = NULL;
    FILE *FP = NULL;
    char buf[80];
    char or_str[4] = "\0";
    char lc = 0;
    struct bpf_program fp;
    int ch;
    int skip_bogon = 0;

    setbuf(stdout, NULL);
    setbuf(stderr, NULL);
    memset(Hash, '\0', sizeof(Hash));
    LRU = calloc(1, sizeof(LRU));
    assert(LRU);

    // Process command line
    while ((ch = getopt(argc, argv, "bf:ls")) != -1) {
        switch(ch) {
            case 'b':
                skip_bogon = 1;
                break;
            case 'f':
                filterfile = strdup(optarg);
                break;
            case 'l':
                use_subdirs = 0;
                break;
	    case 's':
		input_sorted = 1;
		break;
            default:
                fprintf(stderr, "usage: %s [-b] [-f addr_list_file]\n", argv[0]);
                exit(1);
                break;
        }
    }
    argc -= optind;
    argv += optind;

    if (NULL != filterfile) {
    // If a filter file was given, read it and prepare
    if ( (FP = fopen(filterfile, "r")) == NULL)
    {
        fprintf(stderr, "Can't read filter file %s, aborting\n",
        filterfile);
        exit(1);
    }
    filterstr = (char *)calloc(1, MAX_FILTER_SZ);
    while (fgets(buf, 80, FP))
    {
        if (strlen(filterstr) > MAX_FILTER_SZ - 12)
            continue;

        if (lc != 0)
            strcpy(or_str, "or");
        snprintf(filterstr, MAX_FILTER_SZ, "%s %s src net %s",
                    filterstr, or_str, buf);
        ++lc;
    }
    fclose(FP);

    // For debugging purporses, write the filter
    FILE *filter_fp = fopen("filter_pcap.dat", "w");
    if (NULL == filter_fp)
        fprintf(stderr, "Can't write filter, skipping\n");
    else {
        fprintf(filter_fp, "%s\n", filterstr);
        fclose(filter_fp);
    }
    }

    in = pcap_open_offline("-", errbuf);
    if (NULL == in) {
	fprintf(stderr, "stdin: %s", errbuf);
	exit(1);
    }

    // Set the filter
    if (NULL != filterstr) {
        memset(&fp, '\0', sizeof(fp));
        if (pcap_compile(in, &fp, filterstr, 1, 0) < 0) {
            fprintf(stderr, "pcap_compile failed: %s\n", pcap_geterr(in));
            exit(1);
        }
        if (pcap_setfilter(in, &fp) < 0) {
            fprintf(stderr, "pcap_setfilter failed: %s\n", pcap_geterr(in));
            exit(1);
        }
        fprintf(stderr, "Filter read and compiled!\n");
    }


    pcap_layers_init(pcap_datalink(in), 0);
    callback_ipv4 = my_ip4_handler;
    callback_ipv6 = my_ip6_handler;

    while ((data = pcap_next(in, &hdr))) {
	struct inx_addr src;
	memset(&src, 0, sizeof(src));
	handle_pcap((u_char *) & src, &hdr, data);
	if (src.family == 0)
	    continue;
    if (skip_bogon && is_rfc1918(src))
        continue;
	stash(src, &hdr, data);
	if (0 == (++pkt_count & 0x3FFF)) {
	    print_stats(hdr.ts, pkt_count);
	    if (npacketsmem > LIMIT_PKTS_IN_MEM) {
		flushall();
		close_lru();
	    }
	}
    }
    flushall();
    fprintf(stderr, "Max RSS  %ld KB \n", getmaxrss());
    return 0;
}
@


1.13
log
@*** empty log message ***
@
text
@d13 1
d28 4
a31 4
#define QUAD_A(ip) ((ntohl(ip.s_addr) >> 24) & 0xFF)
#define QUAD_B(ip) ((ntohl(ip.s_addr) >> 16) & 0xFF)
#define QUAD_C(ip) ((ntohl(ip.s_addr) >>  8) & 0xFF)
#define QUAD_D(ip) ((ntohl(ip.s_addr)      ) & 0xFF)
d55 8
d64 1
a64 1
    struct in_addr addr;
d82 18
d130 1
a130 1
    int i = (f->addr.s_addr >> 24) & 0xFF;
d140 1
a140 1
mksubdir(const struct _client *f)
d142 10
a151 17
    struct in_addr a = f->addr;
    static char dir[128];
    assert(use_subdirs);
    snprintf(dir, 128, "%03d", QUAD_A(a));
    if (mkdir(dir, 0755) < 0 && EEXIST != errno) {
	perror(dir);
	exit(1);
    }
    snprintf(dir, 128, "%03d/%03d", QUAD_A(a), QUAD_B(a));
    if (mkdir(dir, 0755) < 0 && EEXIST != errno) {
	perror(dir);
	exit(1);
    }
    snprintf(dir, 128, "%03d/%03d/%s", QUAD_A(a), QUAD_B(a), inet_ntoa(a));
    if (mkdir(dir, 0755) < 0 && EEXIST != errno) {
	perror(dir);
	exit(1);
d157 15
a171 6
	static char fname[128];
	if (use_subdirs && !input_sorted) {
		snprintf(fname, sizeof(fname), "%03d/%03d/%s/%lu.%06lu.pcap",
                        QUAD_A(f->addr),
                        QUAD_B(f->addr),
                        inet_ntoa(f->addr),
d174 2
a175 14
	} else if (use_subdirs && input_sorted) {
		snprintf(fname, sizeof(fname), "%03d/%03d/%s.pcap",
                        QUAD_A(f->addr),
                        QUAD_B(f->addr),
                        inet_ntoa(f->addr));
	} else if (!use_subdirs && input_sorted) {
		snprintf(fname, sizeof(fname), "%s.pcap",
                        inet_ntoa(f->addr));
	} else if (!use_subdirs && !input_sorted) {
		snprintf(fname, sizeof(fname), "%s.%lu.%06lu.pcap",
                        inet_ntoa(f->addr),
                        (long unsigned int) f->pkthead->hdr.ts.tv_sec,
                        (long unsigned int) f->pkthead->hdr.ts.tv_usec);
	}
d187 7
d195 2
a196 2
    if (NULL == f->fd && use_subdirs && errno == ENOENT) {
        mksubdir(f);
d339 1
a339 1
stash(struct in_addr a, struct pcap_pkthdr *hdr, const unsigned char *data)
d343 1
a343 1
    int i = (a.s_addr >> 24) & 0xFF;
d345 1
a345 1
	if (f->addr.s_addr == a.s_addr)
d373 2
a374 2
	now.tv_sec,
	now.tv_usec / 1000,
d384 12
a395 2
    struct in_addr *a = userdata;
    *a = ip4->ip_src;
d400 1
a400 1
is_rfc1918(uint32_t addr)
d402 3
a404 1
    unsigned long clt_addr = ntohl(addr);
d521 1
d524 1
a524 1
	struct in_addr src;
d527 1
a527 1
	if (src.s_addr == 0)
d529 1
a529 1
    if (skip_bogon && is_rfc1918(src.s_addr))
@


1.12
log
@*** empty log message ***
@
text
@d407 1
a407 1
    while ((ch = getopt(argc, argv, "bf:")) != -1) {
d415 6
@


1.11
log
@*** empty log message ***
@
text
@d143 2
a144 2
                        f->pkthead->hdr.ts.tv_sec,
                        f->pkthead->hdr.ts.tv_usec);
d156 2
a157 2
                        f->pkthead->hdr.ts.tv_sec,
                        f->pkthead->hdr.ts.tv_usec);
@


1.10
log
@*** empty log message ***
@
text
@d138 2
a139 1
	snprintf(fname, sizeof(fname), "%03d/%03d/%s/%lu.%06lu.pcap",
d145 14
@


1.9
log
@make
@
text
@d113 1
a113 1
mksubdir(const struct client *f)
d136 1
a136 1
output_fname(const struct client *f) {
@


1.8
log
@sigh
@
text
@d70 2
d113 1
a113 1
mksubdir(struct in_addr a)
d115 1
d117 1
d135 13
d151 1
a151 2
    char file[128];

d154 1
a154 7

    snprintf(file, 128, "%03d/%03d/%s/%lu.%06lu.pcap",
                        QUAD_A(f->addr),
                        QUAD_B(f->addr),
                        inet_ntoa(f->addr),
                        f->pkthead->hdr.ts.tv_sec,
                        f->pkthead->hdr.ts.tv_usec);
d156 2
a157 2
    if (NULL == f->fd && errno == ENOENT) {
        mksubdir(f->addr);
@


1.7
log
@fix mksubdir() error conditions
@
text
@d115 1
a115 1
    if (mkdir(dir, 0755) < 0 && EEXIST == errno) {
d120 1
a120 1
    if (mkdir(dir, 0755) < 0 && EEXIST == errno) {
d125 1
a125 1
    if (mkdir(dir, 0755) < 0 && EEXIST == errno) {
@


1.6
log
@complain more in mksubdir()
@
text
@d115 1
a115 1
    if (mkdir(dir, 0755)) {
d120 1
a120 1
    if (mkdir(dir, 0755)) {
d125 1
a125 1
    if (mkdir(dir, 0755)) {
@


1.5
log
@pcap_layers_init() now has reassemble flag
@
text
@d115 4
a118 1
    mkdir(dir, 0755);
d120 4
a123 1
    mkdir(dir, 0755);
d125 4
a128 1
    mkdir(dir, 0755);
@


1.4
log
@import version from caida-oarc.caida.org
@
text
@d443 1
a443 1
    pcap_layers_init(pcap_datalink(in));
@


1.3
log
@printf warns
@
text
@d18 1
d32 2
d130 6
a135 5
    snprintf(file, 128, "%03d/%03d/%s/%lld.%06lld.pcap",
	QUAD_A(f->addr), QUAD_B(f->addr),
	inet_ntoa(f->addr),
	(long long int) f->pkthead->hdr.ts.tv_sec,
	(long long int) f->pkthead->hdr.ts.tv_usec);
d138 2
a139 2
	mksubdir(f->addr);
	f->fd = pcap_dump_open(in, file);
d142 2
a143 2
	perror(file);
	exit(1);
d310 1
a310 1
print_stats(struct timeval ts, int pkt_count)
d314 2
a315 2
    fprintf(stderr, "%lld.%03ld: at %ld, %10d pkts, %9d clts, %4d files\n",
	(long long int) now.tv_sec,
d331 17
d352 1
a352 1
    unsigned int pkt_count = 0;
d356 9
d372 50
d427 16
d452 2
@


1.2
log
@fix warning: format '%d' expects type 'int', but argument 7 has type 'time_t'
@
text
@d310 2
a311 2
    fprintf(stderr, "%ld.%03ld: at %ld, %10d pkts, %9d clts, %4d files\n",
	now.tv_sec,
@


1.1
log
@Initial revision
@
text
@d127 1
a127 1
    snprintf(file, 128, "%03d/%03d/%s/%d.%06d.pcap",
d130 2
a131 1
	f->pkthead->hdr.ts.tv_sec, f->pkthead->hdr.ts.tv_usec);
@