pax_global_header 0000666 0000000 0000000 00000000064 14045010307 0014504 g ustar 00root root 0000000 0000000 52 comment=3e8706952f7caaf8ff479b4cd49d1488bff824ce
nfdump-1.6.23/ 0000775 0000000 0000000 00000000000 14045010307 0013066 5 ustar 00root root 0000000 0000000 nfdump-1.6.23/.gitignore 0000664 0000000 0000000 00000001754 14045010307 0015065 0 ustar 00root root 0000000 0000000 # Object files
*.o
*.ko
*.obj
*.elf
# Precompiled Headers
*.gch
*.pch
# Libraries
*.lib
*.a
*.la
*.lo
# Shared objects (inc. Windows DLLs)
*.dll
*.so
*.so.*
*.dylib
# Executables
*.exe
*.out
*.app
*.i*86
*.x86_64
*.hex
# Debug files
*.dSYM/
# Backup files
*~
# Automake generated files
/INSTALL
/Makefile.in
/aclocal.m4
/autom4te.cache/
/bin/Makefile.in
/compile
/config.guess
/config.h.in
/config.sub
/configure
/depcomp
/install-sh
/ltmain.sh
/man/Makefile.in
/missing
/test-driver
/ylwrap
/doc/Makefile.in
# Configure generated files
/Makefile
/bin/.deps/
/bin/Makefile
/config.h
/config.log
/config.status
/extra/nftrack/.deps/
/libtool
/man/Makefile
/stamp-h1
/doc/Doxyfile
/doc/Makefile
# Binaries generated by make
/bin/.libs/
/bin/grammar.c
/bin/grammar.h
/bin/nfanon
/bin/nfcapd
/bin/nfdump
/bin/nfexpire
/bin/nfpcapd
/bin/nfprofile
/bin/nfreplay
/bin/nftrack
/bin/scanner.c
/bin/sfcapd
/extra/nftrack/.dirstamp
# Generated docs
/doc/doxygen-build.stamp
/doc/html/
/doc/man/
/doc/xml/
nfdump-1.6.23/AUTHORS 0000775 0000000 0000000 00000000046 14045010307 0014141 0 ustar 00root root 0000000 0000000 Peter Haag peter@people.ops-trust.net
nfdump-1.6.23/BSD-license.txt 0000664 0000000 0000000 00000003120 14045010307 0015653 0 ustar 00root root 0000000 0000000
The nfdump project is distributed under the BSD license:
Copyright (c) 2016, Peter Haag
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be
used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
nfdump-1.6.23/COPYING 0000777 0000000 0000000 00000000000 14045010307 0016705 2BSD-license.txt ustar 00root root 0000000 0000000 nfdump-1.6.23/ChangeLog 0000775 0000000 0000000 00000071103 14045010307 0014645 0 ustar 00root root 0000000 0000000 2021-05-05
- Release 1.6.23
2021-04-28
- Fix potential FreeNode without valid Node in nfpcapd.
2021-04-21
- Add all non TCP/UDP IP protocols as streams in nfpcapd
2021-04-20
- Add mpls unwrap in nfpcapd. Skip MPLS labels
- Add ESP to processed protocols in nfpcapd.
- Some Code cleanup
2021-04-10
- Change spin lock to native C11 lock
- Cleanup code for issue #283
2021-04-09
- Fix minor nfpcapd issues
- Add mpls unwrap in sflow code - adds mpls labels if available
2021-03-12
- Update rbtree.
- Fix potential deadlock in nfpcapd if it terminates.
2021-03-06
- Add packet capture buffer size to nfpcapd
2021-02-20
- Fix sflow code extended field parsing. #262 and #273
- Fix endless loop of nfexpire, if it does not find files
2021-01-30
- Fix processing deoding error for yaf exporter
- Zero out tcp flags for non TCP records
2021-01-16
- Add reverse element enterprise ID 29305 for counter values
- Add biFlow direction element 239
- Add flow end reason element 136
- Make -Tall the default for nfcapd to collect extensions
2020-12-22
- Code cleanup and boundary checks in option template processing
2020-12-19
- Implement element 160 (SystemInitTime) in option template
2020-12-19
- Add Element 160 (SystemInitTime) in flow record used by Huawei
2020-12-05
- Fix path handling for -l
- Fix print plain numbers #263
2020-11-21
- Release 1.6.22
2020-10-18
- Fix nfreplay v5 time shift bug
2020-10-17
- add support for >=, <= comparators. #256. Thanks to piorek94
- Fix yacc/bison warnings. Cleanup unused tokens
- Fix syntax error 'flags AS' as AS is a reserved word. #255
- Add element 139 for ICMP type/code in IPv6. #250
- Fix IPv4/IPv6 statistics representation #252
2020-09-12
- Cleanup nip/xip filter syntax. Add filter syntax 'nip in [ ]'. Request #246
2020-09-03
- Add nfversion to nfpcapd
2020-08-31
- Add collected netflow/sflow version in nfdump record. Request #242
- Fix GuessDir bug - issue #215
2020-08-02
- Re-address issue #231 - remove strict rule rfc 7011
2020-08-02
- Release 1.6.21
- Address issue #159. Implement rfc 7011 and include sender UDP port into unique template identification
- Address issue #236 Add token 'dir' equivalent to 'flowdir' in filter syntax
2020-07-25
- Add optional print direction ascending or descending to output of statistics -s and oredered printing -O. Request #235
- Fix issue #234
- Fix #230 - Avoid use_syslog name clash on certain OS
2020-06-20
- Honor -n flag when printing sorted flow cache
- Fix uninitialized variable printPlain
- Fix bug #223 limit matchig flows -c
- Restore old behaviour unlimiting output flows unless in -s stat
- Fix ft2nfdump nexthop fields
- Fix ft2nfdump extension map size
- internal: put output parameters in a single struct
- Fix GuessDir bug - issue #215
- Compact Changelog
- Fix GuessDir bug - issue #215 in flow exporter
2020-03-29
- Release 1.6.20
- More cleanup on plain number printing
- Fix plain numbers bug #213
- Fix profiler filer bug
2020-02-22
- Release 1.6.19
- Add Source security group tag (SGT 34000) issue #192
- Modify heuristic for bidir flows #59
- Add Push for new sgt tag.
- Cleanup unused nffile records. Add nffileV2.h
- Fix various compiler warnings and automake issues
- Remove external global vars
- Set verbose logging off by default in non daemon mode
- Fix make check
- Remove old legacy records
- Major code rearrangement in order to prepare for futur versions of nfdump
Move extension definitions into extension files nfx.h
Move exporter definition into extension.h
nffile.h should only contain nffile data block handlich definitions
- Fix compile error for FreeBSD #203
- Cleanup header code - add new filter.h - free nfdump.h from dust
- More code cleanup - delete nf_common - add output_fmt
- Fix bug for IPv6 network cidr representation in raw and cvs output mode.
- Remove unused old code. Fix for #197
- Fix bidir export issues. Cleanup code. Fix for #195
- Enable small time intervals < 60s. - #185
- Cleanup old code
- Fix minor nfpcapd issues
- Fix gcc compile issue
- Fix corrupt file handling #193
- More code cleanup. Move code to dedicated files.
- Restructure code for output into separate files.
- Cleanup old code.
- Fix bug #189 - valid json output
- Fix issue #190. Add compile time option for JunOS.
- Cleanup configure.ac
- Fix various c11 compile issues
- Cleanup old compat15 comments
- Add vlan tags dot1qVlanId, 243, 254 - #182
- Fix sflow issue with Arista switches
- Fix ft2nfdump next hop and ip router fields
- Cleanup and fix IPv6 network display in war records.
- Fix compile issues
- Fix output buffer size for lzo1x_decompress_safe()
- Fix VerifyExtensionMap #179
- Fix compile errors
2019-08-05
- Release 1.6.18
- Fix nfdump.1 man page. #175
- Fix off by 1 array. #173
- Fix use after free in ModifyCompressFile
- Add bound checks in AddExporterStat #174
- Add bound checks in AddSamplerInfo #176
- Add bound checks in AddExporterInfo
- Fix checks in InsertExtensionMap #177
- Remove COMPAT15 code - should no longer be needed.
- Merge pull request #167
- Cleanup old code
- Replace depricated pcap_lookupdev call in nfpcapd
2019-07-31
- Add early record size sanity check also for nfprofile, nfanon and nfreplay
2019-07-26
- nfpcapd cleanup, add some more monitoring
- Fix hbo_exporter.c:249_1 segfault
- Fix hbo_nffile_inline.c:85_1 segfault
- Fix hbo_nfx.c:216_3 segfault
- Update minilzo to v2.10
- Change to safe lzo decompress function
2019-07-25
- Rework nfpcapd and add it officially to the nfdump collection.
- Add nfpcapd man page
- Fix potential unsigned integer underflow #171
2019-07-16
- Add latency extension if dumping flowcache
2019-07-15
- Fix typos
- Fix exporter struct inconsistancies. Coredump on ARM otherwise.
2019-07-02
- Add ipfix element #150, #151 unix time start/end
- Fix display bug raw record
2019-06-01
- Add ipfix dyn element handling.
- Add empty m4 directory - keep autoconf happy
2019-06-01
- Fix issue #162 - ipfix mpls sequece.
- Fix issue #156 - print flowtable index error
2019-03-17
- Fix spec file
- Remove non thread safe logging in nfpcapd
2018-11-24
- Fix protocol tag for protocol 87 - TCF - #130
- Add TCP flags ECN,CVR - #132
- Fix some error messages to be printed to the correct stream #135
- Add missing -M command line help to nfcapd
- Remove padding byte warning in log #141
- Fix bug to accept -y compression flag in nfcapd. - #145
2018-06-24
- Fix bookkeeper type - use key_t
- Add multiple packet repeaters to nfcapd/sfcapd. Up to 8 repeaters (-R) can be defined.
- Ignore OSX .DS_Store files in -R file list
- Add CISCO ASA elements initiatorPackets (298) responderPackets (299)
- Merge #120 pull request for -z parameter to nfreplay
- Update man page nfreplay
2018-05-06
- New bookkeeper hash broke NfSen. Fixed. ported back to release 1.6.17
2018-04-20
- Release 1.6.17
2018-04-20
- Fix bug in sorting when guessing flow direction. Issue #92
- Update nfdump.1 man page for xsrcport & xdstport aggregations. Request #109
- Fix minor bugs
- Fix definition for InfluxDB in configure.ac Issue #98
2018-04-01
- Add program exit in nfx.c after panic with correupt data file
- Add missing size check when reading nfdump 1.5.x common record blocks
- Add missing option -M in man page. Issue #103
- Add Fix processing of influx URL in nfprofile
2018-02-11
- Add missing json output format in nfdump help text
- Add missing -v option in nfreplay help text
2018-01-06
- Merge pull request #51 Influxdb from Luca. Thx for the patch
2018-01-01
- IPFIX time stamps - Fix elements #21,#22 offset calculation, but timestamps not yet evaluated. (#160)
- IPFIX add fwd status tag #89 compatible to v9 (1byte)
2017-12-31
- IPFIX sampling - sampling algorithm no longer required for tag #34
- IPFIX sampling add tags #305 and #304 - set them identical to #34, #35
2017-12-30
- Add new output format json. Print each record as individual json object
2017-12-28
- Add sampling elements ID 302,304,305. put them identical to ID 48,49,50
- Add option to label filter terms. syntax: () %labelname.
- Add %lbl option to print flow label in output
- Update nfdump(1) man page for flowlabels
2017-12-27
- Add ipfix delta timestamp elements 158/159.
- Update sflow code to commit 7322984 of https://github.com/sflow/sflowtool
- Cleanup sflow code - uncomment unnecessary code
- Fix header includes"
- Fix 64bit fts compat issue in fts_compat.c
- Add more detailed autogen.sh - softlink bootstrap
2017-12-22
- Fix potential memory leaks in nfpcapd
2017-12-21
- Fix wrong offset calculation if unknown options are found
- Add x-late src/dst ip aggregation, if compiled with NSEL support
2017-12-17
- Add ipfix sampling. Process option template/record with sampling elements 34 and 35
- Report updates on existing samplers in v9 only if values change. issue 84
2017-11-05 v1.6.16
2017-12-10
- Add lz4 compression
- Remove old xstat legancy code, not needed
- Remove automake files from git
2017-12-03
- Fix old 1.6.15 tags
- Fix minor issues and compiler warnings
2017-10-22
- Add support for CISCO IOS 8 bytes timestamps ID 21/22
- Fix issue #72 - multiple stat output
- Change -B behaviour as proposed in issue #59. Should not impact with previous use, but is more flexible
- Add bzip compress switch in usage output of nfpcapd
- Fix compile issues on some platforms
- nfpcapd improvements - still beta software.
- Minor bug fixes
2016-11-25
- Add latency extension to nfpcapd
- Smaller bug fixes to nfpcapd
2016-07-23
- Replace unreliable _ftok with more reliable string hash
2016-07-20
- Aggregate using in+out bytes for bidirectional flows
2016-06-05 v.1.6.15
- Fix Security issue http://www.security-assessment.com/files/documents/advisory/Nfdump%20nfcapd%201.6.14%20-%20Multiple%20Vulnerabilities.pdf
- Fix obyte, opps and obps output records
- Fix wrong bps type case in cvs output. Fix opbs ipbs typos
2016-01-10 v.1.6.14
- Fix CentOS compile issues with flow-tools converter
- Fix FreeBSD,OpenBSD build problems
- Fix timestamp overflow in sflow.c
2015-12-23
- Fix IP Fragmentation in sflow collector
- Create libnfdump for dynamic linking
2015-10-02
- Fix compile errors on other platforms
- Add -R to ModifyCompression
- Add std sampler ID 4 Bytes and allow random sampler (tag 50)
- Add BZ2 compression along existing LZ0
- Add direct write to flowtools converter ft2nfdump
- Fix zero alignment bug, if only half of an extension is sent
- Fix nfanon time window bug in subsequent files in -R list
- Fix CommonRecordV0Type conversion bug
- Fix nfexport bug, if only one single map exists
2014-11-16 v.1.6.13
- Fix v1 extension size bug
- Add htonll check for autoconf
- Fix AddExtensionMap compare bug
- Fix ipfix templare withdraw problems - free all maps correctly
- Add minilzo 2.08 - fixes CVE-2014-4607
- Cleanup some stat code. more needs to be done ..
- Cleanup man pages for -O -n
- Remove SunPro test in configure - no longer supported anyway
- Cleanup NAT/NSEL filter differences
2014-06-15 v1.6.12p1
- Add pblock compare functions
- Update extended filter: Allow modification left/right values
2014-02-16 v1.6.12
- Add NAT pool port allocation
- Modify/fix NAT vrf tags. Add egress vrf ID
- Modify common record due to exporter exhaustion. new common record
type 10 adds 4 extra bytes. Reads v1 common record transparently
- Fix sflow potential crash
2013-11-13 v1.6.11
- Add ASA/NSEL 9.x protcol changes
- Make it llvm compilable
2013-08-12 v1.6.10p1
- Fix -t +/- n timeslot option
- Fix bug in nfanon - stat record update.
- Fix bug in netflow v5 mudule: extension map size wrong.
- Fix bug nfexport: In some cases could result in wrong flow counter.
- Fix nftrack - could coredump in some cases.
2013-05-16 v1.6.10
- Fix SPARC compile/optimise bug
- Add output packet/bytes counter to global stat - importatnt for NSEL flows ASA > 8.5
- Add NSEL filter options xnet
- Modify extension descriptor code for nfdump1.7. Still use 1.6 extension map layout for compatibility
- Add prototype for nfpcapd - pcap -> nfdump collector. Converts traffoc directly to nfdump files.
- Fix bug in ipfix module: uninitialised variable
- Cleanup syslog/LogError calls
- Fix minor non critical bugs and compile issues
2013-03-02 v1.6.9
- Fix some bugs in beta 1.6.9 NSEL code
- Fix bug statistics update with aggreagted flow records
- Fix sflow bug sfcapd stores wrong (ghost) dump by past samples in same sflow datagram
2012-12-31
- Add time received in csv output
- ICMP should handled better now - somewhat
- Implement ASA NSEL records
- Add definitions in nffile and nx for ASA NSEL extensions
2012-11-09 v1.6.8p1
- Add dynamic source directory tree for multiple exporters
- Fix exporter bug: 'too many exporters' with large time windows
- Fix uninitialised exporter sysid in default sampler record - v9
- Fix v9/ipfix cache initialisation with no templates > 1 in same packet
2012-10-26 v1.6.8
- Add ip list option for 'next ip' in filter syntax
- Accept v9 sampler_id in 2bytes
- Fix IPFIX mac address bug - did not get collected
- Add IPFIX packet/octet TotalCount fields 85/86
- Add received timestamp to sflow collector
- Fix long flow duration calculation - 32bit overflow
- Fix v9 sampling ID: allow 2 byte ID
- Add IPFIX options as rfc5101 section-6.2
- Add exporter records for sflow collector
- Fix bug for MAC address printing %idmc and %odmc.
- Add received time stamp extension
- Add recursive format parser. Allows to extend predefined formats.
- Change flow record sorting to heapsort. remove limit 1000
- Merge -m option to -O tstart. -m now depricated.
- Add -O tend. Print order according to tend of flows ascending
- Apply -O print order for printing flow cache. Applies to -A
2012-07-31 v1.6.7-tc-1
- Special version for TC
- Print exporter and sampling records with nfdump -E
- Added exporter and sampling records to file.
2012-07-30 v1.6.7
- Prepare for file catalog in current file format.
- Fix bug in ReadBlock when reading flow from stdin pipe
- Add new more flexible translation engine for v9
- Add nprobe client/server delay fields
- Prepare for NSEL merging
- Fix memory corruption with double -A flags
- Fix bug in nfreader with compat15 mode files
2012-03-12 v1.6.6
- Minor IPFIX bug.
- IPFIX implement template withdraw
- For IPFIX, check packet sequence per template and observation domain
- Fix time window, when no flows collected or no flows matched
while processing
- Fixed typos
- Fix seg fault bug - test for EMPTY_LIST was missing at several places.
2012-02-19 v1.6.6b1
- Fix bps/pps. make it uint64_t, as bps/pps > 4Gb/s overflows.
- In record raw print mode: decode ICMP instead of src/dst ports
- sflow use announced exporter IP instead of sending IP for router ID
- sflow: Ignore extra fill bytes. Do not complain.
- sflow: fix packet length issue.
- Add IPFIX protokoll support
2011-12-31 v1.6.5
- Fix 64bit bug when using byte/packet limits
- for v5 and sampling use 64bit counters to prevent overflow for large sampled flows.
- Fixed Ident printig bug
2011-07-11 v1.6.4
- some code restructuring - prepare for IPFIX module
- Add netflow v1 module. Some routers still use that
- Add %sn, %dn output tags for src/dst networks
- Fix buffer length check in v5.
- Fix export bug: include last flow cache bucket, when exporting
- number in all filter expressions accept hex values
- fix an sflow colletor bug. Missing extension maps in rotated files
- implement extended statistics. Currently ports and bpp distribution
vectors can be collected automatically be nfcapd. Still experimental
2011-02-26 v1.6.3p1
- Fix timebug fix :(, make it a compile time option
- fix v7 sequence errors
2011-02-15
- Zero out unused fields after aggregation
2011-02-05
- Fix SysUptime 32bit overflow in v5 header
- Add fix for strange first/last swap reported by some users.
2011-01-09 v1.6.3
- Fix extension size bug
- Move IP anonymisation to separate binary nfanon
- Fix initialise bug of -o fmt: and not available fields
2010-09-09 v1.6.2
- released
- fixes some sflow bugs in sfcapd
2010-04-28 v1.6.1p0
- Update flow tools converter to build with Google-Code version 0.68.5
- Fix sflow bugs
2010-03-05 v1.6.1
- Fix bug in man page for -t
- Test sampler infos before using them ( nfcapd startup )
- Add sampling tags #34, #35 used by JunOS
- nfexpire: Fix empty .nfsat, when setting limits on an empty directory
- Fix coredump for -B -m (-w) combination
- Optimise some extension map code
2009-12-28 stable v1.6
- Few bug fixes in release candidates rc1, rc2 and rc3
2009-11-16 snapshot-1.6b-20091116
- Update sflow collector with new tags
- Add router IP extension
- Add router ID (engine type/ID) extension
2009-09-30 snapshot-1.6b-20090930
- snapshot bugfix release
2009-11-0801 snapshot-1.6b-20090806
- Add srcmask and dstmask aggregation
- Add csv output mode. -o csv
- Fix some bugs of previous beta
- Add bidirectional aggregation of flows ( -b, -B )
- Add possibility to save aggregated flows into file ( -w )
Note: This results in a behaviour change for -w in combination
with aggragation )
- Extend -N ( do not scale numbers ) to all text output not just summary
- Make extension handling more robust for some moody IOSes.
- Remove header lines of -s stat, when using -q ( quiet )
Note: This results in a behaviour change for -N
- Remove -S option from nfdump ( legacy 1.4 compatibility )
- Make use of log (syslog) functions for nfprofile.
- Move log functions to util.c
2009-06-19 snapshot-1.6b-20090717
- Flow-tools converter updated - supports more common elements.
- Sflow collector updated. Supports more common elements.
- Add sampling to nfdump. Sampling is automatically recognised
in v5 undocumented header fields and in v9 option templates.
see nfcapd.1(1)
- Add @include option for filter to include more filter files.
- Add flexible aggregation comparable to Flexible Netflow (FNF)
- All new tags can be selected in -o fmt:... see nfdump(1)
- topN stat for all new tags is implemented
- Integrate developer code to read from pcap files into stable
- Update filter syntax for new tags
- Added more v9 tags for netflow v9.
The detailed tags are listed in nfcapd(1)
Adding new tags also extended the binary file format with
data block format 2, which is extension based. File format
for version <= 1.5.* ( Data block format 1 ) is read
transparently. Data block 2 are skipped by nfdump 1.5.7.
32bit but AS and interface numbers are supported.
- Add flexible storage option for nfcapd. To save disk space, the
data extensions to be stored in the data file are user selectable.
- Added option for multiple netflow stream to same port.
-n
Example: -n router1,192.168.100.1,/var/nfdump/router1
So multiple -n options may be given at the command line
Old style syntax still works for compatibility, ( -I .. -l ... )
but then only one source is supported.
- Move to automake for building nfdump
- Switch scaling factor ( k, M, G ) from 1024 to 1000.
- Make nfdump fully 64bit compliant. ( 8bit data alignments and access )
2009-04-17 stable 1.5.8
- Fix daylight summer time bug, when guessing sub dirs. file access ( -M, -r )
- Bug fixes for 64bits CPUs
2008-02-22 stable-1,5.7
- Add icmp type/code decoding
- Add proper icmp v9 decoding
- Fix memory leaks in -e auto expire mode in nfcapd.
- Fix somee potential dead locks with file locking, when expiring
- Fix multicast bug in nfreplay
- Add hostname lookup for IP addresses in filter.
2007-10-15 stable-1.5.6
- Fix odd CISCO behaviour for ICMP type/code in src port.
- Add fast LZO1X-1 compression option (-z) for output file.
- Add lists for port in syntax -> port in [ 135 137 445]
- Add lists for AS syntax -> as in [ 1024 1025 ]
- Bug fix in filter for syntax 'src as and dst as'
2007-08-24 stable-1.5.5
- Fix nfprofile bug, nfprofile crashes when last opts line is not valid for
some reason.
- Fix potential hand for nfexpire, on empty flow directories.
2007-08-08 snapshot-20070808
- Idents may contain '-' in name.
- Fixed install bugs in Makefile.in and configure.in
- Installs now cleanly on Solaris
- Handle 4byte interface numbers in v9. Quick fix: 4bytes reduced to 2bytes.
- Fix aggregation bug in statistics.
- ftok(3) C library call replaced by more reliable own implementation.
Did result in error messages like "Another collector is already running"
- Fix minor bugs iin file range selction -R.
- Add recursive behaviour for -R
- New option -i can canche Ident descriptor in data files.
2007-03-12 snapshot-20070312
- Bug fix release of 20070306
2007-03-06 snapshot-20070306
- Fix bug in flist.c. Resulted in a coredump when using sub dirs and -R . ( all files )
- Fix minor bug in nfcapd.c.
- Extend nfprofile for alerting system of nfsen - special version of profiles
- Extend nfprofile for shadow profiles.
2007-08-10 snapshot-20070110
- Fix some compiler warnings, when compiled on a 64bit LINUX
- Fixes an sflow bug: IP address was printed in wrong direction. ( lower bits first )
- Add new IP addr taging option -T for easy parsing for nfsen lookups
- Add new IP list for massive address filtering:
syntax: ip in [ 12345 23456 3456 ....]
- Change nfprofile for channel based profiling. This breaks with old nfprofile
functionality.
- Remove space from ICMP type/code when followed by an IP address
2006-07-21 snapshot-20060809
- Make nfexpire ready for profile expiration
- Fix bug in nfrpofile. sub dir hierarchy not handled correctly.
2006-07-21 snapshot-20060721
- Add -N option for plain number output in summary line
2006-07-21 snapshot-20060721
- Do recursive file selection when a directory is given by -R
2006-06-14 snapshot-20060621
- Add srcas/dstas/proto aggregation.
Note: This changes the default aggregation behaviour, but gives more flexibility
- Add tos to element statistics list
2006-06-14 snapshot-20060614
- Add additional stat line at the end of output
- Add new binary nfexpire. Manages data expiry on time and/or size based limits
Includes new bookkeeping records in nfcapd. See nfexpire(1)
- Add ICMP type/code decoding in flow listing instead of dst port
- Add packet repeater in nfcapd/sfcapd. In addition, incoming UDP packets can
be directly forwarded to another IP address/Port. See new option -R
- Add sub directory hierarchies: Files can be stored into various sub dir levels
based on different time formats. see new option -S
- Some minor bug fixes.
- Code cleanup in nfcapd. better daemonize code and communication with launcher.
2006-04-xx v.1.5.1
Fix bug in nfdump.c: Writing anonymized flows to file did not work corretly
stdin input format now compatible with file format, therefore
'nfdump < file' works again as it did in nfdump 1.4.
Fix bug in nfcapd.c: Error handling not correct when receiving a non
recognized netflow packet. Resulted in an endless loop
2006-03-27 snapshot 1.5-20060327
Make all element statistics -s transport layer protocol
independant by default. Add :p to stat name ( e.g. srcip:p ) to
enable transport layer dependant statistics on request.
2006-03-20 snapshot 1.5-20060320
Fix bug in filter engine: 'not flags xyz' produces wrong results
when more than a single flag is specified.
Minor man page fixes.
2006-03-06 v1.5
Fix bug nfcapd. Laucher signaled too early. File not yet properly
closed.
2006-02-14 v1.5-beta-5
Add srcas, dstas, input and output interfaces in aggregated
output.
Fix IPv6 bug in filter: accept 1234:: address.
rename nfcapd.curent tmp file to nfcapd.curren.. Poorly
configured nfcapd processes may mess up themselves otherwise.
2006-02-02 v1.5-beta-4
Fix netflow v5 dPkts <-> dOctets collector bug.
Update pipe format to include more information
Allow AS number 0 in filter syntax.
Add some more boundary checking - netflow exporters aren't bug free either - sigh ..
2006-01-11 v1.5-beta-3
Fix isnumber incompatibility in grammar.y
Add 'if' statistics
2006-01-10 v1.5-beta-2
nf_common.c Fix bug in format parser.
Extended 'proto ' syntax to support all protocols
Change time format in summary line to ISO format
2005-12-20 v1.5-beta-1
*.* A lot of internal changes, not mentioned here. :(
nfdump Add subnet aggregation for option -A
A new syntax e.g. srcip4/24, dstip6/64 is supported for subnet wise aggregation.
example: traffic of a whole subnet -A srcip4/24 -s srcip/bytes
nfdump Add more stat element option. -s now supports:
srcip, dstip, ip, srcport, dstport, port, srcas, dstas, as, inif, outif, proto
nfdump Add -z. Suppress writing flows to data files. Only stat information is written.
nfprofile Used only be nfsen for upcoming shadow profiles. If you don't understand this
simply ignore it.
nfdump Add -q option to suppress header as well as stat information at the bottom
nfprofile for easier post processing with external programms.
nf_common.c Output format processsing rewritting for more flexibility. Besides standard
nfdump.c output formats line, long extended etc., user defined output formats are now
possible and can even be compiled into nfdump for easy access. See -o fmt:
and nfdump.c around line 100.
*.* Integrate netflow v9 into nfdump. Only a subset of v9 is stored into
the data files, basically everything needed for nfdump to work as it did before.
This also includes IPv6 support for any nfdump options. CryptoPAN extended
to work with IPv6. IPv6 condensed output format for better readability.
Output formats available in long and condensed mode: e.g. line -> line6
extended -> extended6
*.* Replace binary data file format. Old format not flexible enough for
upcoming netflow v9/sflow data. *.stat files are gone. The same
information is now available under nfdump -I
New format about 5% larger in size, but faster for reading and writing.
speed gain eaten up by more complex processing - sigh ..
compat14 mode enables transparent reading of old style format.
nffile.[ch] now handles all data file stuff.
nfreplay Multicast enabled:
Add -j . Joins the specified multicast group ( v4 or v6 )
sending flows to this group.
nfreplay IPv6 enabled:
Add option -4 and -6 to force a specific protocol, otherwise
protocol is automatically selected according the hostname to send flows to.
Add -K key, to send data anonymized, using CryptoPAn
nfcapd Multicast enabled:
Add -j . Joins the specified multicast group ( v4 or v6 )
for listening.
nfcapd IPv6 enabled:
Add option -4 and -6 for IPv4 and IPv6. By default, listen on IPv4.
Option -b to bind for a specific host/IP address automatically
selects appropriate protocol.
nfnet.c All functions to setup network sockets for listening/sending are
put into this file.
2005-08-22 v1.4
- nfreplay: Bug fix sending flows.
- nfdump: Add CryptoPAn code to anonymize IP addresses. New option -K
- nfdump: Change time format in output to ISO 8601 compatible: e.g. 1981-04-05 14:30:30.100
- nfdump: Add scaling factor k,m,g to number in filter syntax: e.g. bytes > 1m
- nfdump: Create new output format extended with additional fields pps, bps and bpp
- nfdump: Rename output format extended to raw
- nfdump: More than one single flow element statistic ( -s ) is now possible
- nfdump: Add user defined sort order in flow element statistic
- nfdump: Flow element statistic can be ordered by more than one order in the same run
- nfdump: Add pps, bps and bpp fields in flow element statistics
- nfdump: Add more symbolic protocols ESP, AH, GRP and RVSP to filter syntax
- nfdump: Add duration, pps, bps and bpp to filter syntax
- nfdump: Make nfdump miliseconds aware. Older versions skipped msecs.
Binary nfdump file format changed due to this.
output formats changed, due to this.
- nfdump: Add interface in/out if syntax to filter
- nfcapd: Add flow_sequence check. Reports missing flows now.
- nfcapd: Report statistics to syslog LOG_INFO when data file is rotated.
- ft2nfdump: Add ft2nfdump to read netflow data from flow-tools
2005-04-21 v1.3
- Add option -A for more flexible aggregation.
- Correct spelling errors :(
2005-03-04 v1.2.1
Bug fix release
- nfcapd: launcher subprocess may hang on Linux 2.6.x kernels.
Cleaned up interrupt handling.
- nfcapd: fix include order of socket.h and types.h in order to
compile cleanly under FreeBSD 4.x
- nfcapd: clean up syslog logging.
- nfdump: Multiple sources ( -M ) and sort flows ( -m ) with
-c did not list the correct flows.
- nfprofile: Profiling with multiple sources may produce incorrect
profiles.
2004-12-20 v1.2
- nfcapd handles transparent v5 and v7 flows. v7 gets converted into v5
- nfcapd can execute any command at the end of interval. New option -x
- nfdump Extended filter syntax for flags, to, bytes and packets
- Rearrange output formats in nfdump: new switch -o, remove switch -E
output formats: 'line', 'long', 'extended' and 'pipe'
- More flexible statistic handling in nfdump: cleanup ugly -s -s -s
syntax. Replaced by -s option. New statistics for Port and AS.
2004-09-20 v 1.1
First public Version.
nfdump-1.6.23/CreateSubHierarchy.pl 0000775 0000000 0000000 00000010027 14045010307 0017142 0 ustar 00root root 0000000 0000000 #!/usr/bin/perl
#
#
# Sample script to clean old data.
# Run this script each hour to cleanup old files to make room for
# new data. When max_size_spool is reached the oldest files are
# deleted down to high_water.
#
# Copyright (c) 2004, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# * Neither the name of SWITCH nor the names of its contributors may be
# used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# $Author: peter $
#
# $Id: CreateSubHierarchy.pl 77 2006-06-14 14:52:25Z peter $
#
# $LastChangedRevision: 77 $
#
use strict;
use warnings;
use POSIX qw(strftime);
use Time::Local;
use Getopt::Std;
our(
$opt_l, # Data directory
$opt_S, # Sub hierarchy format. Correspondes to -S to nfcapd. See nfcapd(1)
);
getopts('l:S:');
my $subdir_format;
my @subdir_formats = (
"",
"%Y/%m/%d",
"%Y/%m/%d/%H",
"%Y/%W/%u",
"%Y/%W/%u/%H",
"%Y/%j",
"%Y/%j/%H",
"%F",
"%F/%H"
);
sub usage {
print "$0 [options]\n",
" -l datadir Data directory\n",
" -S Sub hierarchy format. Correspondes to -S to nfcapd. See nfcapd(1)\n",
"\n";
exit(0);
}
sub ISO2UNIX {
my $isotime = shift;
$isotime =~ s/\-//g; # allow '-' to structur time string
# 2004 02 13 12 45 /
my $sec = 0;
my ( $year, $mon, $mday, $hour, $min ) = $isotime =~ /(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})/;
$mon--;
# round down to nearest 5 min slot
my $diff = $min % 5;
if ( $diff ) {
$min -= $diff;
}
my $unixtime = Time::Local::timelocal($sec,$min,$hour,$mday,$mon,$year);
return $unixtime;
} # End of ISO2UNIX
if ( !defined $opt_l || !defined $opt_S ) {
usage();
}
my $data_dir = $opt_l;
if ( !defined $subdir_formats[$opt_S] ) {
die "Unknown format number $opt_S";
}
$subdir_format = $subdir_formats[$opt_S];
opendir DIR, "$data_dir" || die "Can't open current directory: $!\n";
$| = 1;
print "Reorganizing data files ... ";
while ( my $entry = readdir DIR ) {
next if $entry =~ /^\./;
next unless -f "$data_dir/$entry";
next unless $entry =~ /nfcapd\.(\d{12})$/;
my $date = $1;
my $unix_time = ISO2UNIX($date);
my $sub_path = strftime $subdir_format, localtime($unix_time);
if ( !-d "$data_dir/$sub_path" ) {
print "Need to create '$data_dir/$sub_path'\n";
my @dirlist = split '\/', $sub_path;
my $all_dirs = undef;
foreach my $dir ( @dirlist ) {
$all_dirs = defined $all_dirs ? "$all_dirs/$dir" : $dir;
if ( !-d "$data_dir/$all_dirs" ) {
mkdir "$data_dir/$all_dirs" || die "Can't create subdir '$data_dir/$all_dirs'\n";
}
}
}
rename "$data_dir/$entry", "$data_dir/$sub_path/$entry" || die "Can't move file: $!\n";
}
print "done.\n";
nfdump-1.6.23/LICENSE 0000664 0000000 0000000 00000003120 14045010307 0014067 0 ustar 00root root 0000000 0000000
The nfdump project is distributed under the BSD license:
Copyright (c) 2016, Peter Haag
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be
used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
nfdump-1.6.23/Makefile.am 0000775 0000000 0000000 00000000206 14045010307 0015123 0 ustar 00root root 0000000 0000000
SUBDIRS = . bin man doc
EXTRA_DIST = CreateSubHierarchy.pl LICENSE BSD-license.txt extra/PortTracker.pm extra/nfdump.spec bootstrap
nfdump-1.6.23/NEWS 0000777 0000000 0000000 00000000000 14045010307 0015332 2ChangeLog ustar 00root root 0000000 0000000 nfdump-1.6.23/README 0000777 0000000 0000000 00000000000 14045010307 0015220 2README.md ustar 00root root 0000000 0000000 nfdump-1.6.23/README.md 0000775 0000000 0000000 00000037234 14045010307 0014361 0 ustar 00root root 0000000 0000000 # nfdump
Stable Release v1.6.23
See the Changelog file for all changes in release 1.6.23
nfdump is a toolset in order to collect and process netflow and sflow data, sent from netflow/sflow compatible devices.
The toolset supports netflow __v1__, __v5/v7__,__v9__,__IPFIX__ and __SFLOW__. nfdump supports IPv4 as well as IPv6.
__Note:__ nfdump 1.6.18 and newer versions __not longer__ support nfdump-1.5.x files. If you have nfdump-1.5.x please convert them
before upgrading.
nfdump is used as backend toolset for __NfSen__.
---
## NSEL/ASA, NEL/NAT support
__NSEL__ (Network Event Security Logging) as well as NEL (NAT Event Logging) are technologies invented by __CISCO__ and also use the netflow v9 protocol. However, NSEL and NEL are not flows as commonly known but rather *__Events__!* exported from specific devices such as CISCO ASA. nfdump supports Event looging as part of netflow v9.
__Note:__ The older nfdump-1.5.8-2-NSEL is __not compatible__ with nfdump > 1.6.9 which supports NSEL/NEL.
__Junos NAT Event Logging__ is mostly compatible with CISCO's NAT Event Logging - mostly - it needs another data interpretation.
See __--enable-jnat__ below
---
## IPFIX
nfdump contains an IPFIX module for decoding IPFIX flow data. It
does not support the full IPFIX definition.
* Supports basically same feature set of elements as netflow_v9 module
* Only UDP traffic is accepted no TCP/SCTP
* If you would like to see more IPFIX support, please contact me.
---
## Overview
### Building and config options
The toolset is build upon the autotools framework. Run `./autogen.sh` first.
Afterwards `./configure` `make` and `make install` should do the trick.
The following config options are available:
* __--enable-nsel__
Compile nfdump, to read and process NSEL/NEL event data; default is __NO__
* __--enable-jnat__
compile nfdump, to read and process JunOS NAT event logging __NO__
* __--enable-ftconv__
Build the flow-tools to nfdump converter; default is __NO__
* __--enable-sflow__
Build sflow collector sfcpad; default is __NO__
* __--enable-nfprofile__
Build nfprofile used by NfSen; default is __NO__
* __--enable-nftrack__
Build nftrack used by PortTracker; default is __NO__
This code no longer reads nfdump-1.5.x data files. If needed use nfdump up
to v1.6.17
Development and beta options
* __--enable-devel__
Insert lots of debug and development code into nfdump for testing and debugging; default is __NO__
* __--enable-readpcap__
Add code to nfcapd to read flow data also from pcap files; default is __NO__
* __--enable-nfpcapd__
Build nfpcapd collector to create netflow data from interface traffic or precollected pcap traffic, similar to softflowd; default is __NO__
### The tools
__nfcapd__ - netflow collector daemon.
Collects the netflow data, sent from exporters and stores the flow records
into files. Automatically rotates files every n minutes. ( typically
every 5 min ) The netflow versions mentioned above are read transparently
Multiple netflow streams can be collected by a single or collector.
nfcapd can listen on IPv6 or IPv4. Furthermore multicast is supported.
__nfdump__ - process collected netflow records.
Nfdump reads the netflow data from one or many files stored by nfcapd.
It's filter syntax is similar to tcpdump ( pcap like ) but adapted for netflow.
If you like tcpdump you will like nfdump. nfdump displays netflow
data and/or creates top N statistics of flows, bytes, packets. nfdump
has a powerful and flexible flow aggregation including bi-directional
flows. The output format is user selectable and also includes a simple
csv format for post processing.
__nfanon__ - anonymize netflow records
IP addresses in flow records are anonimized using the CryptoPAn method.
__nfexpire__ - expire old netflow data
Manages data expiration. Sets appropriate limits. Used by NfSen.
__nfreplay__ - netflow replay
Reads the netflow data from the files stored by nfcapd and sends it
over the network to another host.
#### Optional binaries:
__nfpcapd__ - pcap to netflow collector daemon
nfpcapd listens on a network interface, or reads precollected pcap traffic
and stores flow records into nfcapd comaptible files. It is nfcapd's
companion to convert traffic directly into nfdump records.
__sfcapd__ - sflow collector daemon
scfapd collects sflow data and stores it into nfcapd comaptible files.
"sfcapd includes sFlow(TM) code, freely available from https://github.com/sflow/sflowtool.
__nfprofile__ - netflow profiler. Required by NfSen
Reads the netflow data from the files stored by nfcapd. Filters the
netflow data according to the specified filter sets ( profiles ) and
stores the filtered data into files for later use.
__nftrack__ - Port tracking decoder for NfSen plugin PortTracker.
__ft2nfdump__ - flow-tools flow converter
ft2nfdump converts flow-tools data into nfdump format.
__nfreader__ - Framework for programmers
nfreader is a framework to read nfdump files for any other purpose.
Own C code can be added to process flows. nfreader is not installed
__parse_csv.pl__ - Simple reader, written in Perl.
parse_csv.pl reads nfdump csv output and print the flows to stdout.
This program is intended to be a framework for post processing flows
for any other purpose.
#### Notes for sflow users:
sfcapd and nfcapd can be used concurrently to collect netflow and sflow
data at the same time. Generic command line options apply to both
collectors likewise. sfcapd's sflow decoding module is based on InMon's
sflowtool code and supports similar fields as nfcapd does for netflow v9,
which is a subset of all available sflow fields in an sflow record.
More fields may be integrated in future versions of sfcapd.
---
### Compression
Binary data files can optionally be compressed using either the fast LZO1X-1 compression,
LZ4 or the efficient but slow bzip2 method.
If you compress automatically flows while they are collected, LZO1X-1 or LZ4 methods are
recommended. bzip2 uses about 30 times more CPU than LZO1X-1. Use bzip2 to archive netflow
data, which may reduce the disk usage again by a factor of 2. The compression of flow files
can be changed any time with nfdump -J
For more details on each methde, see:
LZO1X-1: http://www.oberhumer.com/opensource/lzo
LZ4: https://github.com/lz4/lz4
bzip2: http://www.bzip.org
You can check the compression speed for your system by running ./nftest .
---
## General Operation of nfdump
The goal of the design is to able to analyze netflow data from
the past as well as to track interesting traffic patterns
continuously. The amount of time back in the past is limited only
by the disk storage available for all the netflow data. The tools
are optimized for speed for efficient filtering. The filter rules
should look familiar to the syntax of tcpdump ( pcap compatible ).
All data is stored to disk, before it gets analyzed. This separates
the process of storing and analyzing the data.
The data is organized in a time-based fashion. Every n minutes
- typically 5 min - nfcapd rotates and renames the output file
with the timestamp nfcapd.YYYYMMddhhmm of the interval e.g.
nfcapd.200907110845 contains data from July 11th 2009 08:45 onward.
Based on a 5min time interval, this results in 288 files per day.
Analyzing the data can be done for a single file, or by concatenating
several files for a single output. The output is either ASCII text
or binary data, when saved into a file, ready to be processed again
with the same tools.
You may have several netflow sources - let's say 'router1' 'router2'
and so on. The data is organized as follows:
/flow_base_dir/router1
/flow_base_dir/router2
which means router1 and router2 are subdirs of the flow_base_dir.
Although several flow sources can be sent to a single collector,
It's recommended to have multiple collector on busy networks for
each source.
Example: Start two collectors on different ports:
nfcapd -w -D -S 2 -B 1024000 -l /flow_base_dir/router1 -p 23456
nfcapd -w -D -S 2 -B 1024000 -l /flow_base_dir/router2 -p 23457
nfcapd can handle multiple flow sources.
All sources can go into a single file or can be split:
All into the same file:
nfcapd -w -D -S 2 -l /flow_base_dir/routers -p 23456
Collected on one port and split per source:
nfcapd -w -D -S 2 -n router1,172.16.17.18,/flow_base_dir/router1 \-n router2,172.16.17.20,/flow_base_dir/router2 -p 23456
See nfcapd(1) for a detailed explanation of all options.
Security: none of the tools requires root privileges, unless you have
a port < 1024. However, there is no access control mechanism in nfcapd.
It is assumed, that host level security is in place to filter the
proper IP addresses.
See the manual pages or use the -h switch for details on using each of
the programs. For any questions send email to peter@people.ops-trust.net
Configure your router to export netflow. See the relevant documentation
for your model.
A generic Cisco sample configuration enabling NetFlow on an interface:
ip address 192.168.92.162 255.255.255.224
interface fastethernet 0/0
ip route-cache flow
To tell the router where to send the NetFlow data, enter the following
global configuration command:
ip flow-export 192.168.92.218 9995
ip flow-export version 5
ip flow-cache timeout active 5
This breaks up long-lived flows into 5-minute segments. You can choose
any number of minutes between 1 and 60;
Netflow v9 full export example of a cisco 7200 with sampling enabled:
interface Ethernet1/0
ip address 192.168.92.162 255.255.255.224
duplex half
flow-sampler my-map
!
!
flow-sampler-map my-map
mode random one-out-of 5
!
ip flow-cache timeout inactive 60
ip flow-cache timeout active 1
ip flow-capture fragment-offset
ip flow-capture packet-length
ip flow-capture ttl
ip flow-capture vlan-id
ip flow-capture icmp
ip flow-capture ip-id
ip flow-capture mac-addresses
ip flow-export version 9
ip flow-export template options export-stats
ip flow-export template options sampler
ip flow-export template options timeout-rate 1
ip flow-export template timeout-rate 1
ip flow-export destination 192.168.92.218 9995
See the relevant documentation for a full description of netflow commands
Note: Netflow version v5 and v7 have 32 bit counter values. The number of
packets or bytes may overflow this value, within the flow-cache timeout
on very busy routers. To prevent overflow, you may consider to reduce the
flow-cache timeout to lower values. All nfdump tools use 64 bit counters
internally, which means, all aggregated values are correctly reported.
The binary format of the data files is netflow version independent.
For speed reasons the binary format is machine architecture dependent, and
as such can not be exchanged between little and big endian systems.
Internally nfdump does all processing IP protocol independent, which means
everything works for IPv4 as well as IPv6 addresses.
See the nfdump(1) man page for details.
netflow version 9:
nfcapd supports a large range of netflow v9 tags. Version 1.6 nfdump
supports the following fields. This list can be found in netflow_v9.h
---
### Flowset record types
Tag | ID
----|---
NF9_IN_BYTES | 1
IN_PACKETS | 2
NF9_FLOWS_AGGR | 3
NF9_IN_PROTOCOL | 4
NF9_SRC_TOS | 5
NF9_TCP_FLAGS | 6
NF9_L4_SRC_PORT | 7
NF9_IPV4_SRC_ADDR | 8
NF9_SRC_MASK | 9
NF9_INPUT_SNMP | 10
NF9_L4_DST_PORT | 11
NF9_IPV4_DST_ADDR | 12
NF9_DST_MASK | 13
NF9_OUTPUT_SNMP | 14
NF9_V4_NEXT_HOP | 15
NF9_SRC_AS | 16
NF9_DST_AS | 17
NF9_BGP_V4_NEXT_HOP | 18
NF9_LAST_SWITCHED | 21
NF9_FIRST_SWITCHED | 22
NF9_OUT_BYTES | 23
NF9_OUT_PKTS | 24
NF9_IPV6_SRC_ADDR | 27
NF9_IPV6_DST_ADDR | 28
NF9_IPV6_SRC_MASK | 29
NF9_IPV6_DST_MASK | 30
NF9_IPV6_FLOW_LABEL | 31
NF9_ICMP_TYPE | 32
NF9_SAMPLING_INTERVAL | 34
NF9_SAMPLING_ALGORITHM | 35
NF9_ENGINE_TYPE | 38
NF9_ENGINE_ID | 39
NF9_FLOW_SAMPLER_ID | 48
FLOW_SAMPLER_MODE | 49
NF9_FLOW_SAMPLER_RANDOM_INTERVAL | 50
NF9_MIN_TTL | 52
NF9_MAX_TTL | 53
NF9_IPV4_IDENT | 54
NF9_DST_TOS | 55
NF9_IN_SRC_MAC | 56
NF9_OUT_DST_MAC | 57
NF9_SRC_VLAN | 58
NF9_DST_VLAN | 59
NF9_DIRECTION | 61
NF9_V6_NEXT_HOP | 62
NF9_BPG_V6_NEXT_HOP | 63
// NF9_V6_OPTION_HEADERS | 64
NF9_MPLS_LABEL_1 | 70
NF9_MPLS_LABEL_2 | 71
NF9_MPLS_LABEL_3 | 72
NF9_MPLS_LABEL_4 | 73
NF9_MPLS_LABEL_5 | 74
NF9_MPLS_LABEL_6 | 75
NF9_MPLS_LABEL_7 | 76
NF9_MPLS_LABEL_8 | 77
NF9_MPLS_LABEL_9 | 78
NF9_MPLS_LABEL_10 | 79
NF9_IN_DST_MAC | 80
NF9_OUT_SRC_MAC | 81
NF9_FORWARDING_STATUS | 89
NF9_BGP_ADJ_NEXT_AS | 128
NF9_BGP_ADJ_PREV_AS | 129
### CISCO ASA NSEL extension - Network Security Event Logging__
Tag | ID
----|---
NF_F_FLOW_BYTES | 85
NF_F_CONN_ID | 148
NF_F_FLOW_CREATE_TIME_MSEC | 152
NF_F_ICMP_TYPE | 176
NF_F_ICMP_CODE | 177
NF_F_ICMP_TYPE_IPV6 | 178
NF_F_ICMP_CODE_IPV6 | 179
NF_F_FWD_FLOW_DELTA_BYTES | 231
NF_F_REV_FLOW_DELTA_BYTES | 232
NF_F_FW_EVENT84 | 233
NF_F_EVENT_TIME_MSEC | 323
NF_F_INGRESS_ACL_ID | 33000
NF_F_EGRESS_ACL_ID | 33001
NF_F_FW_EXT_EVENT | 33002
NF_F_USERNAME | 40000
NF_F_XLATE_SRC_ADDR_IPV4 | 40001
NF_F_XLATE_DST_ADDR_IPV4 | 40002
NF_F_XLATE_SRC_PORT | 40003
NF_F_XLATE_DST_PORT | 40004
NF_F_FW_EVENT | 40005
### Cisco ASR 1000 series NEL extension - Nat Event Logging__
Tag | ID
----|---
NF_N_NAT_EVENT | 230
NF_N_INGRESS_VRFID | 234
NF_N_EGRESS_VRFID | 235
NF_N_NAT_INSIDE_GLOBAL_IPV4 | 225
NF_N_NAT_OUTSIDE_GLOBAL_IPV4 | 226
NF_N_POST_NAPT_SRC_PORT | 227
NF_N_POST_NAPT_DST_PORT | 228
### latency extensions for nfpcapd and nprobe__
Tag | ID
----|---
NF9_NPROBE_CLIENT_NW_DELAY_SEC | 57554
NF9_NPROBE_CLIENT_NW_DELAY_USEC | 57555
NF9_NPROBE_SERVER_NW_DELAY_SEC | 57556
NF9_NPROBE_SERVER_NW_DELAY_USEC | 57557
NF9_NPROBE_APPL_LATENCY_SEC | 57558
NF9_NPROBE_APPL_LATENCY_USEC | 57559
32 and 64 bit counters are supported for any counters. However, internally
nfdump stores packets and bytes counters always as 64bit counters.
16 and 32 bit AS numbers are supported.
Extensions: nfcapd supports a large number of v9 tags. In order to optimise
disk space and performance, v9 tags are grouped into a number of extensions
which may or may not be stored into the data file. Therefore the v9 templates configured on the exporter may be tuned with the collector. Only the tags common to both are stored into the data files. Extensions can be switch on/off by using the -T option. If you want to collect all data, use __-Tall__
### Sampling
By default, the sampling rate is set to 1 (unsampled) or to
any given value specified by the -s cmd line option. If sampling information is found
in the netflow stream, it overwrites the default value. Sampling is automatically
recognised when announced in v9 option templates (tags #48, #49, #50 ), (tag #34, #35)
or in the unofficial v5 header hack.
Note: Not all platforms (or IOS versions) support exporting sampling information in
netflow data, even if sampling is configured. The number of bytes/packets in each
netflow record is automatically multiplied by the sampling rate. The total number of
flows is not changed as this is not accurate enough. (Small flows versus large flows)
### InfluxDB
You can send nfprofile stats data to an influxdb database. The data are the same of rrd files.
For enable this option you need libcurl dev package installed, use --enable-influxdb for configure the project and the nfprofile command should be invoked with option: -i .
Example: -i http://localhost:8086/write?db=mydb&u=user&p=pass
The parameters for auth (&u=user&p=pass) are optional.
Then you get the stats data on influxdb mydb in the measurement nfsen_stats.
For put the stats of live profile you need to apply a patch to nfsen (in extra/nfsen) and add in nfsen.conf the option:
$influxdb_url="http://mydbhost.local:8086/write?db=nfsen";
as example I added a preconfigured grafana dashboard in extra/grafana/Nfsen_Stats.json .
---
For more information, see the GitHub Wiki
nfdump-1.6.23/autogen.sh 0000775 0000000 0000000 00000004145 14045010307 0015073 0 ustar 00root root 0000000 0000000 #!/bin/sh
set -e
case "$(uname)" in
Darwin)
LIBTOOLIZE=${LIBTOOLIZE:-glibtoolize}
;;
*)
LIBTOOLIZE=${LIBTOOLIZE:-libtoolize}
;;
esac
AUTORECONF=${AUTORECONF:-autoreconf}
ACLOCAL=${ACLOCAL:-aclocal}
AUTOCONF=${AUTOCONF:-autoconf}
AUTOHEADER=${AUTOHEADER:-autoheader}
AUTOMAKE=${AUTOMAKE:-automake}
# Check we have all tools installed
check_command() {
command -v "${1}" > /dev/null 2>&1 || {
>&2 echo "autogen.sh: could not find \`$1'. \`$1' is required to run autogen.sh."
exit 1
}
}
check_command "$LIBTOOLIZE"
check_command "$AUTORECONF"
check_command "$ACLOCAL"
check_command "$AUTOCONF"
check_command "$AUTOHEADER"
check_command "$AUTOMAKE"
# Absence of pkg-config or misconfiguration can make some odd error
# messages, we check if it is installed correctly. See:
# https://blogs.oracle.com/mandy/entry/autoconf_weirdness
#
# We cannot just check for pkg-config command, we need to check for
# PKG_* macros. The pkg-config command can be defined in ./configure,
# we cannot tell anything when not present.
check_pkg_config() {
grep -q '^AC_DEFUN.*PKG_CHECK_MODULES' aclocal.m4 || {
cat <&2
autogen.sh: could not find PKG_CHECK_MODULES macro.
Either pkg-config is not installed on your system or
\`pkg.m4' is missing or not found by aclocal.
If \`pkg.m4' is installed at an unusual location, re-run
\`autogen.sh' by setting \`ACLOCAL_FLAGS':
ACLOCAL_FLAGS="-I /share/aclocal" ./autogen.sh
EOF
exit 1
}
}
echo "autogen.sh: reconfigure with autoreconf"
${AUTORECONF} -vif -I m4 || {
echo "autogen.sh: autoreconf has failed ($?), let's do it manually"
[ -f ./configure.ac ] || [ -f ./configure.in ] || continue
echo "autogen.sh: configure `basename $PWD`"
${ACLOCAL} -I m4 ${ACLOCAL_FLAGS}
check_pkg_config
${LIBTOOLIZE} --automake --copy --force
${ACLOCAL} -I m4 ${ACLOCAL_FLAGS}
${AUTOCONF} --force
${AUTOHEADER}
${AUTOMAKE} --add-missing --copy --force-missing
}
echo "autogen.sh: for the next step, run './configure' [or './configure --help' to check available options]"
exit 0
nfdump-1.6.23/bin/ 0000775 0000000 0000000 00000000000 14045010307 0013636 5 ustar 00root root 0000000 0000000 nfdump-1.6.23/bin/AddExtension.txt 0000775 0000000 0000000 00000000542 14045010307 0016770 0 ustar 00root root 0000000 0000000 Add extensions:
===============
1. Add appropriate definitions for new extension in nffile.h
2. Update extension_descriptor array in nfx.c to reflect the new extension.
Increase user index ( 3rd value )
3. Add filter in grammar.y
4. test filter with nftest.
5. nffile_inline.c
6. nf_common.c define printing functions
7. nfsta.c add stat definitions
nfdump-1.6.23/bin/Makefile.am 0000775 0000000 0000000 00000010340 14045010307 0015673 0 ustar 00root root 0000000 0000000
bin_PROGRAMS = nfcapd nfdump nfreplay nfexpire nfanon
check_PROGRAMS = nftest nfgen nfreader
EXTRA_DIST = applybits_inline.c nffile_inline.c collector_inline.c inline.c nfdump_inline.c heapsort_inline.c test.sh nfdump.test.out nfdump.test.diff
check_PROGRAMMS = test.sh
TESTS = nftest test.sh
if SFLOW
bin_PROGRAMS += sfcapd
endif
if NFPROFILE
bin_PROGRAMS += nfprofile
endif
if NFTRACK
bin_PROGRAMS += nftrack
endif
if FT2NFDUMP
bin_PROGRAMS += ft2nfdump
endif
BUILT_SOURCES = grammar.h
AM_YFLAGS = -d
AM_CPPFLAGS = $(DEPS_CFLAGS)
LDADD = $(DEPS_LIBS)
#Add extra debug info for gdb
AM_CFLAGS = -ggdb
# libnfdump sources
output = output_util.h output_util.c output_raw.h output_raw.c
output += output_json.c output_json.h output_csv.c output_csv.h output_pipe.c output_pipe.h
output += output_fmt.c output_fmt.h
util = util.c util.h
filelzo = minilzo.c minilzo.h lzoconf.h lzodefs.h lz4.c lz4.h
nffile = nffile.c nffile.h nfx.c nfx.h
nflist = flist.c flist.h fts_compat.c fts_compat.h
filter = grammar.y scanner.l nftree.c nftree.h ipconv.c ipconv.h rbtree.h
exporter = exporter.c exporter.h
nfprof = nfprof.c nfprof.h
nfnet = nfnet.c nfnet.h
anon = panonymizer.c panonymizer.h rijndael.c rijndael.h
collector = collector.c collector.h
nfv1 = netflow_v1.c netflow_v1.h
nfv9 = netflow_v9.c netflow_v9.h
# pcaproc = pcaproc.c pcaproc.h flowtree.c flowtree.h ipfrag.c ipfrag.h malloc_hook.c
pcaproc = pcaproc.c pcaproc.h flowtree.c flowtree.h ipfrag.c ipfrag.h
content = content_dns.c content_dns.h
netflow_pcap = netflow_pcap.c netflow_pcap.h
ipfix = ipfix.c ipfix.h
nfv5v7 = netflow_v5_v7.c netflow_v5_v7.h
nfstatfile = nfstatfile.c nfstatfile.h
nflowcache = nflowcache.c nflowcache.h
bookkeeper = bookkeeper.c bookkeeper.h
expire= expire.c expire.h
launch = launch.c launch.h
lib_LTLIBRARIES = libnfdump.la
libnfdump_la_SOURCES = $(output) $(util) $(filelzo) $(nffile) $(nflist) $(filter) $(exporter)
libnfdump_la_LDFLAGS = -release 1.6.23
nfdump_SOURCES = nfdump.c nfdump.h nfstat.c nfstat.h nfexport.c nfexport.h \
$(nflowcache) $(nfprof)
nfdump_LDADD = libnfdump.la
nfreplay_SOURCES = nfreplay.c $(nfprof) \
$(nfnet) $(collector) $(nfv1) $(nfv9) $(nfv5v7) $(ipfix)
nfreplay_LDADD = libnfdump.la
nfprofile_SOURCES = nfprofile.c profile.c profile.h $(nfstatfile)
nfprofile_LDADD = libnfdump.la -lrrd
nftrack_SOURCES = ../extra/nftrack/nftrack.c \
../extra/nftrack/nftrack_rrd.c ../extra/nftrack/nftrack_rrd.h \
../extra/nftrack/nftrack_stat.c ../extra/nftrack/nftrack_stat.h
nftrack_CFLAGS = -I ../extra/nftrack
nftrack_LDADD = libnfdump.la -lrrd
nfcapd_SOURCES = nfcapd.c \
$(nfstatfile) $(launch) \
$(nfnet) $(collector) $(nfv1) $(nfv5v7) $(nfv9) $(ipfix) $(bookkeeper) $(expire)
nfcapd_LDADD = libnfdump.la
nfpcapd_SOURCES = nfpcapd.c \
$(pcaproc) $(netflow_pcap) \
$(nfstatfile) $(launch) \
$(nfnet) $(collector) $(bookkeeper) $(expire) $(content)
nfpcapd_LDADD = libnfdump.la
if READPCAP
nfcapd_CFLAGS = -DPCAP
nfcapd_SOURCES += pcap_reader.c pcap_reader.h
nfcapd_LDADD += -lpcap
endif
if BUILDNFPCAPD
bin_PROGRAMS += nfpcapd
nfpcapd_CFLAGS = -D_BSD_SOURCE -D_DEFAULT_SOURCE
nfpcapd_LDADD += -lpcap
nfpcapd_LDFLAGS = -pthread
endif
sfcapd_SOURCES = sfcapd.c sflow_nfdump.c sflow_nfdump.h sflow.h sflow_v2v4.h sflow_process.c sflow_process.h\
$(nfstatfile) $(launch) \
$(nfnet) $(collector) $(bookkeeper) $(expire)
sfcapd_LDADD = libnfdump.la
if READPCAP
sfcapd_CFLAGS = -DPCAP
sfcapd_SOURCES += pcap_reader.c pcap_reader.h
sfcapd_LDADD += -lpcap
endif
nfreader_SOURCES = nfreader.c
nfreader_LDADD = libnfdump.la
nfanon_SOURCES = nfanon.c $(anon)
nfanon_LDADD = libnfdump.la
nfgen_SOURCES = nfgen.c
nfgen_LDADD = libnfdump.la
nfexpire_SOURCES = nfexpire.c \
$(bookkeeper) $(expire) $(nfstatfile)
nfexpire_LDADD = libnfdump.la @FTS_OBJ@
nftest_SOURCES = nftest.c
nftest_LDADD = libnfdump.la
nftest_DEPENDENCIES = nfgen
if FT2NFDUMP
ft2nfdump_SOURCES = ft2nfdump.c
ft2nfdump_CFLAGS = @FT_INCLUDES@
ft2nfdump_LDADD = libnfdump.la -lft -lz
ft2nfdump_LDADD += @FT_LDFLAGS@
endif
check_DIST = inline.c collector_inline.c nffile_inline.c nfdump_inline.c heapsort_inline.c applybits_inline.c
check_DIST += test.sh nfdump.test.out parse_csv.pl AddExtension.txt nfdump.test.diff
CLEANFILES = lex.yy.c grammar.c grammar.h scanner.c scanner.h $(check_PROGRAMS) *.gch
nfdump-1.6.23/bin/anon.flows 0000664 0000000 0000000 00000011770 14045010307 0015653 0 ustar 00root root 0000000 0000000 ¥ N `‹E l“æ E åKŠE Œ òÏ ´3 ¤¢å ½ &Ä