ipfdemo_example.c Source File

Reference Documentation

Platform
Intel® PAC
Napatech SmartNIC
Content Type
Reference Information
Capture Software Version
Link™ Capture Software 12.10
Napatech Software Suite: examples/net/ipfdemo/ipfdemo_example.c Source File
ipfdemo_example.c
Go to the documentation of this file.
1 /*
2  *
3  * Copyright 2023 Napatech A/S. All Rights Reserved.
4  *
5  * 1. Copying, modification, and distribution of this file, or executable
6  * versions of this file, is governed by the terms of the Napatech Software
7  * license agreement under which this file was made available. If you do not
8  * agree to the terms of the license do not install, copy, access or
9  * otherwise use this file.
10  *
11  * 2. Under the Napatech Software license agreement you are granted a
12  * limited, non-exclusive, non-assignable, copyright license to copy, modify
13  * and distribute this file in conjunction with Napatech SmartNIC's and
14  * similar hardware manufactured or supplied by Napatech A/S.
15  *
16  * 3. The full Napatech Software license agreement is included in this
17  * distribution, please see "NP-0405 Napatech Software license
18  * agreement.pdf"
19  *
20  * 4. Redistributions of source code must retain this copyright notice,
21  * list of conditions and the following disclaimer.
22  *
23  * THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES, EXPRESS OR
24  * IMPLIED, AND NAPATECH DISCLAIMS ALL IMPLIED WARRANTIES INCLUDING ANY
25  * IMPLIED WARRANTY OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, OR OF
26  * FITNESS FOR A PARTICULAR PURPOSE. TO THE EXTENT NOT PROHIBITED BY
27  * APPLICABLE LAW, IN NO EVENT SHALL NAPATECH BE LIABLE FOR PERSONAL INJURY,
28  * OR ANY INCIDENTAL, SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES WHATSOEVER,
29  * INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF PROFITS, CORRUPTION OR
30  * LOSS OF DATA, FAILURE TO TRANSMIT OR RECEIVE ANY DATA OR INFORMATION,
31  * BUSINESS INTERRUPTION OR ANY OTHER COMMERCIAL DAMAGES OR LOSSES, ARISING
32  * OUT OF OR RELATED TO YOUR USE OR INABILITY TO USE NAPATECH SOFTWARE OR
33  * SERVICES OR ANY THIRD PARTY SOFTWARE OR APPLICATIONS IN CONJUNCTION WITH
34  * THE NAPATECH SOFTWARE OR SERVICES, HOWEVER CAUSED, REGARDLESS OF THE THEORY
35  * OF LIABILITY (CONTRACT, TORT OR OTHERWISE) AND EVEN IF NAPATECH HAS BEEN
36  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. SOME JURISDICTIONS DO NOT ALLOW
37  * THE EXCLUSION OR LIMITATION OF LIABILITY FOR PERSONAL INJURY, OR OF
38  * INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY TO YOU.
39  *
40  *
41 
42  */
43 
44 /**
45  * @example net/ipfdemo/ipfdemo_example.c
46  * @section ipfdemo_example_description Description
47  *
48  * This source file is an example of how to write an IP fragment re-assembling code,
49  * using the Napatech FPGA IPFMode feature. It accelerates IP re-assembling by
50  * making it possible to load balance using a 5 tuple hash algoritm instead of
51  * a 2 tuple hash algorithm. This is done for fragments by learning from the first
52  * fragment and use this knowledge to calculate the 5 tuple for belonging fragments.
53  * The complexing case is the situation where the FPGA was not able to learn
54  * and identify a fragment and therefore it is sent to one of the un-matching streams.
55  * In this situation the code needs to exchange the un-matched packet and deliver
56  * it to the right re-assembling thread.
57  *
58  *
59  * The following NTAPI functions are used:
60  * - @ref NT_Init()
61  * - @ref NT_NetRxOpen()
62  * - @ref NT_NTPL()
63  * - @ref NT_NetRxGet()
64  * - @ref NT_NET_GET_PKT_L2_FRAME_TYPE()
65  * - @ref NT_NET_GET_PKT_L3_FRAME_TYPE()
66  * - @ref NT_NET_GET_PKT_L3_FRAGMENTED()
67  * - @ref NT_NET_GET_PKT_DESCRIPTOR_FORMAT()
68  * - @ref NT_NET_GET_PKT_L3_FIRST_FRAG()
69  * - @ref NT_NET_GET_PKT_L2_PTR()
70  * - @ref NT_NET_GET_PKT_L3_OFFSET()
71  * - @ref NT_NET_GET_SEGMENT_TIMESTAMP()
72  * - @ref NT_NET_GET_PKT_IPF_LAST_FRAGMENT()
73  * - @ref NT_NET_GET_PKT_IPF_UNMATCHED_STREAMID()
74  * - @ref NT_NetRxRelease()
75  * - @ref NT_NetRxClose()
76  * - @ref NT_ExplainError()
77  *
78  * @section ipfdemo_example_prerequisites Prerequisites
79  * A Napatech capture accelerator is needed to run this example with IPFMode support.
80  * The ntservice.ini file must have enough HostBuffersRx defined to honor
81  * the requested streams needed when running this demo application.
82  * Below is an example of a minimum ini-file. It will create 20 64MB RX
83  * hostbuffer from NUMA node 0.
84  *
85  * Note that the PacketDescriptor is set to Ext9, This is needed to run
86  * IPFMode at all. If anything less is specified, then the IPFMode will
87  * not be used and it will revert into a Hash2Tuple mode.
88  *
89  * Note2 that the TimeSyncReferencePriority is set to OsTime. This is critical, when
90  * using a fragtimeout value (default), because the timestamp of each
91  * packet is used to calculate its timeout.
92  * @code
93  * [System]
94  * TimestampFormat = NATIVE_UNIX
95  *
96  * [Adapter0]
97  * AdapterType = NT20E2
98  * BusId = 00:02:00:00
99  * PacketDescriptor = Ext9
100  * HostBuffersRx = [20,64,0]
101  * TimeSyncReferencePriority = OsTime
102  * @endcode
103  *
104  *
105  * ************************* Overview of the algorithm used ******************************
106  *
107  * The FPGA enables you to match IP fragments and distribute them using a multi CPU
108  * buffer splitting scheme based on a 5-tuple hash algorithm when using the IPFMode feature.
109  * If a multi CPU splitting scheme based on a 2-tuple hash algorithm gives good enough
110  * distribution, then the IPFMode feature in the FPGA should not be used. All belonging
111  * fragments will end up in same stream using a 2-tuple hash algorithm.
112  * IPFMode: how it works:
113  * - Learns from first fragment
114  * - First fragment contains streamID of unmatched fragments
115  * - Timeout setup for releasing learned entry
116  *
117  * @code
118  * Graphical overview:
119  *
120  * +--------------+
121  * | DOI hash |
122  * +------>| |------+
123  * | | tables | |
124  * | +--------------+ |
125  * | |
126  * | |
127  * +----------------+ | | +----------------+
128  * | |------+ +----->| |
129  * | Reassembling | +--------------+ | Unmatched |
130  * | | | Return | | |
131  * | threads |------------->| msg box |------------>| threads |
132  * | | | FIFOs | | |
133  * | |<-----+ +--------------+ +------| |
134  * +----------------+ | | +----------------+
135  * ^ ^ ^ | | ^ ^
136  * | | | | | | |
137  * | | | | | | |
138  * | | | | +--------------+ | | |
139  * | | | | | Msg box | | | |
140  * | | | +-------| |<-----+ | |
141  * | | | | FIFOs | | |
142  * | | | +--------------+ | |
143  * | | | | |
144  * 5 tuple hash splitting 2 tuple hash splitting
145  * | | | | |
146  * | | | | |
147  * +-------------------------------------------------------------------------------+
148  * | |
149  * | NTAPI/FPGA |
150  * | |
151  * +-------------------------------------------------------------------------------+
152  *
153  * @endcode
154  *
155  * Description of algorithm:
156  *
157  *
158  * Threads:
159  * --------
160  *
161  * Unmatched threads – variable number (N) – number of unmatched streams to use
162  * (2-tuple hash cpu-splitting)
163  *
164  * Each unmatched thread receives the unmatched fragments. When a fragment is received,
165  * it is checked in the DOI tables to find a datagram ID match. If a match is found, the
166  * NetBuf containing the fragment is send to that reassembling Msg box FIFO, of which
167  * the DOI table belongs to. Otherwise, if no match found (yet), the fragment is
168  * temporarily put into a wait list. The wait list is periodically checked against the
169  * DOI tables.
170  *
171  * Reassembling threads – variable number (M) – number of streams to use
172  * (5-tuple hash cpu-splitting)
173  *
174  * When each reassembling thread receives IP fragments, it collects them into complete
175  * datagrams. If all fragments are received in order, or at least the first fragment is
176  * received first, then the need for unmatched fragment collection is not needed and all
177  * datagrams are reassembled in the reassembling threads.
178  * When the reassembling thread receives a first fragment, it then stores that fragment
179  * in its local collection hash table (tbl) and makes that datagram ID available for the
180  * specific unmatched thread to inform about its interest in fragments belonging to this
181  * datagram, using the specific DOI table connecting the reassembling thread with the
182  * unmatched thread.
183  * When all fragments are received for a datagram, they are all released and the DOI
184  * table entry is cleared.
185  * Each reassembling thread first reads from the Msg box for a NetBuf, and if none found,
186  * then it reads the NTAPI stream for a NetBuf. If any fragments are received from here,
187  * they are put into the collection table (tbl).
188  *
189  *
190  * Inter-thread communication messages:
191  * ------------------------------------
192  *
193  * Msg boxes (FIFOs):
194  *
195  * Used by Unmatched threads to send unmatched fragments to the reassembling threads.
196  * [[[unm1 Msgbox],[ unm2 Msgbox],…[ unmN Msgbox]], [[unm1 Msgbox],[ unm2 Msgbox],…[ unmN Msgbox]]…M times]
197  * reasm1 reasm2 …reasmM
198  *
199  * The Msg boxes are built as FIFOs and are used by the unmatched threads to send the
200  * NetBuf of a received datagram fragment to the belonging reassembling thread.
201 
202  * Return Msg boxes (FIFOs):
203  *
204  * Used by Reassmbling threads to send unmatched-fragment NetBufs back to the unmatched
205  * threads originally received them.
206  * [[[reasm1 Msgbox],[ reasm2 Msgbox],…[ reasmM Msgbox]], [[reasm1 Msgbox],[ reasm2 Msgbox],…[ reasmN Msgbox]]…N times]
207  * unm1 unm2 …unmN
208  *
209  * This FIFO is only needed to make the complete algorithm lockless. The NTAPI cannot
210  * handle multi-threaded access to the packet interface without serialization.
211 
212  * DOI tables (Datagram of interest):
213  *
214  * The DOI tables are a set of hash tables containing a list of datagram IDs that the
215  * reassembling threads are interested in.
216  *
217  * The FPGA specifies the unmatched stream ID together with the reception of the first
218  * fragment. This information is used by the reassembling threads to inform the specific
219  * unmatched thread about where to send unmatched fragments matching the first fragments
220  * datagram ID.
221  *
222  * Each reassembling thread has one DOI table allocated for each unmatched thread, thus
223  * M*N DOI hash tables are used. This way each reassembling thread has exclusive write
224  * access to one table for each unmatched tread.
225  * [[[reasm1 tbl],[reasm2 tbl],…[reasmM tbl]], [[reasm1 tbl],[reasm2 tbl],…[reasmM tbl]]…N times]
226  * Unmatched1 Unmatched2 …UnmatchedN
227  *
228  * Each unmatched thread only reads information from the DOI tables. When an unmatched
229  * fragment matches an entry in the DOI table, then this fragment is send to the
230  * reassembling thread using the dedicated Msg box FIFO.
231  *
232  * Datagram ID calculation:
233  *
234  * The fragments belonging to a datagram are identified by the calculated datagram ID:
235  * It consists of information from 4 fields in the IP header (source IP, destination IP,
236  * Identification and protocol).
237 
238  *<hr>
239  * @section ipfdemo_example_code Code
240  * @}
241  */
242 
243 // Include this in order to access the Napatech API
244 #include <nt.h>
245 
246 #if defined(__linux__) || defined(__FreeBSD__)
247  #include <unistd.h>
248  #include <signal.h>
249  #include <assert.h>
250  #include <pthread.h>
251  #include <arpa/inet.h>
252  #include <sys/time.h>
253 #elif defined(WIN32) || defined (WIN64)
254  #include <winsock2.h> // ntohs()
255  #include <time.h>
256  #include <sys/timeb.h>
257  #include <process.h> // threading
258 #endif
259 
260 #include <argparse.h>
261 
262 #if defined(WIN32) || defined (WIN64)
263  #define snprintf(dst, ...) _snprintf_s((dst), _countof(dst), __VA_ARGS__)
264 
265 #if defined(_MSC_VER)
266  // Time
267  struct timezone {
268  int tz_minuteswest; /* minutes west of Greenwich */
269  int tz_dsttime; /* type of DST correction */
270  };
271 #endif // _MSC_VER
272 
273  int gettimeofday(struct timeval *tv, struct timezone *tz);
274 
275  void sleep(int time);
276  void usleep(unsigned long usec);
277 
278  // Threading
279  typedef HANDLE pthread_t;
280  typedef unsigned (__stdcall *start_address_t)(void *parameter);
281  int pthread_create(HANDLE *thread, DWORD *attr, start_address_t start_routine, void *parameter);
282  int pthread_join(HANDLE thread, void **value_ptr);
283 
284 #endif
285 
286 #pragma pack(push, 1)
287 /* IPv4 header structure */
288 typedef struct iphdr_s {
289  uint8_t ipHlen:4; // little endian layout
290  uint8_t ipVer:4; // little endian layout
291  uint8_t tos;
292  uint16_t ipTotlen;
293  uint16_t Id;
294 #define FRAG_OFFS_MASK 0x1FFF
295 #define LAST_FRAG_BITS 0x00E0
296  uint16_t fragOffs;
297  uint8_t ttl;
298  uint8_t prot;
299  uint16_t csum;
300  uint32_t srcAddr;
301  uint32_t dstAddr;
302 } iphdr_t;
303 #pragma pack(pop)
304 
305 #define L3_ADDR(_NetBuf_) ((uint8_t *)NT_NET_GET_PKT_L2_PTR(_NetBuf_)+NT_NET_GET_PKT_L3_OFFSET(_NetBuf_))
306 #define IPVERSION(_NetBuf_) (((iphdr_t *)L3_ADDR(_NetBuf_))->ipVer)
307 
308 /* macros for IPv4 packets */
309 #define IPV4_HDR_LENGTH(_NetBuf_) ((uint8_t)(((iphdr_t *)L3_ADDR(_NetBuf_))->ipHlen)<<2)
310 #define IPV4_TOT_LEN(_NetBuf_) (ntohs(((iphdr_t *)L3_ADDR(_NetBuf_))->ipTotlen))
311 #define IPV4_FRAGMENT_ID(_NetBuf_) (ntohs(((iphdr_t *)L3_ADDR(_NetBuf_))->Id))
312 #define IPV4_FRAGMENT_OFFSET(_NetBuf_) ((ntohs(((iphdr_t *)L3_ADDR(_NetBuf_))->fragOffs)&FRAG_OFFS_MASK)<<3)
313 #define IPV4_LAST_FRAG(_NetBuf_) (((((iphdr_t *)L3_ADDR(_NetBuf_))->fragOffs)&LAST_FRAG_BITS)==0)
314 #define IPV4_SRC_ADDR(_NetBuf_) (((iphdr_t *)L3_ADDR(_NetBuf_))->srcAddr)
315 #define IPV4_DST_ADDR(_NetBuf_) (((iphdr_t *)L3_ADDR(_NetBuf_))->dstAddr)
316 #define IPV4_PROTOCOL(_NetBuf_) (((iphdr_t *)L3_ADDR(_NetBuf_))->prot)
317 
318 #define IPV4_GET_DGRAM_ID(_NetBuf_, _src_, _dst_, _id_, _prot_) \
319  do {iphdr_t *_ip_=(iphdr_t *)L3_ADDR(_NetBuf_);_src_=_ip_->srcAddr;_dst_=_ip_->dstAddr;\
320  _id_=_ip_->Id;_prot_=_ip_->prot;}while(0)
321 
322 #define IPV4_DATA_LEN(_NetBuf_) (IPV4_TOT_LEN(_NetBuf_)-IPV4_HDR_LENGTH(_NetBuf_))
323 
324 
325 /* Help structure to concatenate ID source fields */
326 typedef union _DOI_dgramId_u {
327  volatile uint64_t src_id;
328  struct {
329  volatile uint32_t src;
330  volatile uint32_t id;
331  };
332 } DOI_dgramId_u;
333 
334 #define MAX_SRC_ID 8
335 typedef struct _DOI_dgramTbl_t {
336  volatile uint64_t aSrc_id[MAX_SRC_ID]; // Source and frag ID concatenated into ID1
337  volatile uint64_t aDst_pr[MAX_SRC_ID]; // Destination and protocol concatenated into ID2
339 
340 #define DOI_FRAG_TBL_SIZE 8081 /* Prime number */
341 #define DOI_HASH_GET_KEY(_entry_, _src_, _id_) (_entry_ = (_src_+_id_)%DOI_FRAG_TBL_SIZE)
342 
343 
344 /* Re-assembler hash table types and definitions */
345 struct frag_s {
346  NtNetBuf_t hNetBuf; // hNetBuf containing fragment buffer
347  uint16_t offset; // Fragment offset
348  uint16_t size; // Size of this fragment
349  uint8_t firstFrag; // If marked as first fragment
350  uint8_t lastFrag; // If marked as last fragment
351  uint8_t fromUnm; // Is this packet received from an un-matched thread (needed to lock on release)
352  uint8_t unmIndex; // if from unmatched stream, then this tells which unmatched thread index
353 };
354 
355 #define MAX_FRAG_CNT 18
356 #define REASSEMBLY_HASH_TBL_SIZE 1021 /* Prime number */
357 #define INITIAL_HASH_TBL_ENTRY_CNT 1024
358 typedef struct _hashTbl_entry_s {
359  uint64_t id1; // Src and prot id for ID1
360  uint64_t id2; // Dest and prot Id for ID2
361  volatile uint64_t *pDOI_Src_id; // Source and frag ID field
362  volatile uint64_t *pDOI_Dst_pr; // Dest and protocol ID field
363  uint16_t fragCnt; // Number of fragments in aFrag
364  struct frag_s aFrag[MAX_FRAG_CNT]; // Array of all fragments belonging to the same datagram
365  struct _hashTbl_entry_s *pNext; // Pointer to next element in list
366 } tbl_entry_t;
367 
368 /* Hash table macros */
369 #define LOOKUP_ENTRY(_tbl_base_, _tbl_entry_type_, _tbl_size_, _id1_, _id2_, _tbl_entry_) \
370 { \
371  uint32_t _key_ = (uint32_t)((_id1_ ^ _id2_) % _tbl_size_); \
372  _tbl_entry_type_ *_tbl_; \
373  _tbl_ = _tbl_base_[_key_]; \
374  while (_tbl_ && (_tbl_->id1 != _id1_ || _tbl_->id2 != _id2_)) { \
375  _tbl_ = _tbl_->pNext; \
376  } \
377  _tbl_entry_=_tbl_; \
378 }
379 
380 #define GET_NEW_TBL_ENTRY(_tbl_free_, _tbl_entry_type_, _tbl_entry_) \
381 { \
382  if (!_tbl_free_) { \
383  int __o__; \
384  _tbl_entry_type_ *__plm__; \
385  for (__o__ = 0; __o__ < 64; __o__++) { \
386  __plm__ = calloc(1, sizeof(_tbl_entry_type_)); \
387  if (!__plm__) assert(0); \
388  __plm__->pNext = _tbl_free_; \
389  _tbl_free_ = __plm__; \
390  } \
391  } \
392  _tbl_entry_ = _tbl_free_; \
393  _tbl_free_ = _tbl_free_->pNext; \
394  tbl_entry->pNext = NULL; \
395 }
396 
397 #define RELEASE_TBL_ENTRY(_tbl_free_, _tbl_entry_) \
398 { \
399  _tbl_entry_->pNext = _tbl_free_; \
400  _tbl_free_ = _tbl_entry_; \
401 }
402 
403 #define ADD_ENTRY_TO_TBL(_tbl_base_, _tbl_entry_type_, _tbl_size_, _tbl_entry_) \
404 { \
405  uint32_t _key_ = (uint32_t)((_tbl_entry_->id1 ^ _tbl_entry_->id2) % _tbl_size_); \
406  _tbl_entry_type_ *_tbl_; \
407  _tbl_ = _tbl_base_[_key_]; \
408  while (_tbl_ && (_tbl_->id1 != _tbl_entry_->id1 || _tbl_->id2 != _tbl_entry_->id2)) { \
409  _tbl_ = _tbl_->pNext; \
410  } \
411  if (_tbl_ == NULL) { \
412  _tbl_entry_->pNext = _tbl_base_[_key_]; \
413  _tbl_base_[_key_] = _tbl_entry_; \
414  } \
415  else { \
416  assert(_tbl_==_tbl_entry_); \
417  } \
418 }
419 
420 #define DEL_ENTRY_FROM_TBL(_tbl_base_, _tbl_entry_type_, _tbl_size_, _tbl_entry_) \
421 { \
422  uint32_t _key_ = (uint32_t)((_tbl_entry_->id1 ^ _tbl_entry_->id2) % _tbl_size_); \
423  _tbl_entry_type_ *_tbl_, *_prev_ = NULL; \
424  _tbl_ = _tbl_base_[_key_]; \
425  while (_tbl_ && (_tbl_->id1 != _tbl_entry_->id1 || _tbl_->id2 != _tbl_entry_->id2)) { \
426  _prev_ = _tbl_;_tbl_ = _tbl_->pNext; \
427  } \
428  if (_tbl_) { \
429  if (_prev_) { \
430  _prev_->pNext = _tbl_->pNext; \
431  } \
432  else { \
433  _tbl_base_[_key_] = _tbl_->pNext; \
434  } \
435  } \
436  else { \
437  assert(0); \
438  } \
439 }
440 
441 
442 typedef struct _ipDefrag ipDefrag_t;
443 
444 
445 #if defined(__linux__) || defined(__FreeBSD__)
446  #define compiler_barrier() __asm__ __volatile__("":::"memory")
447 #elif defined(WIN32) || defined (WIN64)
448  #define compiler_barrier()
449 #endif
450 
451 
452 /* Msg box definition and control type and macros */
453 #define MSG_BOX_DEPTH 0x2000
454 typedef struct _msg_box_s {
455  volatile uint32_t rdIdx; // Read index of msg box
456  volatile uint32_t wrIdx; // Write index of msg box
457  NtNetBuf_t data[MSG_BOX_DEPTH]; // Msg box elements (pointers to hNetBufs)
458 } msg_box_t;
459 
460 /* Macros to use the msg box */
461 #define MSG_BOX_EMPTY(_fifo_) ((_fifo_)->wrIdx==(_fifo_)->rdIdx)
462 #define MSG_BOX_FULL(_fifo_) (((_fifo_)->wrIdx-(_fifo_)->rdIdx) >= MSG_BOX_DEPTH)
463 #define MSG_BOX_GET(_fifo_) (_fifo_)->data[(_fifo_)->rdIdx&(MSG_BOX_DEPTH-1)]; compiler_barrier(); (_fifo_)->rdIdx++
464 #define MSG_BOX_PUT(_fifo_, _elem_) \
465  do { \
466  (_fifo_)->data[(_fifo_)->wrIdx&(MSG_BOX_DEPTH-1)]=_elem_; \
467  compiler_barrier(); \
468  (_fifo_)->wrIdx++; \
469  } while (0)
470 
471 
472 
473 /* Wait-list type definition */
474 #define MAX_WAIT_ELEM 1024
475 typedef struct wait_list_s {
478 } wait_list_t;
479 
480 /* Un-matched thread variable definition */
481 typedef struct _unmThread_s {
482  pthread_t thread; // This thread
483  int streamId; // NT Stream Id to service by this thread
484  DOI_dgramTbl_t *pDOI; // Datagram Of Interest tables. For this Unmatched Thread only, indexed for each Reassembly Thread
485  msg_box_t *pMsgboxReturn; // All msg boxes used by the current unmatched thread indexed [reasmCnt]
486  wait_list_t WaitListElem[MAX_WAIT_ELEM]; // All awailable wait-list elements
487  wait_list_t *pFree; // Linked list of free wait-list elements
488  wait_list_t *pWait; // Linked list of waiting wait-list elements
489  ipDefrag_t *pIpDefrag; // Pointer to main instance variable
490  /* Statistics */
491  int fragRecv; // Counter for all fragments received by this un-matched thread
492  int fragsDeleted; // Counter for all timed out and deleted fragments in wait-list
493 } unmThread_t;
494 
495 
496 /* Re-assembling thread variable definition */
497 typedef struct _reasmThread_s {
498  pthread_t thread; // This thread
499  int idx; // Index number of this re-assembling thread (first re-assembling thread starts with index 0)
500  int streamId; // NT Stream Id to service by this thread
501  tbl_entry_t *tbl[REASSEMBLY_HASH_TBL_SIZE]; // Hash table for the current re-assembling thread
502  tbl_entry_t *tblFree; // Free elements for the hash table
503  msg_box_t *pMsgbox; // All msg boxes used by the current re-assembling thread indexed [unmCnt]
504  ipDefrag_t *pIpDefrag; // Pointer to main instance variable
505  /* statistics */
506  int datagramCompleted; // Counter for datagrams sent in fragments and re-assembled sucessfully
507  int fragmentsTimedout; // Counter for all timed out and deleted fragments in hash table
508  int firstFragRcv; // Counter for all first-fragments received by this re-assembling thread
509  int nonFragments; // Counter for all non-fragmented packets received by this re-assembling stream
510  int Src_IdClash; // Counter for all Id clashes encountered (frags with src,dst,prot,fragId identical)
511  int msgboxPackets; // Counter for remaining msg box entries on exit
512 } reasmThread_t;
513 
514 struct _ipDefrag {
515  int Running; // Application running control variable
516  int adapterNo; // Adapter number to run on
517  int unmCnt; // number of un-matched streams
518  int unmStart; // Value of first un-matched streamid
519  int reasmCnt; // Number of re-assembling streams
520  int reasmStart; // Value of first re-assembling streamId
521  uint64_t fragTimeout; // Fragment timeout setting in ms
522  unmThread_t *pUnmThreads; // All un-matched thread instance pointers
523  DOI_dgramTbl_t *pDOI_Tbls; // Hash tables to notify interest of fragments to the Unmatched Threads - indexed [unmCnt][reasmCnt]
524  reasmThread_t *pReasmThreads; // All re-assembling thread instance pointers
525  msg_box_t *pReasmMsgboxes; // Pointer to all re-assembling threads message boxes
526  msg_box_t *pReasmReturnMsgboxes; // Pointer to all NetBuf return message boxes
527  int ExtDescrType; // The NT extended descriptor type used by FPGA to access and use the correct macros
528  int allReasmClosed; // To let all Unm-threads/streams close after re-assembling threads are closed
529 };
530 
531 
532 #define NO_OPTIONS 0
533 #define OPTION_HELP (1<<1)
534 #define OPTION_ADAPTER (1<<2)
535 #define OPTION_NUM_REASM (1<<3)
536 #define OPTION_NUM_UNM (1<<4)
537 #define OPTION_TIMEOUT (1<<5)
538 #define OPTION_TABLE_PERSIST (1<<6)
539 #define OPTION_TABLE_TIMEOUT (1<<7)
540 
541 static int opt_adapter = -1;
542 static int opt_reasm = -1;
543 static int opt_unm = -1;
544 static int opt_frag = -1;
545 static char *opt_persist = NULL;
546 static int opt_timeout = -1;
547 
548 /**
549  * Table of valid options.
550  */
552  OPT_HELP(),
553  OPT_INTEGER('a', "adapter", &opt_adapter, "The adapter to run tests on", NULL, 0, 0, "adapter number"),
554  OPT_INTEGER('r', "reasm", &opt_reasm, "Number of concurrent IP fragments re-assembling streams/threads", NULL, 0, 0, "number"),
555  OPT_INTEGER('u', "unm", &opt_unm, "Number of concurrent un-matched streams/threads", NULL, 0, 0, "number"),
556  OPT_INTEGER('f', "fragtimeout", &opt_frag, "How old fragments may get before they are deleted from tables (ms)", NULL, 0, 0, "ms"),
557  OPT_STRING( 'p', "tablepersist", &opt_persist, "FPGA TablePersist", NULL, 0, 0, "timeout|lastfragment"),
558  OPT_INTEGER('t', "tabletimeout", &opt_timeout, "FPGA table timeout", NULL, 0, 0, "value"),
559  OPT_END(),
560 };
561 
562 static struct _ipDefrag IpDefrag; // The main instance handle
563 
564 static const char *usageText[] = {
565  "Syntax:\n"
566  "\n"
567  "ipfdemo_example [-a <adapter number>] [-r <number>] [-u <number>] [-f <ms>] [-p <timeout|lastfragment>] [-t <value>]\n"
568  "\nCommands:\n",
569  NULL};
570 
571 /*****************************************************************************
572  Function to get system time in milliseconds
573 ******************************************************************************/
574 static uint64_t GetSystemTimeNoArg(void)
575 {
576  struct timeval tv;
577  gettimeofday(&tv, NULL);
578  return (uint64_t) (tv.tv_sec * 1000000 + tv.tv_usec);
579 }
580 
581 /*****************************************************************************
582  Called from the un-matched threads, to check if any re-assembling threads
583  are waiting for this fragment. It is communicated using the DOI hash table.
584  One table for each re-assembling/un-matched thread combination. This is done
585  to make singular write access to each table (no locks needed).
586 ******************************************************************************/
587 static int SendToReasm(unmThread_t *pUnm, NtNetBuf_t *phNetBuf)
588 {
589  int i;
590  int32_t entry;
591  uint32_t src, dst;
592  uint8_t prot;
593  uint16_t id;
596  msg_box_t *pMsgbox;
597  int idx;
598  NtNetBuf_t hNetBuf = *phNetBuf;
599 
600  IPV4_GET_DGRAM_ID(hNetBuf, src, dst, id, prot);
601 
602  val.src = src;
603  val.id = id;
604  val1.src = dst;
605  val1.id = prot;
606 
607  for (i = 0; i < pUnm->pIpDefrag->reasmCnt; i++) {
608  pTbl = &pUnm->pDOI[i * DOI_FRAG_TBL_SIZE];
609  DOI_HASH_GET_KEY(entry, src, id);
610  pMsgbox = &pUnm->pIpDefrag->pReasmMsgboxes[i * pUnm->pIpDefrag->unmCnt + (pUnm->streamId - pUnm->pIpDefrag->unmStart)];
611 
612  idx=0;
613  while (idx < MAX_SRC_ID) {
614  if (pTbl[entry].aSrc_id[idx] == val.src_id &&
615  pTbl[entry].aDst_pr[idx] == val1.src_id) {
616  if (!MSG_BOX_FULL(pMsgbox)) {
617  MSG_BOX_PUT(pMsgbox, hNetBuf);
618  *phNetBuf = NULL;
619  return 1;
620  } else {
621  /* Msgbox full, we try again later */
622  return 0;
623  }
624  }
625  idx++;
626  }
627  }
628  return 0;
629 }
630 
631 /*****************************************************************************
632  This is the main un-matched thread routine. One thread for each un-matched
633  stream is spawned to handle all the un-matched streams configured.
634  Upon receival of an IP fragment from hardware, this routine searches the DOI
635  tables (restricted to the ones associated with this stream ID) for interest
636  in this fragment from any re-assembling threads. If one is waiting for fragments
637  belonging to this particular datagram, it is send to that re-assembling thread
638  using the msg box mechanism. If none interested (yet), the fragment is temporarily
639  stored in a wait-list. Frequently the wait-list is scanned together with the DOI
640  tables to make sure the fragments are sent to the corresponding re-assembling
641  thread when it raises interest herein.
642  If the wait-list is full, no more fragments are read until the last hNetBuf
643  can be delivered or added to wait-list.
644 ******************************************************************************/
645 #if defined(__linux__) || defined(__FreeBSD__)
646  static void *_UnMatchedThread(void *arg)
647 #elif defined(WIN32) || defined (WIN64)
648  static unsigned __stdcall _UnMatchedThread(void *arg)
649 #endif
650 {
651  unmThread_t *pUnm = (unmThread_t *)arg; // this pointer for this thread
652  char streamName[20]; // Stream name
653  char errorBuffer[NT_ERRBUF_SIZE]; // Error buffer
654  NtNetStreamRx_t hNetRx; // Handle to the RX stream
655  NtNetBuf_t hNetBuf = NULL; // Net buffer container
656  int i; // loop variable
657  int status; // status from NTAPI calls
658  int invalidPkt; // Invalid packet indicator (we only receive IP fragments with these streams)
659  int checkTimeout = 0; // Indicator for fragment timeout check
660  uint64_t sysTs = 0; // System Timestamp
661  int deleteIt; // If we should delete the entry from the wait-list. Is delivered or timed out
662  wait_list_t *pWait; // Help pointer to cleanup wait list on exit
663 
664  snprintf(streamName, sizeof(streamName), "IPFUnm_%i", pUnm->streamId);
665 
666  /* Fill free list with elements for the wait-list */
667  pUnm->WaitListElem[0].pNext = NULL;
668  for (i = 1; i < MAX_WAIT_ELEM; i++) {
669  pUnm->WaitListElem[i].pNext = &pUnm->WaitListElem[i - 1];
670  }
671  pUnm->pFree = &pUnm->WaitListElem[MAX_WAIT_ELEM - 1];
672 
673  // Get a stream handle
674  if ((status = NT_NetRxOpen(&hNetRx, streamName, NT_NET_INTERFACE_PACKET, pUnm->streamId, -1)) != NT_SUCCESS) {
675  // Get the status code as text
676  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
677  fprintf(stderr, "[%i] NT_NetRxOpen() failed: %s\n", pUnm->streamId, errorBuffer);
678 #if defined(__linux__) || defined(__FreeBSD__)
679  return (void *)NULL;
680 #elif defined(WIN32) || defined (WIN64)
681  { _endthreadex(0); return 0; }
682 #endif
683  }
684 
685  while (pUnm->pIpDefrag->Running) {
686 
687  /* Check all Return Msg boxes for fragment packets to be freed */
688  for (i = 0; i < pUnm->pIpDefrag->reasmCnt; i++) {
689  while (!MSG_BOX_EMPTY(&pUnm->pMsgboxReturn[i])) {
690  NtNetBuf_t hNetBuf1;
691  hNetBuf1 = MSG_BOX_GET(&pUnm->pMsgboxReturn[i]);
692  // printf("Releasing unmatched fragment %i (%llx) by thread index %i\n", i, hNetBuf1, pUnm->streamId - pUnm->pIpDefrag->unmStart);
693  NT_NetRxRelease(hNetRx, hNetBuf1);
694  }
695  }
696 
697  /* Check for entries in the wait-list to be delivered to re-assembling thread who has signaled interest through (DOI hash table) */
698  if (pUnm->pWait) {
699  wait_list_t *pNxt, *pPrev = NULL, *pWaitFrag = pUnm->pWait;
700  if (checkTimeout) {
701  sysTs = GetSystemTimeNoArg()*100; // make it 10ns units
702  }
703  /* Loop all elements in list */
704  while (pWaitFrag) {
705  /* Try to send to an interested re-assembly tread */
706  deleteIt = SendToReasm(pUnm, &pWaitFrag->hNetBuf);
707  if (!deleteIt && checkTimeout) {
708  /* check for timed out condition */
709  if (sysTs - NT_NET_GET_PKT_TIMESTAMP(pWaitFrag->hNetBuf) > pUnm->pIpDefrag->fragTimeout) {
710  deleteIt=1;
711  }
712  }
713  /* Remove from list if send or timed out */
714  if (deleteIt) {
715  if (pWaitFrag->hNetBuf) {
716  NT_NetRxRelease(hNetRx, pWaitFrag->hNetBuf);
717  pWaitFrag->hNetBuf = NULL;
718  pUnm->fragsDeleted++;
719  }
720  pNxt = pWaitFrag->pNext;
721  /* Move base if needed */
722  if (pUnm->pWait == pWaitFrag) {
723  pUnm->pWait = pNxt;
724  } else
725  if (pPrev) {
726  pPrev->pNext = pWaitFrag->pNext;
727  }
728  /* Put onto free list */
729  pWaitFrag->pNext = pUnm->pFree;
730  pUnm->pFree = pWaitFrag;
731  /* Step to next element */
732  pWaitFrag = pNxt;
733  } else {
734  /* Jump over element */
735  pPrev = pWaitFrag;
736  pWaitFrag = pWaitFrag->pNext;
737  }
738  }
739  }
740 
741  if (hNetBuf == NULL) {
742  /* Read new fragments from stream */
743  if ((status = NT_NetRxGet(hNetRx, &hNetBuf, 1)) != NT_SUCCESS) {
744  if ((status == NT_STATUS_TIMEOUT) || (status == NT_STATUS_TRYAGAIN)) {
745  if (pUnm->pIpDefrag->fragTimeout) checkTimeout = 1;
746  continue;
747  }
748  // Get the status code as text
749  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
750  fprintf(stderr, "[%i] NT_NetRxGet() failed: %s\n", pUnm->streamId, errorBuffer);
751  #if defined(__linux__) || defined(__FreeBSD__)
752  return (void *)NULL;
753  #elif defined(WIN32) || defined (WIN64)
754  { _endthreadex(0); return 0; }
755  #endif
756  }
757  }
758 
759  assert(hNetBuf != NULL);
760 
761  /* New packet received */
762  invalidPkt = 1;
763 
764  if (NT_NET_GET_PKT_L2_FRAME_TYPE(hNetBuf) == NT_L2_FRAME_TYPE_ETHER_II &&
765  NT_NET_GET_PKT_L3_FRAME_TYPE(hNetBuf) == NT_L3_FRAME_TYPE_IPv4 &&
766  NT_NET_GET_PKT_L3_FRAGMENTED(hNetBuf)) {
767 
768  /* Packet is fragmented and valid for our use */
769  if (NT_NET_GET_PKT_DESCRIPTOR_FORMAT(hNetBuf) == 9) {
770  if (NT_NET_GET_PKT_L3_FIRST_FRAG(hNetBuf)) {
771  fprintf(stderr, "[%i] ERROR un-matched streams may never receive first fragment\n", pUnm->streamId);
772  exit(0);
773  }
774 
775  invalidPkt = 0;
776  if (SendToReasm(pUnm, &hNetBuf) == 0) {
777  /* No match found. Put entry into wait list */
778  if (pUnm->pFree == NULL) {
779  /* We wait for available slots */
780  usleep(1);
781  if (pUnm->pIpDefrag->fragTimeout) checkTimeout = 1;
782  } else {
783  wait_list_t *pNxt;
784  pNxt = pUnm->pFree->pNext;
785  pUnm->pFree->pNext = pUnm->pWait;
786  pUnm->pWait = pUnm->pFree;
787  pUnm->pFree = pNxt;
788  pUnm->pWait->hNetBuf = hNetBuf;
789  /* Buffer deliverd to wait-list */
790  hNetBuf = NULL;
791  }
792  }
793 
794  if (hNetBuf == NULL) {
795  pUnm->fragRecv++;
796  }
797  }
798  }
799 
800  if (invalidPkt && !NT_NET_GET_PKT_L3_FRAGMENTED(hNetBuf)) {
801  fprintf(stderr, "[%i] ERROR, invalid non-ipv4-fragment received on one un-matched fragment stream\n", pUnm->streamId);
802  exit(0);
803  }
804  }
805 
806 /* Exiting section */
807  pWait = pUnm->pWait;
808  while (pWait) {
809  // Release pending fragment packets
810  NT_NetRxRelease(hNetRx, pWait->hNetBuf);
811  pWait = pWait->pNext;
812  }
813 
814  /* All re-assembling threads has to close before we may close the un-matched threads.
815  We need these threads to continue running while freeing hNetBuf's sent from the un-matched streams */
816  while (pUnm->pIpDefrag->allReasmClosed == 0) {
817  usleep(1000);
818  }
819 
820  /* Empty all Return Msg Boxes */
821  for (i = 0; i < pUnm->pIpDefrag->reasmCnt; i++) {
822  while (!MSG_BOX_EMPTY(&pUnm->pMsgboxReturn[i])) {
823  NtNetBuf_t buf = MSG_BOX_GET(&pUnm->pMsgboxReturn[i]);
824  NT_NetRxRelease(hNetRx, buf);
825  }
826  }
827 
828  /* Close NT stream */
829  if ((status = NT_NetRxClose(hNetRx)) != NT_SUCCESS) {
830  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
831  fprintf(stderr, "[%i] Un-matched: NT_NetRxGet() failed: %s\n", pUnm->streamId, errorBuffer);
832  }
833 
834 #if defined(__linux__) || defined(__FreeBSD__)
835  return (void *)NULL;
836 #elif defined(WIN32) || defined (WIN64)
837  { _endthreadex(0); return 0; }
838 #endif
839 }
840 
841 
842 
843 static void _FreeUnmFragment(reasmThread_t *pReasm, int unmIndex, NtNetBuf_t hNetBuf)
844 {
845  msg_box_t *pMsgbox;
846  int waitCnt;
847  pMsgbox = &pReasm->pIpDefrag->pReasmReturnMsgboxes[unmIndex * pReasm->pIpDefrag->reasmCnt + (pReasm->streamId - pReasm->pIpDefrag->reasmStart)];
848 
849  waitCnt=0;
850  while (waitCnt < 100) {
851  if (MSG_BOX_FULL(pMsgbox)) {
852  usleep(1000);
853  waitCnt++;
854  } else {
855  MSG_BOX_PUT(pMsgbox, hNetBuf);
856  break;
857  }
858  };
859 
860  if (waitCnt == 100) {
861  fprintf(stderr, "[%i] Return Msg Box full - failed to put element.\n", pReasm->streamId);
862  }
863 }
864 
865 
866 
867 /*****************************************************************************
868  Free a complete re-assembled datagram
869 ******************************************************************************/
870 static void _ReleaseDgramTblEntry(reasmThread_t *pReasm, tbl_entry_t *tbl_entry, NtNetStreamRx_t hNetRx)
871 {
872  int i;
874 
875  for (i = 0; i < tbl_entry->fragCnt; i++) {
876  if (tbl_entry->aFrag[i].fromUnm) {
877  /* Find Return Msg box to return NetBuf to */
878  _FreeUnmFragment(pReasm, tbl_entry->aFrag[i].unmIndex, tbl_entry->aFrag[i].hNetBuf);
879  } else {
880  NT_NetRxRelease(hNetRx, tbl_entry->aFrag[i].hNetBuf);
881  }
882  }
883 
884  /* zero out interest in DOI tbl */
885  if (tbl_entry->pDOI_Src_id) {
886  *tbl_entry->pDOI_Src_id = 0;
887  }
888  if (tbl_entry->pDOI_Dst_pr) {
889  *tbl_entry->pDOI_Dst_pr = 0;
890  }
891 
892  tbl_entry->id1 = 0;
893  tbl_entry->id2 = 0;
894  RELEASE_TBL_ENTRY(pReasm->tblFree, tbl_entry);
895 }
896 
897 /*****************************************************************************
898  Check if the datagram entry specified in tbl_entry in hash table is complete
899 ******************************************************************************/
900 static int _CheckDatagramComplete(reasmThread_t *pReasm, tbl_entry_t *tbl_entry, NtNetStreamRx_t hNetRx)
901 {
902  int size = 0, i;
903  int fstFrag = -1;
904  int lstFrag = -1;
905 
906  /* Check all fragments in this datagram entry */
907  for (i = 0; i < tbl_entry->fragCnt; i++) {
908  fstFrag = i;
909  if (tbl_entry->aFrag[i].lastFrag) {
910  lstFrag = i;
911  }
912  size += tbl_entry->aFrag[i].size;
913  }
914 
915  /* if all received, process and release it */
916  if (fstFrag >= 0 && lstFrag >= 0) {
917  /* Got both first and last frag */
918  if (tbl_entry->aFrag[lstFrag].offset + tbl_entry->aFrag[lstFrag].size == size) {
919  pReasm->datagramCompleted++;
920 
921  /* A complete datagram received and collected. Should do a checksum calculation on it to validate correctness */
922 
923  _ReleaseDgramTblEntry(pReasm, tbl_entry, hNetRx);
924  return 1;
925  }
926  }
927  return 0;
928 }
929 
930 /*****************************************************************************
931  The main re-assembling thread routine. This function is called for each stream
932  specified in the load-balance configuration. It collects packets and fragments.
933  On fragment receival, it collects them into one datagram collection for
934  re-assembling. The hNetBufs are released. The thread may receive the fragments
935  from the stream it reads from, or it receives it from a msg box from one of
936  the un-matched threads. On receival of a first-fragment, the interest of that
937  datagram is raised in the DOI table associated with the un-matched thread
938  these fragments will be sent on. A first-fragment will always be sent on the
939  ordinary data streams, never on one of the un-matched streams specified.
940 ******************************************************************************/
941 #if defined(__linux__) || defined(__FreeBSD__)
942  static void *_ReassemblyThread(void *arg)
943 #elif defined(WIN32) || defined (WIN64)
944  static unsigned __stdcall _ReassemblyThread(void *arg)
945 #endif
946 {
947  reasmThread_t *pReasm = (reasmThread_t *)arg;
948  char tmpBuffer[20]; // Buffer used to create NTPL expressions
949  char errorBuffer[NT_ERRBUF_SIZE]; // Error buffer
950  NtNetStreamRx_t hNetRx; // Handle to the RX stream
951  NtNetBuf_t hNetBuf=NULL; // Net buffer container
952  int i,ii; // Loop counters
953  int status; // NTAPI call return value
954  uint8_t fromUnm; // Flag to indicate that the hNetBuf is sent from an un-matched thread
955 
956  uint8_t unmIndex;
957 
958  int timeoutChkCnt; // Counter to control packet timeout checking
959  uint64_t sysTs; // Variable to hold system time stamp
960  DOI_dgramId_u val, val1; // DOI datagram ID variables
961  tbl_entry_t *pTbl; // Hash table traverse helper variable
962 
963  snprintf(tmpBuffer, sizeof(tmpBuffer), "IPFReasm_%i", pReasm->streamId);
964 
965  if ((status = NT_NetRxOpen(&hNetRx, tmpBuffer, NT_NET_INTERFACE_PACKET, pReasm->streamId, -1)) != NT_SUCCESS) {
966  // Get the status code as text
967  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
968  fprintf(stderr, "[%i] NT_NetRxOpen() failed: %s\n", pReasm->streamId, errorBuffer);
969 #if defined(__linux__) || defined(__FreeBSD__)
970  return (void *)NULL;
971 #elif defined(WIN32) || defined (WIN64)
972  { _endthreadex(0); return 0; }
973 #endif
974  }
975 
976  timeoutChkCnt = 0;
977  while (pReasm->pIpDefrag->Running) {
978 
979  hNetBuf=NULL;
980  /* Check all Msg boxes for fragment packets from un-matched threads */
981  for (i = 0; i < pReasm->pIpDefrag->unmCnt; i++) {
982  if (!MSG_BOX_EMPTY(&pReasm->pMsgbox[i])) {
983  hNetBuf = MSG_BOX_GET(&pReasm->pMsgbox[i]);
984  fromUnm=1;
985  unmIndex = (uint8_t)i;
986  break;
987  }
988  }
989 
990  if (hNetBuf == NULL) {
991  if ((status = NT_NetRxGet(hNetRx, &hNetBuf, 1)) != NT_SUCCESS) {
992  if ((status == NT_STATUS_TIMEOUT) || (status == NT_STATUS_TRYAGAIN)) {
993  if (pReasm->pIpDefrag->fragTimeout > 0) {
994  if (++timeoutChkCnt >= 100) {
995  int deleteIt = 0;
996  /* Run through list of fragments to delete timed out fragments */
997  sysTs = GetSystemTimeNoArg()*100; // make it 1/10ns units
998 
999  for (i = 0; i < REASSEMBLY_HASH_TBL_SIZE; i++) {
1000  if (pReasm->tbl[i]) {
1001  tbl_entry_t *pNxt, *tbl_entry = pReasm->tbl[i];
1002  while (tbl_entry) {
1003  deleteIt = 0;
1004  for (ii = 0; ii < tbl_entry->fragCnt; ii++) {
1005  if (sysTs - NT_NET_GET_PKT_TIMESTAMP(tbl_entry->aFrag[ii].hNetBuf) > pReasm->pIpDefrag->fragTimeout) {
1006  deleteIt=1;
1007  break;
1008  }
1009  }
1010  if (deleteIt) {
1011  pNxt = tbl_entry->pNext;
1012  _ReleaseDgramTblEntry(pReasm, tbl_entry, hNetRx);
1013  pReasm->fragmentsTimedout++;
1014  tbl_entry = pNxt;
1015  } else {
1016  tbl_entry = tbl_entry->pNext;
1017  }
1018  }
1019  }
1020  }
1021  timeoutChkCnt = 0;
1022  }
1023  }
1024  continue;
1025  }
1026 
1027  // Get the status code as text
1028  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1029  fprintf(stderr, "[%i] NT_NetRxGet() failed: %s\n", pReasm->streamId, errorBuffer);
1030  goto EXIT;
1031  }
1032  fromUnm = 0;
1033  unmIndex = 0;
1034  }
1035 
1036  timeoutChkCnt = 0;
1037  if (NT_NET_GET_PKT_L2_FRAME_TYPE(hNetBuf) == NT_L2_FRAME_TYPE_ETHER_II &&
1038  NT_NET_GET_PKT_L3_FRAME_TYPE(hNetBuf) == NT_L3_FRAME_TYPE_IPv4 &&
1039  NT_NET_GET_PKT_L3_FRAGMENTED(hNetBuf)) {
1040  uint32_t unmEntry;
1041  uint32_t src, dst;
1042  uint8_t prot;
1043  uint16_t id;
1044  int offset;
1045  tbl_entry_t *tbl_entry=NULL;
1046  DOI_dgramTbl_t *unmTbl;
1047 
1048  if (NT_NET_GET_PKT_L3_FIRST_FRAG(hNetBuf)) pReasm->firstFragRcv++;
1049  IPV4_GET_DGRAM_ID(hNetBuf, src, dst, id, prot);
1050  val.src = src;
1051  val.id = id;
1052  val1.src = dst;
1053  val1.id = prot;
1054 
1055  LOOKUP_ENTRY(pReasm->tbl, tbl_entry_t, REASSEMBLY_HASH_TBL_SIZE, val.src_id, val1.src_id, tbl_entry);
1056  if (tbl_entry == NULL) {
1057  GET_NEW_TBL_ENTRY(pReasm->tblFree, tbl_entry_t, tbl_entry);
1058  }
1059 
1060  if (tbl_entry->id1 == 0) {
1061  // New datagram fragment received
1062  tbl_entry->id1 = val.src_id;
1063  tbl_entry->id2 = val1.src_id;
1064  tbl_entry->fragCnt = 0;
1065  tbl_entry->pDOI_Src_id = NULL;
1066  tbl_entry->pDOI_Dst_pr = NULL;
1068  } else {
1069  // Error check
1070  if (tbl_entry->fragCnt == MAX_FRAG_CNT) {
1071  fprintf(stderr, "[%i] ERROR - MAX frag cnt reached\n", pReasm->streamId);
1072  exit(0);
1073  }
1074  }
1075 
1076  // Add fragment to datagram in list
1077  tbl_entry->aFrag[tbl_entry->fragCnt].hNetBuf = hNetBuf;
1078  tbl_entry->aFrag[tbl_entry->fragCnt].fromUnm = fromUnm;
1079  tbl_entry->aFrag[tbl_entry->fragCnt].unmIndex = unmIndex;
1080 
1081 /*
1082 * warning You may get conversion warnings here caused by ntohs(). This is a bug in /usr/include/bits/byteswap.h
1083 */
1084  tbl_entry->aFrag[tbl_entry->fragCnt].offset = (uint16_t)IPV4_FRAGMENT_OFFSET(hNetBuf);
1085  tbl_entry->aFrag[tbl_entry->fragCnt].size = (uint16_t)IPV4_DATA_LEN(hNetBuf);
1086  tbl_entry->aFrag[tbl_entry->fragCnt].firstFrag = (uint8_t)NT_NET_GET_PKT_L3_FIRST_FRAG(hNetBuf);
1087 
1088  if (pReasm->pIpDefrag->ExtDescrType == 9) {
1089  tbl_entry->aFrag[tbl_entry->fragCnt].lastFrag = (uint8_t)NT_NET_GET_PKT_IPF_LAST_FRAGMENT(hNetBuf);
1090  } else {
1091  tbl_entry->aFrag[tbl_entry->fragCnt].lastFrag = IPV4_LAST_FRAG(hNetBuf);
1092  }
1093 
1094  tbl_entry->fragCnt++;
1095 
1096  /* Check datagram for completion */
1097  if (_CheckDatagramComplete(pReasm, tbl_entry, hNetRx) == 0) {
1098  if (NT_NET_GET_PKT_DESCRIPTOR_FORMAT(hNetBuf) == 9) {
1099  if (NT_NET_GET_PKT_L3_FIRST_FRAG(hNetBuf)) {
1100  int idx=-1;
1101 
1102  /* Find thread-index of the un-matched stream receiving un-matched fragments */
1103  offset = NT_NET_GET_PKT_IPF_UNMATCHED_STREAMID(hNetBuf) - pReasm->pIpDefrag->unmStart;
1104  /* Get a pointer to the DOI table between this re-assembling thread and the un-matching fragments thread */
1105  unmTbl = &pReasm->pIpDefrag->pDOI_Tbls[(offset * pReasm->pIpDefrag->reasmCnt + pReasm->idx)* DOI_FRAG_TBL_SIZE];
1106  DOI_HASH_GET_KEY(unmEntry, src, id);
1107 
1108  // On first fragment receival - Notify the corresponding un-matched thread about interest in fragments of this datagram
1109  for (i=0;i<MAX_SRC_ID;i++) {
1110  if (unmTbl[unmEntry].aSrc_id[i] == val.src_id &&
1111  unmTbl[unmEntry].aDst_pr[i] == val1.src_id ) {
1112  pReasm->Src_IdClash++;
1113  /* Ok already there! */
1114  idx = i;
1115  break;
1116  }
1117  if (idx < 0 && unmTbl[unmEntry].aSrc_id[i] == 0) {
1118  idx = i;
1119  }
1120  }
1121 
1122  if (idx >= 0) {
1123  tbl_entry->pDOI_Src_id = &unmTbl[unmEntry].aSrc_id[idx];
1124  tbl_entry->pDOI_Dst_pr = &unmTbl[unmEntry].aDst_pr[idx];
1125  *tbl_entry->pDOI_Dst_pr = val1.src_id;
1126  *tbl_entry->pDOI_Src_id = val.src_id;
1127  } else {
1128  fprintf(stderr, "[%i] Too many clashes in DOI (more than %i - raise MAX_SRC_ID)\n", pReasm->streamId, MAX_SRC_ID);
1129  exit(0);
1130  }
1131  }
1132  }
1133  }
1134 
1135  } else {
1136  pReasm->nonFragments++;
1137 
1138  // Release packet as it is not fragmented.
1139  if ((status = NT_NetRxRelease(hNetRx, hNetBuf)) != NT_SUCCESS) {
1140  // Get the status code as text
1141  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1142  fprintf(stderr, "[%i] NT_NetRxRelease() failed: %s\n", pReasm->streamId , errorBuffer);
1143  goto EXIT;
1144  }
1145  }
1146  }
1147 
1148 EXIT:
1149  for (i = 0; i < REASSEMBLY_HASH_TBL_SIZE; i++) {
1150  pTbl = pReasm->tbl[i];
1151  while (pTbl) {
1152  // Release all hash table fragment packets
1153  for (ii = 0; ii < pTbl->fragCnt; ii++) {
1154  if (pTbl->aFrag[ii].fromUnm) {
1155  _FreeUnmFragment(pReasm, pTbl->aFrag[ii].unmIndex, pTbl->aFrag[ii].hNetBuf);
1156  } else {
1157  NT_NetRxRelease(hNetRx, pTbl->aFrag[ii].hNetBuf);
1158  }
1159  }
1160  pTbl = pTbl->pNext;
1161  }
1162  }
1163 
1164  /* Empty Msg boxes */
1165  for (i = 0; i < pReasm->pIpDefrag->unmCnt; i++) {
1166  while (!MSG_BOX_EMPTY(&pReasm->pMsgbox[i])) {
1167  hNetBuf = MSG_BOX_GET(&pReasm->pMsgbox[i]);
1168  _FreeUnmFragment(pReasm, i, hNetBuf);
1169  pReasm->msgboxPackets++;
1170  }
1171  }
1172 
1173  // Close the stream
1174  if ((status = NT_NetRxClose(hNetRx)) != NT_SUCCESS) {
1175  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1176  fprintf(stderr, "[%i] Re-asm thread: NT_NetRxGet() failed: %s\n", pReasm->streamId, errorBuffer);
1177  }
1178 #if defined(__linux__) || defined(__FreeBSD__)
1179  return (void *)NULL;
1180 #elif defined(WIN32) || defined (WIN64)
1181  { _endthreadex(0); return 0; }
1182 #endif
1183 }
1184 
1185 /*****************************************************************************
1186  Ctrl-C signal handler routine.
1187 ******************************************************************************/
1188 #if defined(WIN32) || defined (WIN64)
1189 static BOOL WINAPI StopApplication(int sig)
1190 #else
1191 static void StopApplication(int sig __attribute__((unused)))
1192 #endif
1193 {
1194 #ifdef WIN32
1195  IpDefrag.Running = 0;
1196  return TRUE;
1197 #else
1198  if (sig == SIGINT)
1199  IpDefrag.Running = 0;
1200 #endif
1201 }
1202 
1203 /* Default paramters */
1204 #define INPUT_ADAPTER 0
1205 #define FRAGMENT_TIMEOUT 2000 // mSec
1206 #define NUM_STREAMS 4
1207 #define NUM_UNM_STREAMS 4
1208 #define TABLE_TIMEOUT 125
1209 #define TABLE_PERSIST_TIMEOUT 0
1210 
1211 /*****************************************************************************
1212  Main routine. It initializes, configures FPGA, starts needed threads
1213  (un-matched and re-assembling threads), and loops while printing
1214  out statistics. On Ctrl-C it stops looping and begins the cleanup process.
1215 ******************************************************************************/
1216 int main(int argc, const char *argv[])
1217 {
1218  char tmpBuffer[100]; // Buffer to build filter string
1219  char errorBuffer[NT_ERRBUF_SIZE]; // Error buffer
1220  int status; // Status variable
1221  NtNtplInfo_t ntplInfo; // Return data structure from the NT_NTPL() call.
1222  NtInfoStream_t hInfoStream; // Info stream handle
1223  NtInfo_t hInfo; // Info handle
1224  int i, ii; // Counter variables
1225  int inp_1, inp_2; // Input port numbers for selected adapter
1226  NtConfigStream_t hCfgStream; // Handle to a config stream
1227  int hashTblFreed; // Counter for re-assemble hash table entries remained on exit
1228  int waitTblFreed; // Counter for un-matched wait table entries remained on exit
1229  int msgboxElmFreed; // Counter for hNetBuf elements remained in msg boxes
1230  int tablePersistTimeout; // Control parameter for NTPL IPFMode TablePersist setting
1231  int tableTimeout; // Control parameter for NTPL IPFMode timeout setting
1232  struct argparse argparse;
1233 #define NUM_NTPL 3
1234  const char *ntplExpr[NUM_NTPL] =
1235  {"Assign[streamid=(%i..%i)]=port==(%i..%i)",
1236  "HashMode=Hash%iTuple",
1237  "IPFMode[StreamId=(%i..%i);timeout=%i;TablePersist=%s]=port==(%i..%i)"};
1238 
1239 // Set up ctrl+c handler
1240 #if defined(WIN32) || defined (WIN64)
1241  SetConsoleCtrlHandler((PHANDLER_ROUTINE)StopApplication, TRUE);
1242 #else
1243  struct sigaction newaction; // Ctrl+c handle
1244  memset(&newaction, 0, sizeof(newaction));
1245  newaction.sa_handler = StopApplication;
1246  if (sigaction(SIGINT, &newaction, NULL) < 0) {
1247  fprintf(stderr, "Failed to register sigaction.\n");
1248  exit(1);
1249  }
1250 #endif
1251 
1252  // Setup default parameter settings
1253  memset(&IpDefrag, 0 , sizeof(struct _ipDefrag));
1256  IpDefrag.reasmStart = 0;
1259  IpDefrag.fragTimeout = FRAGMENT_TIMEOUT * 100000ULL;
1260  tableTimeout = TABLE_TIMEOUT * 10;
1261  tablePersistTimeout = TABLE_PERSIST_TIMEOUT;
1262 
1263  argparse_init(&argparse, arg_options, usageText, 0);
1264  argparse_parse(&argparse, argc, argv);
1265 
1266  if (opt_adapter != -1) {
1268  }
1269  if (opt_reasm != -1) {
1271  }
1272  if (opt_unm != -1) {
1274  }
1275  if (opt_frag != -1) {
1276  IpDefrag.fragTimeout = (uint64_t)opt_frag * 100000ULL;
1277  }
1278  if (opt_persist != NULL) {
1279  if (strcmp(opt_persist, "timeout") == 0) {
1280  tablePersistTimeout = 1;
1281  }
1282  }
1283  if (opt_timeout != -1) {
1284  tableTimeout = opt_timeout * 10;
1285  }
1286 
1287  // Initialize the NTAPI library and thereby check if NTAPI_VERSION can be used together with this library
1288  if ((status = NT_Init(NTAPI_VERSION)) != NT_SUCCESS) {
1289  // Get the status code as text
1290  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1291  fprintf(stderr, "NT_Init() failed: %s\n", errorBuffer);
1292  return -1;
1293  }
1294 
1295  /* Open the info stream */
1296  if ((status = NT_InfoOpen(&hInfoStream, "IPFExample")) != NT_SUCCESS) {
1297  // Get the status code as text
1298  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1299  fprintf(stderr, "NT_InfoOpen() failed: %s\n", errorBuffer);
1300  return -1;
1301  }
1302 
1303  /* Read number of adapter */
1305  hInfo.u.adapter_v6.adapterNo = (uint8_t) IpDefrag.adapterNo;
1306 
1307  if ((status = NT_InfoRead(hInfoStream, &hInfo)) != 0) {
1308  // Get the status code as text
1309  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1310  fprintf(stderr, "NT_InfoRead() failed: %s\n", errorBuffer);
1311  return -1;
1312  }
1313  /* collect port range for the selected adapter */
1314  inp_1 = hInfo.u.adapter_v6.data.portOffset;
1315  inp_2 = hInfo.u.adapter_v6.data.portOffset + hInfo.u.adapter_v6.data.numPorts - 1;
1316 
1317  /* Ensure that Ext9 was configured. */
1320  } else {
1321  fprintf(stderr, "The packet descriptor is not Ext9. Please set 'PacketDescriptor = Ext9' in ntservice.ini file for the selected adapter.\n");
1322  return -1;
1323  }
1324 
1325  /* Get timesync configuration. Must use OS mode to run timeouts with this demo */
1326  if (IpDefrag.fragTimeout) {
1328  hInfo.u.timeSync_v4.adapterNo = (uint8_t) IpDefrag.adapterNo;
1329  if ((status = NT_InfoRead(hInfoStream, &hInfo)) != NT_SUCCESS) {
1330  // Get the status code as text
1331  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1332  fprintf(stderr, "Failed to read timesync info: %s\n", errorBuffer);
1333  return -1;
1334  }
1335 
1337  fprintf(stderr, "The timesync reference clock on the selected adapter is not OS. Please set TimeSyncReferencePriority=OsTime in ntservice.ini file for the selected adapter\n");
1338  return -1;
1339  }
1340  }
1341 
1342  /* Close info stream */
1343  if ((status = NT_InfoClose(hInfoStream)) != NT_SUCCESS) {
1344  // Get the status code as text
1345  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1346  fprintf(stderr, "NT_InfoClose() failed: %s\n", errorBuffer);
1347  return -1;
1348  }
1349 
1350  // Open a config stream to assign a filter to a stream ID.
1351  if ((status = NT_ConfigOpen(&hCfgStream, "IPFExample")) != NT_SUCCESS) {
1352  // Get the status code as text
1353  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1354  fprintf(stderr, "NT_ConfigOpen() failed: %s\n", errorBuffer);
1355  return -1;
1356  }
1357 
1358  /* Build final NTPL command strings */
1359  for (i = 0; i < NUM_NTPL; i++) {
1360  switch (i) {
1361  case 0: snprintf(tmpBuffer, sizeof(tmpBuffer), ntplExpr[i], IpDefrag.reasmStart, IpDefrag.reasmStart + IpDefrag.reasmCnt - 1, inp_1, inp_2); break;
1362  case 1: snprintf(tmpBuffer, sizeof(tmpBuffer), ntplExpr[i], (IpDefrag.ExtDescrType == 9)?5:2); break;
1363  case 2:
1364  if (IpDefrag.ExtDescrType == 9) {
1365  if (tablePersistTimeout) {
1366  snprintf(tmpBuffer, sizeof(tmpBuffer), ntplExpr[i], IpDefrag.unmStart, IpDefrag.unmStart + IpDefrag.unmCnt - 1, tableTimeout, "TimeoutOnly", inp_1, inp_2);
1367  } else {
1368  snprintf(tmpBuffer, sizeof(tmpBuffer), ntplExpr[i], IpDefrag.unmStart, IpDefrag.unmStart + IpDefrag.unmCnt - 1, tableTimeout, "LastFragment", inp_1, inp_2);
1369  }
1370  } else {
1371  IpDefrag.unmCnt = 0;
1372  continue;
1373  }
1374  break;
1375  default:
1376  strncpy(tmpBuffer, ntplExpr[i], sizeof(tmpBuffer) - 1);
1377  tmpBuffer[sizeof(tmpBuffer) - 1] = '\0';
1378  break;
1379  }
1380 
1381  // Assign NTPL expressions needed
1382  if ((status = NT_NTPL(hCfgStream, tmpBuffer, &ntplInfo, NT_NTPL_PARSER_VALIDATE_NORMAL)) != NT_SUCCESS) {
1383  fprintf(stderr, "ERROR --> %s\n", tmpBuffer);
1384  // Get the status code as text
1385  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1386  fprintf(stderr, "NT_NTPL() failed: %s\n", errorBuffer);
1387  fprintf(stderr, ">>> NTPL errorcode: %X\n", ntplInfo.u.errorData.errCode);
1388  fprintf(stderr, ">>> %s\n", ntplInfo.u.errorData.errBuffer[0]);
1389  fprintf(stderr, ">>> %s\n", ntplInfo.u.errorData.errBuffer[1]);
1390  fprintf(stderr, ">>> %s\n", ntplInfo.u.errorData.errBuffer[2]);
1391  return -1;
1392  }
1393  }
1394 
1395  IpDefrag.Running = 1;
1397 
1398  /* Create IPv4 Reassembly threads */
1399  IpDefrag.pReasmThreads = calloc(1, sizeof(reasmThread_t) * IpDefrag.reasmCnt);
1400  if (IpDefrag.pReasmThreads == NULL) {
1401  fprintf(stderr, "Memory allocation failed\n");
1402  return -1;
1403  }
1404 
1405  /* Create communication message boxes. One msg box for each Rasm thread from each Unm thread */
1406  /* Create twice - one for NetBuf send from unmatched thread to reassembling thread and one for */
1407  /* returning the NetBuf again to the unmatched thread */
1409  calloc(1, sizeof(msg_box_t) * IpDefrag.reasmCnt * 2 *
1410  /* IpDefrag.unmCnt == 0 when IpDefrag.ExtDescrType != 8 */
1411  (IpDefrag.unmCnt == 0 ? 1 : IpDefrag.unmCnt));
1412 
1413  if (IpDefrag.pReasmMsgboxes == NULL) {
1414  fprintf(stderr, "Memory allocation failed\n");
1415  return -1;
1416  }
1418 
1419  /* initialize and start all IP fragment re-assembling threads */
1420  for (i = 0; i < IpDefrag.reasmCnt; i++) {
1421  tbl_entry_t *tbl;
1422 
1427 
1428  /* Create initial hash table elements and fill in free list. May grow dynamically later */
1429  IpDefrag.pReasmThreads[i].tblFree = calloc(1, sizeof(tbl_entry_t));
1431  if (tbl) {
1432  for (ii = 1; ii < INITIAL_HASH_TBL_ENTRY_CNT; ii++) {
1433  tbl->pNext = calloc(1, sizeof(tbl_entry_t));
1434  tbl=tbl->pNext;
1435  if (tbl == NULL) break;
1436  }
1437  }
1438 
1439  if ((status = pthread_create(&IpDefrag.pReasmThreads[i].thread, NULL, _ReassemblyThread, (void*)&IpDefrag.pReasmThreads[i])) != 0) {
1440  fprintf(stderr, "Unable to create Unmatched stream thread");
1441  return status;
1442  }
1443  }
1444 
1445  if (IpDefrag.unmCnt) {
1446  /* Create all DOI-tables for all un-matched threads */
1448 
1449  /* Create Unmatched fragment threads */
1450  IpDefrag.pUnmThreads = calloc(1, sizeof(unmThread_t) * IpDefrag.unmCnt);
1451  if (IpDefrag.pUnmThreads == NULL) {
1452  fprintf(stderr, "Memory allocation failed\n");
1453  return -1;
1454  }
1455 
1456  for (i = 0; i < IpDefrag.unmCnt; i++) {
1459  IpDefrag.pUnmThreads[i].pDOI = &IpDefrag.pDOI_Tbls[i * DOI_FRAG_TBL_SIZE * IpDefrag.reasmCnt];
1461 
1462  if ((status = pthread_create(&IpDefrag.pUnmThreads[i].thread, NULL, _UnMatchedThread, (void*)&IpDefrag.pUnmThreads[i])) != 0) {
1463  fprintf(stderr, "Unable to create Unmatched stream thread");
1464  return status;
1465  }
1466  }
1467  }
1468 
1469  /* main statistics loop */
1470  while (IpDefrag.Running) {
1471  int sum1,sum2,sum3,sum4,sum5;
1472  printf("Parameters:\n");
1473  printf("Adapter number to run on %i\n", IpDefrag.adapterNo);
1474  printf("Number of streams/threads %i\n", IpDefrag.reasmCnt);
1475  printf("Number of un-matched streams/threads %i\n", IpDefrag.unmCnt);
1476  printf("Fragment timeout (ms) %i\n", (int)(IpDefrag.fragTimeout/100000));
1477  printf("IPFMode table timeout (ms) %i\n", tableTimeout/10);
1478  printf("IPFMode TablePersist ");
1479  if (tablePersistTimeout) {
1480  printf("TimeoutOnly\n");
1481  } else {
1482  printf("LastFragment\n");
1483  }
1484  printf("\nRunning with extended descriptor %i\n", IpDefrag.ExtDescrType);
1485  printf("-------------------------------------------------------------------\n");
1486 
1487  printf("Re-assembling streams:\n");
1488  printf(" # Completed First frag Deleted partial Non-fragmented Id\n");
1489  printf(" (timed out) packets clashes\n");
1490 
1491  sum1 = sum2 = sum3 = sum4 = sum5 = 0;
1492  for (i = 0; i < IpDefrag.reasmCnt; i++) {
1493  printf("%2d:%8d, %8d, %10d, %14d, %12d\n", IpDefrag.pReasmThreads[i].streamId,
1502  }
1503  printf("sum:%7d, %8d, %10d, %14d, %12d\n", sum1,sum2,sum3,sum4,sum5);
1504 
1505  if (IpDefrag.unmCnt) {
1506  printf("Un-matched fragment streams:\n");
1507  printf(" # Fragments Deleted frags\n");
1508  printf(" received (timed out)\n");
1509  for (i = 0; i< IpDefrag.unmCnt; i++) {
1511  }
1512  }
1513  printf("\n-------------------------------------------------------------------\n");
1514  fflush(stdout);
1515  // Sleep 2 sec
1516  sleep(2);
1517  }
1518 
1519  /* Cleanup section */
1520  hashTblFreed = 0;
1521  waitTblFreed = 0;
1522  msgboxElmFreed = 0;
1523  for (i = 0; i < IpDefrag.reasmCnt; i++) {
1524  tbl_entry_t *tbl;
1526  /* Free all free table entries */
1527  while (IpDefrag.pReasmThreads[i].tblFree) {
1529  free(IpDefrag.pReasmThreads[i].tblFree);
1531  }
1532  /* Free all hash table entries containing fragments */
1533  for (ii = 0; ii < REASSEMBLY_HASH_TBL_SIZE; ii++) {
1534  while (IpDefrag.pReasmThreads[i].tbl[ii]) {
1535  tbl = IpDefrag.pReasmThreads[i].tbl[ii]->pNext;
1536  hashTblFreed++; // already released on thread termination
1537  free(IpDefrag.pReasmThreads[i].tbl[ii]);
1539  }
1540  }
1541  msgboxElmFreed += IpDefrag.pReasmThreads[i].msgboxPackets;
1542  }
1543  free(IpDefrag.pReasmThreads);
1544 
1546 
1547  for (i = 0; i < IpDefrag.unmCnt; i++) {
1549  }
1550 
1551  for (i = 0; i < IpDefrag.unmCnt; i++) {
1553  while (pWait) {
1554  waitTblFreed++; // already released on thread termination
1555  pWait=pWait->pNext;
1556  }
1557  }
1558 
1559  free(IpDefrag.pUnmThreads);
1560  free(IpDefrag.pReasmMsgboxes);
1561  free(IpDefrag.pDOI_Tbls);
1562 
1563  if (msgboxElmFreed || hashTblFreed || waitTblFreed) {
1564  printf("\n----------------------------------------------------------");
1565  if (hashTblFreed) {
1566  printf("\n%i entries remained in hash table on exit", hashTblFreed);
1567  }
1568  if (waitTblFreed) {
1569  printf("\n%i entries remained in un-matched fragments wait-table on exit", waitTblFreed);
1570  }
1571  if (msgboxElmFreed) {
1572  printf("\n%i entries remained in msg boxes on exit", msgboxElmFreed);
1573  }
1574  printf("\n----------------------------------------------------------\n");
1575  }
1576 
1577  // Delete the filter
1578  snprintf(tmpBuffer, sizeof(tmpBuffer), "delete=%d", ntplInfo.ntplId);
1579  if ((status = NT_NTPL(hCfgStream, tmpBuffer, &ntplInfo, NT_NTPL_PARSER_VALIDATE_NORMAL)) != NT_SUCCESS) {
1580  // Get the status code as text
1581  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1582  fprintf(stderr, "NT_NTPL() failed: %s\n", errorBuffer);
1583  fprintf(stderr, ">>> NTPL errorcode: %X\n", ntplInfo.u.errorData.errCode);
1584  fprintf(stderr, ">>> %s\n", ntplInfo.u.errorData.errBuffer[0]);
1585  fprintf(stderr, ">>> %s\n", ntplInfo.u.errorData.errBuffer[1]);
1586  fprintf(stderr, ">>> %s\n", ntplInfo.u.errorData.errBuffer[2]);
1587  return -1;
1588  }
1589 
1590  // Close the config stream
1591  if ((status = NT_ConfigClose(hCfgStream)) != NT_SUCCESS) {
1592  // Get the status code as text
1593  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
1594  fprintf(stderr, "NT_ConfigClose() failed: %s\n", errorBuffer);
1595  return -1;
1596  }
1597  return 0;
1598 }