transmit_multifunction_example.c Source File

Reference Documentation

Platform
Intel® PAC
Napatech SmartNIC
Content Type
Reference Information
Capture Software Version
Link™ Capture Software 12.10
Napatech Software Suite: examples/net/transmit_multifunction/transmit_multifunction_example.c Source File
transmit_multifunction_example.c
Go to the documentation of this file.
1 /*
2  *
3  * Copyright 2023 Napatech A/S. All Rights Reserved.
4  *
5  * 1. Copying, modification, and distribution of this file, or executable
6  * versions of this file, is governed by the terms of the Napatech Software
7  * license agreement under which this file was made available. If you do not
8  * agree to the terms of the license do not install, copy, access or
9  * otherwise use this file.
10  *
11  * 2. Under the Napatech Software license agreement you are granted a
12  * limited, non-exclusive, non-assignable, copyright license to copy, modify
13  * and distribute this file in conjunction with Napatech SmartNIC's and
14  * similar hardware manufactured or supplied by Napatech A/S.
15  *
16  * 3. The full Napatech Software license agreement is included in this
17  * distribution, please see "NP-0405 Napatech Software license
18  * agreement.pdf"
19  *
20  * 4. Redistributions of source code must retain this copyright notice,
21  * list of conditions and the following disclaimer.
22  *
23  * THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES, EXPRESS OR
24  * IMPLIED, AND NAPATECH DISCLAIMS ALL IMPLIED WARRANTIES INCLUDING ANY
25  * IMPLIED WARRANTY OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, OR OF
26  * FITNESS FOR A PARTICULAR PURPOSE. TO THE EXTENT NOT PROHIBITED BY
27  * APPLICABLE LAW, IN NO EVENT SHALL NAPATECH BE LIABLE FOR PERSONAL INJURY,
28  * OR ANY INCIDENTAL, SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES WHATSOEVER,
29  * INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF PROFITS, CORRUPTION OR
30  * LOSS OF DATA, FAILURE TO TRANSMIT OR RECEIVE ANY DATA OR INFORMATION,
31  * BUSINESS INTERRUPTION OR ANY OTHER COMMERCIAL DAMAGES OR LOSSES, ARISING
32  * OUT OF OR RELATED TO YOUR USE OR INABILITY TO USE NAPATECH SOFTWARE OR
33  * SERVICES OR ANY THIRD PARTY SOFTWARE OR APPLICATIONS IN CONJUNCTION WITH
34  * THE NAPATECH SOFTWARE OR SERVICES, HOWEVER CAUSED, REGARDLESS OF THE THEORY
35  * OF LIABILITY (CONTRACT, TORT OR OTHERWISE) AND EVEN IF NAPATECH HAS BEEN
36  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. SOME JURISDICTIONS DO NOT ALLOW
37  * THE EXCLUSION OR LIMITATION OF LIABILITY FOR PERSONAL INJURY, OR OF
38  * INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY TO YOU.
39  *
40  *
41 
42  */
43 
44 /**
45  * @example net/transmit_multifunction/transmit_multifunction_example.c
46  * @section transmit_multifunction_example_description Description
47  *
48  * This source file is an example of how to transmit data in segments using
49  * NTAPI. The example is able to transmit different packet sizes at different
50  * rates on different ports. The example shows how to create an incrementing
51  * or decrementing 32-bit pattern. When the rate is fixed, the example reuses
52  * segments to reduce the CPU load to virtually zero.
53  *
54  * The following NTAPI functions are used:
55  * - @ref NT_Init()
56  * - @ref NT_ConfigOpen
57  * - @ref NT_ConfigRead
58  * - @ref NT_ConfigWrite
59  * - @ref NT_ConfigClose
60  * - @ref NT_InfoOpen
61  * - @ref NT_InfoRead
62  * - @ref NT_InfoClose
63  * - @ref NT_StatOpen
64  * - @ref NT_NetTxRead
65  * - @ref NT_NetTxOpen()
66  * - @ref NT_NetTxGet()
67  * - @ref NT_NETTX_SEGMENT_OPTION_RAW
68  * - @ref _nt_net_build_pkt_netbuf
69  * - @ref NT_NET_SET_PKT_CLEAR_DESCR_EXT7
70  * - @ref NT_NET_SET_PKT_DESCR_TYPE_EXT7
71  * - @ref NT_NET_SET_PKT_CAP_LENGTH
72  * - @ref NT_NET_SET_PKT_CAP_LENGTH_NOALIGN
73 * - @ref NT_NET_SET_PKT_WIRE_LENGTH
74  * - @ref NT_NET_SET_PKT_RECALC_L2_CRC
75  * - @ref NT_NET_SET_PKT_TXNOW
76  * - @ref NT_NET_SET_PKT_TXSETCLOCK
77  * - @ref NT_TIMESTAMP_METHOD_EOF
78  * - @ref NT_NET_SET_PKT_TIMESTAMP
79  * - @ref _nt_net_get_next_packet
80  * - @ref NT_NET_SET_PKT_TXIGNORE
81  * - @ref NT_NET_GET_PKT_DESCR_LENGTH
82  * - @ref NT_DESCR_EXT7_LENGTH
83  * - @ref NT_NetTxRelease()
84  * - @ref NT_StatRead
85  * - @ref NT_NetTxClose()
86  * - @ref NT_StatClose
87  * - @ref NT_Done()
88  * - @ref NT_ExplainError()
89  *
90  * @section transmit_multifunction_example_prerequisites Prerequisites
91  * - The ntservice.ini must have at least one HostBuffersTx defined. Below is
92  * an example of a minimum ini-file. It will create a 4MB TX hostbuffer from
93  * NUMA node 0.
94  *
95  * @code
96  * [System]
97  * TimestampFormat = NATIVE
98  *
99  * [Adapter0]
100  * AdapterType = NT20E2
101  * BusId = 00:0a:00.00
102  * HostBuffersTx = [1,4,0]
103  * @endcode
104  *
105  * @section transmit_multifunction_example_flow Program flow
106  *
107  * The following is required to transmit packages:
108  * - \#include/nt.h - Applications/Tools only need to include @ref
109  * nt.h to obtain prototypes, macros etc. from NTAPI.
110  * - @ref NT_Init(@ref NTAPI_VERSION) - Initialize the NTAPI
111  * library. @ref NTAPI_VERSION is a define that describes the version
112  * of the API described in the header files included by @ref
113  * nt.h. NT_Init() will ask the NTAPI library to convert return data
114  * to the @ref NTAPI_VERSION if possible. This will ensure that
115  * applications can run on NTAPI libraries of newer versions.
116  * - @ref NT_ConfigOpen() - Open the configuration stream
117  * - @ref NT_ConfigRead() - Read configuration data; used to retrieve
118  * the transmission limit on a port on a so-called 4GA adapter.
119  * - @ref NT_ConfigWrite() - Write configuration data; used to set the
120  * transmission limit on a port on a so-called 4GA adapter.
121  * - @ref NT_ConfigClose() - Close the configuration stream
122 * - @ref NT_InfoOpen() - Open the information stream
123  * - @ref NT_InfoRead() - Read information data. Used to decide the timestamp method,
124  * whether or not absolute Tx timing is supported, and whether or not it is enabled and
125  * the timstamp with.
126  * - @ref NT_InfoClose() - Close the information stream.
127  * - @ref NT_StatOpen() - Open the statistics stream.
128  * - @ref NT_StatRead() - Read the statistics in order to clear the statistics.
129  * - @ref NT_NetTxOpen() - Open a hostbuffer that can transmit packets to port 0.
130  * - @ref NT_NetTxGet() - Get an empty segment from the wanted transmit port.
131  * - @ref NT_NETTX_SEGMENT_OPTION_RAW - Parameter to @ref NT_NetTxGet.
132  * Returns an empty segment.
133  * - @ref _nt_net_build_pkt_netbuf - Returns a pointer to the first
134  * packet in the segment
135  * - @ref NT_NET_SET_PKT_CLEAR_DESCR_EXT7 - Clear the packet descriptor. Assumes
136  * that an NT extended 7 packet descriptor is going to be used.
137  * - @ref NT_NET_SET_PKT_DESCR_TYPE_EXT7 - Set the packet descriptor to be NT extend 7
138  * - @ref NT_NET_SET_PKT_CAP_LENGTH - Set the captured length of the packet.
139  * This includes the packet data length and the length of the NT extended 7
140  * packet descriptor
141  * - @ref NT_NET_SET_PKT_WIRE_LENGTH - Set the wire length of the packet.
142  * - @ref NT_NET_SET_PKT_RECALC_L2_CRC - Force the FPGA to recalculate the level 2 crc.
143  * - @ref NT_NET_SET_PKT_TXNOW - Force the FPGA to transmit the packet immediately.
144  * The packet timestamp is ignored when setting txnow.
145  * - @ref NT_NET_SET_PKT_TXSETCLOCK - Synchronize the TX clock when using absolut
146  * TX timing.
147  * - @ref NT_TIMESTAMP_METHOD_EOF - Timestamp method status - The timestamp is
148  * placed at the end of the packet. The opposite @ref NT_TIMESTAMP_METHOD_SOF -
149  * The timestamp is placed at the beginning og the packet.
150  * - @ref _nt_net_get_next_packet - Get a pointer to the next packet in the segment.
151  * The remaining length of the segment is returned. <b>Note: the segment must be completely
152  * filled otherwise it will not be sent and can cause the FPGA to crash.</b>
153  * - @ref NT_NET_SET_PKT_TXIGNORE - Force the FPGA to discard the packet.
154  * - @ref NT_NET_GET_PKT_L2_PTR() is used to get the L2 pointer to the tx
155  * buffer, this is where the payload is placed.
156  * - @ref NT_NET_SET_PKT_TIMESTAMP() is used to timestamp the packets. The
157  * packets will be transmitted based on the timing provided in the
158  * timestamps. The timestamp format <b>must</b>. be
159  * NT_TIMESTAMP_TYPE_NATIVE, hence TimestampFormat=NATIVE in
160  * ntservice.ini
161  * - @ref NT_NET_GET_PKT_DESCR_LENGTH - Get the size of the packet descriptor.
162  * - @ref NT_DESCR_EXT7_LENGTH - The sizeof the the extended 7 packet descriptor.
163  * - @ref NT_NetTxRead - Read information about the tx host buffer.
164  * - @ref NT_NetTxRelease() - Release the tx packet buffer. Once a tx
165  * buffer is released it will be transmitted according to the
166  * timestamp set in the packet.
167  * - @ref NT_StatRead() - Read the statistics in order to see how many packets that are
168  * transmitted.
169  * - @ref NT_NetTxClose() - Close the TX stream.
170  * - @ref NT_StatClose() - Close the statistics stream.
171  * - @ref NT_Done() - Close down the NTAPI library.
172  *<hr>
173  * @section transmit_multifunction_example_code Code
174  */
175 
176 // Include this in order to access the Napatech API
177 #include <nt.h>
178 
179 #if defined(__linux__) || defined(__FreeBSD__)
180  #include <unistd.h>
181  #include <signal.h>
182 #elif defined(WIN32) || defined(WIN64)
183  // ctrl, strings
184  #include <winsock2.h> // SetConsoleCtrlHandler(), for Windows 2000 Pro/Server or later; Sleep
185  #define strtoll _strtoi64
186 #endif
187 
188 #include <argparse.h>
189 #include <string.h>
190 #include <assert.h>
191 
192 #define MIN_PACKET_SIZE 64 // Minimum packet size to transmit
193 #define MAX_PACKET_SIZE 8192 // Maximum packet size to transmit
194 #define SEGMENT_LENGTH (1024*1024) // Segment size to transmit
195 
196 // Macro to get a random number
197 #define RANDOM(_min_, _max_) ((uint32_t)_min_ + (uint32_t)((_max_-_min_+1) * (rand() /(RAND_MAX + 1.0))))
198 
199 volatile int appRunning=1; // The application will run as long as appRunning==1
200 
201 /**
202  * Print command line info
203  */
204 static const char *usageText[] = {
205  "Syntax:\n"
206  "transmit_multifunction [-h][-p <port no.>][-r <tx rate>][-s <size>|r|i|d]\n"
207  "\nCommands:\n",
208  NULL};
209 
210 static int opt_port = -1;
211 static char *opt_rate = NULL;
212 static char *opt_size = NULL;
213 
214 /**
215  * Table of valid options.
216  */
218  OPT_HELP(),
219  OPT_INTEGER('p', "port", &opt_port, "Output port", NULL, 0, 0, "port number"),
220  OPT_STRING( 'r', "rate", &opt_rate, "Transmit rate\n"
221  "0 = line rate\n"
222  "<rate> = <rate> Mbps\n"
223  "<rate>K = <rate> Kbps\n"
224  "<rate>M = <rate> Mbps\n"
225  "<rate>G = <rate> Gbps", NULL, 0, 0, "value"),
226  OPT_STRING( 's', "size", &opt_size, "Packet size - <value> = packet size (Size is aligned to 4 byte boundary\n"
227  "r = random size\n"
228  "i = increasing size from 64 bytes to 8192 bytes.\n"
229  "d = decreasing size from 8192 bytes to 64 bytes.", NULL, 0, 0, "value"),
230  OPT_END(),
231 };
232 
233 /**
234  * The function called when user is pressing CTRL-C
235  */
236 #if defined(WIN32) || defined (WIN64)
237 static BOOL WINAPI StopApplication(int sig)
238 #else
239 static void StopApplication(int sig)
240 #endif
241 {
242 #ifdef WIN32
243  // Force the application to stop.
244  appRunning=0;
245  return TRUE;
246 #else
247  if (sig == SIGINT)
248  // Force the application to stop.
249  appRunning=0;
250 #endif
251 }
252 
253 static int is4GATxSupported = 0;
254 static int _Is4GATxSupported(NtInfoStream_t hInfo, uint8_t adapterNo)
255 {
256  NtInfo_t infoRead;
257 
258  (void)memset(&infoRead, 0, sizeof(NtInfo_t));
260  infoRead.u.adapter_v6.adapterNo = adapterNo;
261  if (NT_InfoRead(hInfo, &infoRead) != 0) {
262  return 0;
263  }
267  // 4GA adapters with ANL9 feature level support native TX with inline capabilities
268  // NOTE: 4GA adapters with ANL9 feature level doesn't support
269  // NT_NET_SET_PKT_DESCR_TYPE_EXT7 descriptor type
270  return 1;
271  }
272  return 0;
273 }
274 
275 //
276 // Function to set the transmission rate/limit of a port on a 4GA
277 // adapter. (This program uses the packets' time stamp to control the
278 // transmission rate for older Napatech adapters.
279 //
280 static int _TxPortRateLimit(int port, uint64_t new, uint64_t *pOld)
281 {
282  NtConfigStream_t hConfig = NULL;
283  uint64_t oldRateLimit = 0;
284 
285  int status = NT_ConfigOpen(&hConfig, "tx_port_rate_limit");
286  if (status) {
287  char errorBuffer[1024];
288  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
289  fprintf(stderr, "NT_ConfigOpen failed: %s\n", errorBuffer);
290  return status;
291  }
292 
293  NtConfig_t configRead;
294  (void)memset(&configRead, 0, sizeof(configRead));
296  configRead.u.portSettings_v2.portNo = (uint8_t)port;
297 
298  //
299  // Set port transmission limit in two steps:
300  // First read the port's current settings,
301  //
302  status = NT_ConfigRead(hConfig, &configRead);
303  //
304  // and then set the new transmission limit.
305  //
306  if (status == NT_SUCCESS) {
307  oldRateLimit = configRead.u.portSettings_v2.data.txPortRateLimit;
308 
309  configRead.u.portSettings_v2.data.txPortRateLimit = new;
310  status = NT_ConfigWrite(hConfig, &configRead);
311  }
312 
313  if (status) {
314  char errorBuffer[1024];
315  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
316  fprintf(stderr, "NT_ConfigRead/Write failed: %s\n", errorBuffer);
317  }
318 
319  (void)NT_ConfigClose(hConfig);
320 
321  if (status == NT_SUCCESS && pOld != NULL)
322  *pOld = oldRateLimit;
323 
324  return status;
325 }
326 
327 //
328 // Function that checks whether the packets in the TX host buffer have
329 // reached the adapter's FPGA.
330 //
331 static int _NetTxDataCheckHbInfo(NtNetStreamTx_t* phNetTx, bool bWaitDone)
332 {
333  do {
334  NtNetTx_t ntNetTx;
336  const int status = NT_NetTxRead(*phNetTx, &ntNetTx);
337  if (status == NT_SUCCESS) {
338  assert(ntNetTx.u.hbInfo.numHostBuffers == 1);
339  const size_t nHbSizeTotal = ntNetTx.u.hbInfo.aHostBuffer[0].size;
340  const size_t nHbSizeAvail = ntNetTx.u.hbInfo.aHostBuffer[0].available;
341  const size_t nHbSizeRel = ntNetTx.u.hbInfo.aHostBuffer[0].released;
342  const size_t nHbSizeDeq = ntNetTx.u.hbInfo.aHostBuffer[0].dequeued;
343 
344  if (nHbSizeAvail == nHbSizeTotal && nHbSizeRel == 0 && nHbSizeDeq == 0) {
345  return 0; // zero int is success
346  }
347  } else {
348  char errorBuffer[1024];
349  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
350  fprintf(stderr, "NT_NetTxRead failed: %s\n", errorBuffer);
351  return -1; // negative int is error
352  }
353  if (bWaitDone) {
354 #if defined(__linux__) || defined(__FreeBSD__)
355  (void)usleep(1000); // pause to avoid busy waiting
356 #elif defined(WIN32) || defined(WIN64)
357  Sleep(1); // 1 milli-sec
358 #endif
359  }
360  } while (bWaitDone && appRunning);
361 
362  return 1; // positive int means `not done`
363 }
364 
365 //
366 //
367 //
368 int main(int argc, const char *argv[])
369 {
370  char errorBuffer[NT_ERRBUF_SIZE]; // Error buffer
371  int status; // Status variable
372  NtNetStreamTx_t hNetTx; // Handle to the TX stream
373  NtNetBuf_t hNetBufTx; // Net buffer container. Used when getting a transmit segment
374  struct NtNetBuf_s pktNetBuf; // Net buffer container. Used when getting a transmit buffer
375  NtStatStream_t hStatStream; // Statistics stream handle
376  NtStatistics_t hStat; // Stat handle.
377  uint32_t i; // Counter
378  uint32_t payload = 0; // Payload content/counter
379  uint32_t currentPacketSize = MIN_PACKET_SIZE; // Packet size. Default set to MIN_PACKET_SIZE
380  uint32_t txPacketSize; // Size of next packet to transmit
381  uint32_t segmentsSent = 0; // Number of segments sent
382  uint64_t packetsSent = 0; // Number of packets sent.
383  uint32_t txPort = 0; // Transmit port
384  uint32_t txSize = MIN_PACKET_SIZE; // Transmit packet size
385  uint32_t txType = 0; // Transmit type - Increasing, decreasing or random size
386  uint64_t txRate = 0; // Transmit rate
387  NtInfoStream_t hInfo; // Handle to an info stream
388  NtInfo_t infoRead; // Info data
389  uint64_t ts=0; // Timestamp counter
390  uint64_t tsR=0; // The 10ns roundup time that the adapter is actually to run
391  int firstPacket = 0; // First packet to transmit
392  enum NtTimestampMethod_e timestampMethod; // Timestamp method
393  enum NtTxTimingMethod_e txTiming; // TX mode - Do we use absolut or relative mode
394  int timestamp64BitSupport; // Is 64 bit timestamp supported
395  uint32_t maxPacketsInSegment; // Max number of packets we want in the segment
396  uint32_t maxPacketsInAdapter; // Max number of packets we want in the adapter
397  struct argparse argparse;
398 #define NUM_MB 10 /* Number of segments to pre-fill/reuse */
399  void *seg_hdr_ptr[NUM_MB];
400  uint32_t packetsInSegment[NUM_MB]; // Number of packets in current segment
401  const int min_hb_size_in_mb = NUM_MB;
402  uint32_t reuse_idx;
403 
404  printf("\nNapatech example: Segment based transmit\n");
405  printf("----------------------------------------\n\n");
406 
407  (void)memset(seg_hdr_ptr, 0, sizeof(seg_hdr_ptr)); /* Make ptrs NULL */
408  (void)memset(packetsInSegment, 0, sizeof(packetsInSegment));
409 
410  // Register ctrl+c handler so we are able to stop again
411 #if defined(WIN32) || defined (WIN64)
412  SetConsoleCtrlHandler((PHANDLER_ROUTINE)StopApplication, TRUE);
413 #else
414  struct sigaction newaction; // Ctrl+c signal handler container
415  memset(&newaction, 0, sizeof(newaction));
416  newaction.sa_handler = StopApplication;
417  if (sigaction(SIGINT, &newaction, NULL) < 0) {
418  fprintf(stderr, "Failed to register SIGINT sigaction.\n");
419  exit(-1);
420  }
421 #endif
422 
423  argparse_init(&argparse, arg_options, usageText, 0);
424  argparse_parse(&argparse, argc, argv);
425 
426  if (opt_port != -1) {
427  txPort = (uint32_t)opt_port;
428  }
429  if (opt_rate != NULL) {
430  int len;
431  char *endptr;
432  txRate=strtoll(opt_rate, &endptr, 0);
433  len = (int)strlen(endptr);
434 
435  if (len > 1) {
436  fprintf(stderr, "\"-r %s\" is not valid.\n", opt_rate);
437  return 1;
438  }
439  else if (len == 0) {
440  // value is in megabit;
441  txRate*=1000000;
442  }
443  else {
444  switch (endptr[0]) {
445  case 'K':
446  // Rate is in kilo bit per second.
447  txRate*=1000;
448  break;
449  case 'M':
450  // Rate is in Mega bit per second.
451  txRate*=1000000;
452  break;
453  case 'G':
454  // Rate is in Mega bit per second.
455  txRate*=1000000000;
456  break;
457  default:
458  fprintf(stderr, "\"-r %s\" is not valid.\n", opt_rate);
459  return 1;
460  }
461  }
462  }
463  if (opt_size != NULL) {
464  if (strlen(opt_size) > 1) {
465  // Align packet size to uint32_t
466  txSize = (atoi(opt_size)+3)&~3;
467  txType = 0;
468  if (txSize < MIN_PACKET_SIZE || txSize > MAX_PACKET_SIZE) {
469  fprintf(stderr, ">>> Error: Illegal packet size: %d\n", txSize);
470  exit(1);
471  }
472  }
473  else {
474  switch (opt_size[0]) {
475  case 'i':
476  txType = 'i';
477  break;
478  case 'd':
479  txType = 'd';
480  break;
481  case 'r':
482  txType = 'r';
483  break;
484  default:
485  txType = 'r';
486  break;
487  }
488  }
489  }
490 
491  if (txRate < 10000000 && txRate != 0) {
492  // When transmitting slowly we don't want to fill the adapter with data as we
493  // need to wait until all data are transmitted. When transmitting slower
494  // than 10M we want to fill the segments partly and we do only want to send the
495  // segments to the adapter when some the data is transmitted. If sending all the
496  // segments to the adapter, they will be queued internally in the adapter.
497  // We only want <maxPacketsInSegment> packets in each segment and we only want
498  // the send the segment to the adapter when adapter has less than <maxPacketsInAdapter>
499  // packets waiting to be transmitted.
500  maxPacketsInSegment = (uint32_t)(txRate / 1000);
501  maxPacketsInAdapter = 3 * maxPacketsInSegment;
502  }
503  else {
504  // We are going to transmit fast, so there are no limits. Just fill the
505  // segments and send them to the adapter as quicky as possible.
506  maxPacketsInSegment = 0xFFFFFFFF;
507  maxPacketsInAdapter = 0xFFFFFFFF;
508  }
509 
510  // Print the settings used.
511  printf("Transmit port: %d\n", txPort);
512  if (txRate == 0) {
513  printf("Transmit Rate: Line rate\n");
514  }
515  else {
516  printf("Transmit Rate: %llu bits/second\n", (unsigned long long int)txRate);
517  }
518 
519  if (txType == 0) {
520  printf("Transmit size: %d bytes\n\n", txSize);
521  printf("The program will pre-fill/reuse %d TX segments to save CPU.\n", NUM_MB);
522  }
523  else {
524  switch (txType) {
525  case 'i':
526  printf("Transmit size: Increasing from %d bytes to %d bytes\n\n", MIN_PACKET_SIZE, MAX_PACKET_SIZE);
527  break;
528  case 'd':
529  printf("Transmit size: Decreasing from %d bytes to %d bytes\n\n", MAX_PACKET_SIZE, MIN_PACKET_SIZE);
530  break;
531  case 'r':
532  printf("Transmit size: Random bewteen %d bytes and %d bytes\n\n", MIN_PACKET_SIZE, MAX_PACKET_SIZE);
533  break;
534  }
535  }
536 
537  // Initialize the NTAPI library and thereby check if NTAPI_VERSION can be used together with this library
538  if ((status = NT_Init(NTAPI_VERSION)) != NT_SUCCESS) {
539  // Get the status code as text
540  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
541  fprintf(stderr, "NT_Init() failed: %s\n", errorBuffer);
542  return status;
543  }
544 
545  // Open the info stream to get the timestamp method.
546  if ((status = NT_InfoOpen(&hInfo, "Info")) != 0) {
547  // Get the status code as text
548  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
549  fprintf(stderr, "NT_InfoOpen failed: %s\n", errorBuffer);
550  return status;
551  }
552 
553  // Find the adapter on which the port is placed.
554  infoRead.cmd=NT_INFO_CMD_READ_PORT_V9;
555  infoRead.u.port_v9.portNo = (uint8_t) txPort;
556  if ((status = NT_InfoRead(hInfo, &infoRead)) != 0) {
557  // Get the status code as text
558  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
559  fprintf(stderr, "NT_InfoRead failed: %s\n", errorBuffer);
560  NT_InfoClose(hInfo);
561  return status;
562  }
563 
564  // Do we have 64 bit timestamp support
565  timestamp64BitSupport = infoRead.u.port_v9.data.capabilities.featureMask & NT_PORT_FEATURE_64BIT_TIMESTAMP;
566 
567  // Check whether or not TX is supported on the selected port
569  fprintf(stderr, "Transmit is not possible on the selected port %d\n", txPort);
570  NT_InfoClose(hInfo);
571  return 1;
572  }
573 
574  // Check which settings that are enabled.
575  timestampMethod = infoRead.u.port_v9.data.adapterInfo.timestampMethod;
576  txTiming = infoRead.u.port_v9.data.adapterInfo.txTiming;
577 
578  // Check whether 4GA TX is supported. If yes, EXT7-desc is not supported
579  is4GATxSupported = _Is4GATxSupported(hInfo, infoRead.u.port_v9.data.adapterNo);
580 
581  // Check transmission port is up
582  if (infoRead.u.port_v9.data.state != NT_LINK_STATE_UP) {
583  fprintf(stderr, "Port %d is down.\n", txPort);
584  (void)NT_InfoClose(hInfo);
585  return 1;
586  }
587 
588  // Close info stream again
589  NT_InfoClose(hInfo);
590 
591  // Check that the adapter has support for 64bit timestamp
592  if (timestamp64BitSupport == 0 && txRate < 1000000) {
593  fprintf(stderr, "The adapter does not support %lld bits/second\n", (long long int)txRate);
594  return 1;
595  }
596 
597  // Open the stat stream.
598  if ((status = NT_StatOpen(&hStatStream, "ExampleStat")) != NT_SUCCESS) {
599  // Get the status code as text
600  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
601  fprintf(stderr, "NT_StatOpen() failed: %s\n", errorBuffer);
602  return status;
603  }
604 
605  // Read the statistics counters to clear the statistics
607  hStat.u.query_v3.poll=0; // Wait for a new set
608  hStat.u.query_v3.clear=1; // Clear statistics
609  if ((status = NT_StatRead(hStatStream, &hStat)) != NT_SUCCESS) {
610  // Get the status code as text
611  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
612  fprintf(stderr, "NT_StatRead() failed: %s\n", errorBuffer);
613  (void)NT_StatClose(hStatStream);
614  return status;
615  }
616 
617  // Open a TX hostbuffer from TX host buffer pool on the NUMA node of the adapter with the txPort
618  if (txType == 0) {
619  status = NT_NetTxOpen(&hNetTx, "TxStreamPort", (uint64_t)0x1<<txPort, NT_NETTX_NUMA_ADAPTER_HB, min_hb_size_in_mb); /* Prefill segments */
620  } else {
621  status = NT_NetTxOpen(&hNetTx, "TxStreamPort", (uint64_t)0x1<<txPort, NT_NETTX_NUMA_ADAPTER_HB, 0);
622  }
623  if (status != NT_SUCCESS) {
624  // Get the status code as text
625  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
626  fprintf(stderr, "NT_NetTxOpen() failed: %s\n", errorBuffer);
627  NT_StatClose(hStatStream);
628  return status;
629  }
630 
631  //
632  // Set transmission limit
633  //
634  uint64_t oldTxRate;
635  int bTxPortRateLimitSet = 0;
636  if (is4GATxSupported && txRate > 0) {
637  status = _TxPortRateLimit(txPort, txRate, &oldTxRate);
638  bTxPortRateLimitSet = status == NT_SUCCESS;
639  }
640  const size_t ntDescrLength =
641  is4GATxSupported ? NT_DESCR_NT_LENGTH : NT_DESCR_EXT7_LENGTH;
642 
643  // Create segments and transmit to port txPort
644  while (status == NT_SUCCESS && appRunning==1) {
645  uint64_t spaceLeftInSegment = SEGMENT_LENGTH;
646  // Get a TX segment of size SEGMENT_LENGTH
647  status = NT_NetTxGet(hNetTx, &hNetBufTx, txPort, SEGMENT_LENGTH, NT_NETTX_SEGMENT_OPTION_RAW, 1000);
648  if (status == NT_STATUS_TIMEOUT) {
649  status = NT_SUCCESS;
650  continue;
651  }
652  if (status != NT_SUCCESS) {
653  // Get the status code as text
654  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
655  fprintf(stderr, "NT_NetTxGet() failed: %s\n", errorBuffer);
656  continue;
657  }
658 
659  reuse_idx = segmentsSent % NUM_MB;
660  // Get a packet buffer pointer from the segment.
661  _nt_net_build_pkt_netbuf(hNetBufTx, &pktNetBuf);
662  if (txType != 0 /* pre-fill/reuse not done */ ||
663  seg_hdr_ptr[reuse_idx] == NULL /* pre-fill segment */) {
664  // Add packets to the segment.
665  // IMPORTANT: The segment must be compeletely filled before it can be sent.
666  // Failing to do this causes the adapter to malfunction. In order to fill
667  // the segment a 64 byte or larger dummy packet is added to the end of the
668  // segment.
669  packetsInSegment[reuse_idx] = 0;
670  for (;;) {
671  // Build the packet
672  if (is4GATxSupported) {
673  NT_NET_SET_PKT_CLEAR_DESCR_NT((&pktNetBuf)); // Clear the packet descriptor. Must be done first.
674  NT_NET_SET_PKT_DESCR_TYPE_NT((&pktNetBuf)); // Set the packet to use Std0 descriptor
675  }
676  else {
677  NT_NET_SET_PKT_CLEAR_DESCR_EXT7((&pktNetBuf)); // Clear the packet descriptor. Must be done first.
678  NT_NET_SET_PKT_DESCR_TYPE_EXT7((&pktNetBuf)); // Set the packet to use extended descriptor 7
679  }
680  NT_NET_SET_PKT_RECALC_L2_CRC((&pktNetBuf), 1); // Force the FPGA to calculate a new L2 crc.
681  NT_NET_SET_PKT_TXPORT((&pktNetBuf), txPort); // Only necessary if transmitting on a capture adapter.
682  NT_NET_UPDATE_PKT_L2_PTR((&pktNetBuf)); // Update the L2 ptr of the packet to match the new descriptor
683 
684  // We only want <maxPacketsInSegment> in each segment.
685  // When we reach the limit, then fill the rest of the
686  // segments with dummy packets
687  if (packetsInSegment[reuse_idx] < maxPacketsInSegment) {
688  // Set the capture/stored length of the packet.
689  // This includes the packet header and the alignment padding
690  if (is4GATxSupported) {
691  NT_NET_SET_PKT_CAP_LENGTH_NOALIGN((&pktNetBuf), (uint16_t)currentPacketSize);
692  } else {
693  NT_NET_SET_PKT_CAP_LENGTH((&pktNetBuf), (uint16_t)currentPacketSize);
694  }
695  NT_NET_SET_PKT_WIRE_LENGTH((&pktNetBuf), (uint16_t)currentPacketSize);// Set the wire length of the packet.
696  if (bTxPortRateLimitSet || txRate == 0) {
697  // Transmit at the port's configured rate
698  NT_NET_SET_PKT_TXNOW((&pktNetBuf), 1);
699  }
700  else {
701  // Set the timestamp so the rate can be controlled
702  // If packets are timestamped when last/first octet is received
703  // the timestamp must reflect this.
704  if (timestampMethod==NT_TIMESTAMP_METHOD_EOF) {
705  ts += (((uint64_t)NT_NET_GET_PKT_WIRE_LENGTH((&pktNetBuf))+20ULL)*8ULL*1000000000ULL)/txRate;
706  // Round up to nearest 10ns
707  tsR+=((((((uint64_t)NT_NET_GET_PKT_WIRE_LENGTH((&pktNetBuf))+20ULL)*8ULL*1000000000ULL)/txRate)+9)/10)*10;
708  // Compensate for the round up error?
709  if ((tsR-ts)>10) {
710  tsR-=10;
711  }
712  NT_NET_SET_PKT_TIMESTAMP((&pktNetBuf), tsR/10);
713  } else {
714  NT_NET_SET_PKT_TIMESTAMP((&pktNetBuf), tsR/10);
715  ts += (((uint64_t)NT_NET_GET_PKT_WIRE_LENGTH((&pktNetBuf))+20ULL)*8ULL*1000000000ULL)/txRate;
716  // Round up to nearest 10ns
717  tsR+=((((((uint64_t)NT_NET_GET_PKT_WIRE_LENGTH((&pktNetBuf))+20ULL)*8ULL*1000000000ULL)/txRate)+9)/10)*10;
718  // Compensate for the round up error?
719  if ((tsR-ts)>10) {
720  tsR-=10;
721  }
722  }
723 
724  if (firstPacket == 0) {
725  if (txTiming == NT_TX_TIMING_ABSOLUTE) {
726  // Use absolut tx timing
727  // Setup first packet to transmit. Only the first packet must be setup
728  // to Synchronize tx clock. TXNOW must be set to 0 for absolut TX timing
729  // to work. The default is 0.
730  NT_NET_SET_PKT_TXNOW((&pktNetBuf), 0); // Wait for tx delay before the packet is sent
731  NT_NET_SET_PKT_TXSETCLOCK((&pktNetBuf), 1); // Synchronize tx clock to timestamp in first packet.
732  }
733  else {
734  // Use Legacy mode/Relative tx timing.
735  // First packet must be sent with txnow=1
736  NT_NET_SET_PKT_TXNOW((&pktNetBuf), 1);
737  }
738  firstPacket = 1;
739  }
740  }
741 
742  // Create a packet with an incrementing 32bit pattern.
743  // Note that the packet does not contain valid MAC address etc.
744  for (i = 0; i < currentPacketSize/4; i++) {
745  *((uint32_t*)NT_NET_GET_PKT_L2_PTR((&pktNetBuf))+i) = payload++;
746  }
747 
748  // Packets in the segment
749  packetsInSegment[reuse_idx]++;
750 
751  // Get the next packet size to transmit.
752  switch (txType) {
753  default:
754  case 0:
755  // static packet size
756  currentPacketSize = txSize;
757  break;
758  case 'i':
759  // Increment packet size
760  currentPacketSize += (uint32_t)sizeof(uint32_t);
761  if (currentPacketSize > MAX_PACKET_SIZE) {
762  currentPacketSize = MIN_PACKET_SIZE;
763  }
764  break;
765  case 'd':
766  // Decrement packet size
767  currentPacketSize -= (uint32_t)sizeof(uint32_t);
768  if (currentPacketSize < MIN_PACKET_SIZE) {
769  currentPacketSize = MAX_PACKET_SIZE;
770  }
771  break;
772  case 'r':
773  // Random packet size aligned packet size to uint32_t
774  currentPacketSize=(RANDOM(MIN_PACKET_SIZE, MAX_PACKET_SIZE)+3)&~3;
775  break;
776  }
777  txPacketSize = currentPacketSize;
778  }
779  else {
780  // We don't want to fill more packets into the segment, so create a
781  // dummy packet to fill up the segment.
782 
783  // Set size of the dummy packet.
784  if (is4GATxSupported) {
785  NT_NET_SET_PKT_CAP_LENGTH_NOALIGN((&pktNetBuf), (uint16_t)128);
786  } else {
787  NT_NET_SET_PKT_CAP_LENGTH((&pktNetBuf), (uint16_t)128);
788  }
789  NT_NET_SET_PKT_WIRE_LENGTH((&pktNetBuf), (uint16_t)128);// Size of the dummy packet.
790  NT_NET_SET_PKT_TXIGNORE((&pktNetBuf), 1); // Do not transmit the packet
791  txPacketSize = 128;
792  }
793 
794  // Move the pointer to next packet and get the length of the remining space in the segment
795  spaceLeftInSegment = _nt_net_get_next_packet(hNetBufTx, NT_NET_GET_SEGMENT_LENGTH(hNetBufTx), &pktNetBuf);
796 
797  // Check if we have space for excately one packet.
798  if (spaceLeftInSegment == txPacketSize + ntDescrLength) {
799  // There is exactly space for one packet more.
800  continue;
801  }
802 
803  // Check if there is space for a packet of currentPacketSize + extended 7 header
804  // and a 64 byte dummy packet + extended header 7. We must be sure that we are able to create a dummy
805  // packet to be able to close/fill the segment.
806  if (spaceLeftInSegment < (txPacketSize + ntDescrLength) + (64 + ntDescrLength)) {
807  if (spaceLeftInSegment > 0) {
808  // No space for another packet. But we need to deliver a completely filled segment,
809  // therefore we must create a dummy packet to fill the segment.
810 
811  if (is4GATxSupported) {
812  NT_NET_SET_PKT_CLEAR_DESCR_NT((&pktNetBuf)); // Clear the packet descriptor. Must be done first.
813  NT_NET_SET_PKT_DESCR_TYPE_NT((&pktNetBuf)); // Set the packet to use Std0 descriptor
814  }
815  else {
816  NT_NET_SET_PKT_CLEAR_DESCR_EXT7((&pktNetBuf)); // Clear the packet descriptor. Must be done first.
817  NT_NET_SET_PKT_DESCR_TYPE_EXT7((&pktNetBuf)); // Set the packet to use extended descriptor 7
818  }
819  spaceLeftInSegment -= ntDescrLength;
820 
821  // Make space for packet header
822  if (is4GATxSupported) {
823  NT_NET_SET_PKT_CAP_LENGTH_NOALIGN((&pktNetBuf), (uint16_t)spaceLeftInSegment);
824  } else {
825  NT_NET_SET_PKT_CAP_LENGTH((&pktNetBuf), (uint16_t)spaceLeftInSegment);
826  }
827  NT_NET_SET_PKT_WIRE_LENGTH((&pktNetBuf), (uint16_t)spaceLeftInSegment);
828 
829  // We don't want to send this packet. Set tx ignore to tell the FPGA,
830  // not to send the packet.
831  NT_NET_SET_PKT_TXIGNORE((&pktNetBuf), 1);
832 
833  // As this is a dummy packet. We don't care about the payload.
834  }
835  // break to send the segment
836  break;
837  }
838  } // end-for `build packet`
839  }
840 
841  if (txType == 0) { /* pre-fill/reuse */
842  if (seg_hdr_ptr[reuse_idx] == NULL) {
843  /* Remember that segment is now initialized */
844  seg_hdr_ptr[reuse_idx] = (void *)hNetBufTx->hHdr;
845  } else {
846  /* Check integrity of driver/application */
847  assert(seg_hdr_ptr[reuse_idx] == hNetBufTx->hHdr);
848  }
849  }
850 
851  // Release the TX buffer and the segment will be transmitted
852  if ((status = NT_NetTxRelease(hNetTx, hNetBufTx)) != NT_SUCCESS) {
853  // Get the status code as text
854  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
855  fprintf(stderr, "NT_NetTxRelease() failed: %s\n", errorBuffer);
856  continue;
857  }
858 
859  // Update our statistics counters
860  segmentsSent++;
861  packetsSent += packetsInSegment[reuse_idx];
862 
863  if (((segmentsSent & 0xF) == 0) || (txRate < 10000000)) {
864  printf("Sent: %8d segments\r", segmentsSent);
865  fflush(stdout);
866  }
867 
868  if (maxPacketsInSegment != 0xFFFFFFFF) {
869  // If we are using slow transmit, we don't want to fill up the adapter with
870  // packets. Therefore we only want to have <maxPacketsInAdapter> number of
871  // packets waiting in the adapter to be transmitted. Taht means that the
872  // difference between the number of packets sent to the adapter and the
873  // number of packets TX'ed by the adapter must be less than <maxPacketsInAdapter>
874  // before we are sending more packets to the adapter.
875  do {
876  // Read the statistics counters
878  hStat.u.query_v3.poll=0; // Wait for a new set
879  hStat.u.query_v3.clear=0; // Clear statistics
880  if ((status = NT_StatRead(hStatStream, &hStat)) != NT_SUCCESS) {
881  // Get the status code as text
882  NT_ExplainError(status, errorBuffer, sizeof(errorBuffer));
883  fprintf(stderr, "NT_StatRead() failed: %s\n", errorBuffer);
884  }
885  hStat.u.query_v3.data.port.aPorts[txPort].tx.valid.RMON1 = 0;
886  } while (hStat.u.query_v3.data.port.aPorts[txPort].tx.valid.RMON1 &&
887  (maxPacketsInAdapter < (packetsSent - hStat.u.query_v3.data.port.aPorts[txPort].tx.RMON1.pkts))
888  && (appRunning==1) && status == NT_SUCCESS);
889  }
890  }
891 
892  if (status == NT_SUCCESS) {
893  //
894  // Wait for the adapter to transmit all of the buffered packets.
895  //
896  appRunning=1; // Wait for completion or ctrl-c
897  (void)_NetTxDataCheckHbInfo(&hNetTx, true /* Wait for completion */);
898  }
899 
900  if (packetsSent > 0)
901  printf("\n\nSent: %d segments and %llu packets\n\n", segmentsSent, (long long int)packetsSent);
902 
903  // Close the TX stream
904  (void)NT_NetTxClose(hNetTx);
905 
906  // Close the stat stream
907  (void)NT_StatClose(hStatStream);
908 
909  //
910  // Restore transmission limit
911  //
912  if (bTxPortRateLimitSet) {
913  (void)_TxPortRateLimit(txPort, oldTxRate, NULL);
914  }
915 
916  // Close down the NTAPI library
917  NT_Done();
918 
919  return status;
920 }