transmit_segment_example.c Source File

/*
 *
 * Copyright 2023 Napatech A/S. All Rights Reserved.
 *
 * 1. Copying, modification, and distribution of this file, or executable
 * versions of this file, is governed by the terms of the Napatech Software
 * license agreement under which this file was made available. If you do not
 * agree to the terms of the license do not install, copy, access or
 * otherwise use this file.
 *
 * 2. Under the Napatech Software license agreement you are granted a
 * limited, non-exclusive, non-assignable, copyright license to copy, modify
 * and distribute this file in conjunction with Napatech SmartNIC's and
 * similar hardware manufactured or supplied by Napatech A/S.
 *
 * 3. The full Napatech Software license agreement is included in this
 * distribution, please see "NP-0405 Napatech Software license
 * agreement.pdf"
 *
 * 4. Redistributions of source code must retain this copyright notice,
 * list of conditions and the following disclaimer.
 *
 * THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES, EXPRESS OR
 * IMPLIED, AND NAPATECH DISCLAIMS ALL IMPLIED WARRANTIES INCLUDING ANY
 * IMPLIED WARRANTY OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, OR OF
 * FITNESS FOR A PARTICULAR PURPOSE. TO THE EXTENT NOT PROHIBITED BY
 * APPLICABLE LAW, IN NO EVENT SHALL NAPATECH BE LIABLE FOR PERSONAL INJURY,
 * OR ANY INCIDENTAL, SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES WHATSOEVER,
 * INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF PROFITS, CORRUPTION OR
 * LOSS OF DATA, FAILURE TO TRANSMIT OR RECEIVE ANY DATA OR INFORMATION,
 * BUSINESS INTERRUPTION OR ANY OTHER COMMERCIAL DAMAGES OR LOSSES, ARISING
 * OUT OF OR RELATED TO YOUR USE OR INABILITY TO USE NAPATECH SOFTWARE OR
 * SERVICES OR ANY THIRD PARTY SOFTWARE OR APPLICATIONS IN CONJUNCTION WITH
 * THE NAPATECH SOFTWARE OR SERVICES, HOWEVER CAUSED, REGARDLESS OF THE THEORY
 * OF LIABILITY (CONTRACT, TORT OR OTHERWISE) AND EVEN IF NAPATECH HAS BEEN
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. SOME JURISDICTIONS DO NOT ALLOW
 * THE EXCLUSION OR LIMITATION OF LIABILITY FOR PERSONAL INJURY, OR OF
 * INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY TO YOU.
 *
 *

 */

/**
 * @example net/transmit_segment/transmit_segment_example.c
 * @section transmit_segment_example_description Description
 *
 * This source file is an example of how to transmit packets using the segment
 * interface in NTAPI. The example will transmit 2500000 packets with a size of
 * 1024 bytes from port 0. The packet contains an incrementing 32bit pattern.
 *
 * The following NTAPI functions are used:
 * - @ref NT_Init()
 * - @ref NT_NetTxOpen()
 * - @ref NT_NetTxGet()
 *   - @ref NT_NET_GET_PKT_L2_PTR()
 *   - @ref NT_NET_SET_PKT_CAP_LENGTH()
 *   - @ref NT_NET_SET_PKT_WIRE_LENGTH()
 *   - @ref NT_NET_SET_PKT_CLEAR_DESCR_NT()
 *   - @ref NT_NET_SET_PKT_DESCR_TYPE_NT()
 *   - @ref NT_NET_SET_PKT_TXPORT()
 *   - @ref NT_NET_SET_PKT_TXIGNORE()
 *   - @ref NT_NET_UPDATE_PKT_L2_PTR()
 * - @ref NT_NetTxRelease()
 * - @ref NT_NetTxClose()
 * - @ref NT_Done()
 * - @ref NT_ExplainError()
 *
 * @section transmit_segment_example_prerequisites Prerequisites
 * - The ntservice.ini must have at least one HostBuffersTx defined. Below is
 * an example of a minimum ini-file. It will create a 4MB TX hostbuffer from
 * NUMA node 0.
 *
 * @code
 * [System]
 * TimestampFormat = NATIVE
 *
 * [Adapter0]
 * AdapterType     = NT20E2
 * BusId           = 00:0a:00.00
 * HostBuffersTx   = [1,4,0]
 * @endcode
 *
 * @section transmit_segment_example_flow Program flow
 * @{
 * The following is required to transmit packages:
 * - \#include/nt.h - Applications/Tools only need to include @ref
 *   nt.h to obtain prototypes, macros etc. from NTAPI.
 * - @ref NT_Init(@ref NTAPI_VERSION) - Initialize the NTAPI
 *   library. @ref NTAPI_VERSION is a define that describes the version
 *   of the API described in the header files included by @ref
 *   nt.h. NT_Init() will ask the NTAPI library to convert return data
 *   to the @ref NTAPI_VERSION if possible. This will ensure that
 *   applications can run on NTAPI libraries of newer versions.
 * - @ref NT_NetTxOpen() - Open a hostbuffer than can transmit packets to port 0.
 * - @ref NT_NetTxGet() - Get an empty tx buffer. This will get a 1024 byte
 *   wire length packet buffer that will be sent onto port 0 when
 *   released.
 * - @ref _nt_net_build_pkt_netbuf() - Initialize a view into the fiest packet
 *   of the segment.
 * - While there is still room in the segment:
 *   - @ref NT_NET_SET_PKT_CLEAR_DESCR_NT() - Zero the packet descriptor.
 *   - @ref NT_NET_SET_PKT_DESCR_TYPE_NT() - Set the packet descriptor type.
 *   - @ref NT_NET_UPDATE_PKT_L2_PTR() - Recalculate the pointer to the packet
 *   payload.
 *   - @ref NT_NET_SET_PKT_TXPORT() - Set the transmit port in the packet
 *   descriptor.
 *   - @ref NT_NET_SET_PKT_CAP_LENGTH() - Set the aligned packet buffer size.
 *   This macro automatically aligns and adds the descriptor length.
 *   - @ref NT_NET_SET_PKT_WIRE_LENGTH() - Set the payload length.
 *   - @ref NT_NET_SET_PKT_TXIGNORE() - If this is a padding packet, ignore
 *   the packet.
 *   - _nt_net_get_next_packet() - Move the view to the next packet in the
 *   buffer. Loop.
 * - @ref NT_NetTxRelease() - Release the tx packet buffer. Once a tx
 *   buffer is released it will be transmitted
 * - @ref NT_NetTxClose() - Close the TX stream.
 * - @ref NT_Done() - Close down the NTAPI library.
 *
 *<hr>
 * @section transmit_segment_example_code Code
 * @}
 */

// Include this in order to access the Napatech API
#include <nt.h>

#ifdef WIN32
  #include <windows.h>
#else
  #include <unistd.h>
#endif

#if defined(WIN32) || defined(WIN64)
  #define snprintf(dst, ...)    _snprintf_s((dst), _countof(dst), __VA_ARGS__)
#endif

 // default TX packet test setup
#define PACKETS      2500000
#define PACKET_SIZE  1024 // Packet size to transmit (incl crc.)
#define SEGMENT_SIZE (1024 * 1024)
#define PORT         0

// A segment has a fixed size, and if we do not use it all, we have to pad
// it with TX_IGNORE packets. However, a TX_IGNORE packet uses at least
// 64 + NT_DESCR_NT_LENGTH bytes, so we must stop adding packets if it
// would leave less space left than that.
#define MIN_PADDING_SIZE_WITH_DESCR  (64 + NT_DESCR_NT_LENGTH)
#define PADDING_THRESHOLD (PACKET_SIZE + NT_DESCR_NT_LENGTH + MIN_PADDING_SIZE_WITH_DESCR)

static void msleep(int ms)
{
#ifdef _WIN32
    Sleep(ms);
#else
    usleep(ms * 1000);
#endif
}

// printError is a simple convenience function for printing an NTAPI error
// message to stderr.
static void printError(const char *prefix, int errorCode) {
  char errorBuffer[NT_ERRBUF_SIZE];
  NT_ExplainError(errorCode, errorBuffer, sizeof errorBuffer);
  fprintf(stderr, "%s: %s\n", prefix, errorBuffer);
}

int main(void)
{
  //
  // Initialize the API. This also checks if we are compatible with the
  // installed library version.
  //
  int  status;
  bool isFpga4Garch;

  if ((status = NT_Init(NTAPI_VERSION)) != NT_SUCCESS) {
    printError("NT_Init() failed", status);
    return -1;
  }

  //
  // Open info stream to retrieve port info for our port. We are specifically
  // interested in the "maxTxPktSize" parameter, which tells us maximum packet
  // size the adapter can process. This is used later for padding purposes.
  //
  NtInfoStream_t hInfo;
  NtInfo_t info;

  if ((status = NT_InfoOpen(&hInfo,
          "transmit_segment_example_info")) != NT_SUCCESS) {
    printError("NT_InfoOpen() failed", status);
    return -1;
  }

  info.cmd = NT_INFO_CMD_READ_PORT_V9;
  info.u.port_v9.portNo = PORT;
  if ((status = NT_InfoRead(hInfo, &info)) != NT_SUCCESS) {
    NT_InfoClose(hInfo);
    printError("NT_InfoRead() failed", status);
    return -1;
  }

  // Save adapter number of the TX port
  const int adapterNo = info.u.port_v9.data.adapterNo;

  info.cmd = NT_INFO_CMD_READ_PROPERTY;
  snprintf(info.u.property.path, sizeof(info.u.property.path), "Adapter%d.FpgaGeneration", adapterNo);

  if ((status = NT_InfoRead(hInfo, &info)) != NT_SUCCESS) {
    NT_InfoClose(hInfo);
    printError("xNT_InfoRead() failed", status);
    return -1;
  }

  isFpga4Garch = info.u.property.data.u.i >= 4;

  if ((status = NT_InfoClose(hInfo)) != NT_SUCCESS) {
    printError("NT_InfoClose() failed", status);
    return -1;
  }

  // Max packet size (descriptor length added for convenience in later
  // calculations).
  uint64_t maxTxSizeWithDescr = info.u.port_v9.data.capabilities.maxTxPktSize;
  maxTxSizeWithDescr += NT_DESCR_NT_LENGTH;

  //
  // Open a TX stream
  //
  NtNetStreamTx_t hNetTx;

  status = NT_NetTxOpen(&hNetTx, "transmit_segment_example_txstream",
                        1ULL << PORT, NT_NETTX_NUMA_ANY_HB, 0);
  if (status != NT_SUCCESS) {
    printError("NT_NetTxOpen() failed", status);
    return -1;
  }

  //
  // Retrieve a segment, fill it with packets and transmit it, and repeat
  // until we have transmitted the requested amount of packets.
  //
  printf("Commencing transmission\n");

  NtNetBuf_t hNetBufTx;
  int numPackets = 0;

  while (numPackets < PACKETS) {

    // Get a segment TX buffer for this tx stream and port, without timeout.
    if ((status = NT_NetTxGet(hNetTx, &hNetBufTx, PORT, SEGMENT_SIZE,
            NT_NETTX_SEGMENT_OPTION_RAW, -1)) != NT_SUCCESS) {
      printError("NT_NetTxGet() failed", status);
      return -1;
    }

    // Prepare the NetBuf.
    struct NtNetBuf_s pktNetBuf;
    _nt_net_build_pkt_netbuf(hNetBufTx, &pktNetBuf);
    uint64_t spaceLeftInSegment = NT_NET_GET_SEGMENT_LENGTH(hNetBufTx);

    // Fill the segment.
    while (true) {
      if (spaceLeftInSegment == 0)
        break;

      // Our buffers are recycled, so start out by clearing the descriptor.
      NT_NET_SET_PKT_CLEAR_DESCR_NT(&pktNetBuf);
      NT_NET_SET_PKT_DESCR_TYPE_NT(&pktNetBuf);
      NT_NET_UPDATE_PKT_L2_PTR(&pktNetBuf);
      NT_NET_SET_PKT_TXPORT(&pktNetBuf, PORT);

      if (numPackets == PACKETS || (spaceLeftInSegment < PADDING_THRESHOLD)) {

        // We're out of space, or we're done transmitting packets. Pad the rest
        // of the segment. In order to do so, we must calculate the largest
        // possible value for paddingWithDescr that satisfies the following
        // constraint, whilst ensuring that any necessary later padding is also
        // capable of satisfying the constraint:
        //
        //     spaceLeftInSegment >= paddingWithDescr <= maxTxSizeWithDescr
        //
        uint64_t paddingWithDescr;
        if (spaceLeftInSegment <= maxTxSizeWithDescr) {

          // We can fill the rest of the segment in one go.
          paddingWithDescr = spaceLeftInSegment;
        } else if ((spaceLeftInSegment - MIN_PADDING_SIZE_WITH_DESCR) >=
            maxTxSizeWithDescr) {

          // There is room for many more packets, so pad as much as we can.
          paddingWithDescr = maxTxSizeWithDescr;
        } else {

          // We cannot fill the entire segment, but there's not much more room
          // left, so make sure we leave room for another padding packet.
          paddingWithDescr = spaceLeftInSegment - MIN_PADDING_SIZE_WITH_DESCR;
        }

        uint64_t paddingNoDescr = paddingWithDescr - NT_DESCR_NT_LENGTH;

        // NT_NET_SET_PKT_CAP_LENGTH handles alignment, and adds the length
        // of the descriptor before assignment.
        NT_NET_SET_PKT_CAP_LENGTH(&pktNetBuf, (uint16_t)paddingNoDescr);
        NT_NET_SET_PKT_WIRE_LENGTH(&pktNetBuf, (uint16_t)paddingNoDescr);

        // Do not transmit this padding packet.
        NT_NET_SET_PKT_TXIGNORE(&pktNetBuf, 1);
      } else {

        // NT_NET_SET_PKT_CAP_LENGTH handles alignment, and adds the length
        // of the descriptor before assignment.
        NT_NET_SET_PKT_CAP_LENGTH(&pktNetBuf, (uint16_t)PACKET_SIZE);
        NT_NET_SET_PKT_WIRE_LENGTH(&pktNetBuf, (uint16_t)PACKET_SIZE);

        // Fill the packet with an incrementing payload. Note that this will
        // result in a garbage ethernet frame.
        uint32_t *ptr = (uint32_t*)NT_NET_GET_PKT_L2_PTR(&pktNetBuf);
        for (uint32_t i = 0; i < PACKET_SIZE/4; i++) {
          *(ptr+i) = i;
        }

        //On 4Garch the ethernet FCS is calculated by the FPGA itself, and it is
        //not mandatory to specify the recalculation. Calculating it anyway will
        //make no harm however. At 3Garch the recalculation is mandatory.
        if (!isFpga4Garch)
          NT_NET_SET_PKT_RECALC_L2_CRC(&pktNetBuf, 1);

        numPackets++;
      }


      // Get the next NetBuf and get the remaining segment size.
      spaceLeftInSegment = _nt_net_get_next_packet(hNetBufTx,
          NT_NET_GET_SEGMENT_LENGTH(hNetBufTx), &pktNetBuf);
    }

    // Release the TX buffer to transmit the segment.
    if ((status = NT_NetTxRelease(hNetTx, hNetBufTx)) != NT_SUCCESS) {
      printError("NT_NetTxRelease() failed", status);
      return -1;
    }
  }

  //Wait until all packet have been delivered to the FPGA and this when all
  //packets have left host memory
  int timeOut = 0;

  while (true) {
    if (timeOut >= 1000) { //Wait max 1 second
      printError("Timeout waiting for data to be sent", NT_ERROR_OPERATION_TIMEOUT);
      break;
    }

    // Evaluate if hostbuffer contents have been delivered to FPGA
    // This is used to detect when Tx data has left host memory
    NtNetTx_t ntNetTx;
    ntNetTx.cmd = NT_NETTX_READ_CMD_GET_HB_INFO;
    status = NT_NetTxRead(hNetTx, &ntNetTx);

    if (status != NT_SUCCESS) {
      printError("NT_NetTxRead failed", status);
      break;
    }

    size_t nHbSizeTotal = ntNetTx.u.hbInfo.aHostBuffer[0].size;
    size_t nHbSizeAvail = ntNetTx.u.hbInfo.aHostBuffer[0].available;
    size_t nHbSizeRel = ntNetTx.u.hbInfo.aHostBuffer[0].released;
    size_t nHbSizeDeq = ntNetTx.u.hbInfo.aHostBuffer[0].dequeued;

    if ((nHbSizeAvail == nHbSizeTotal) && (nHbSizeRel == 0) && (nHbSizeDeq == 0))
      break;

    msleep(1); // Dont busy wait but sleep 1 ms
    timeOut++;
  };

  //printf("timeOut = %dms\n", timeOut); //Find timeout value
  printf("Done: %d packets sent\n", numPackets);

  // Close the TX stream
  NT_NetTxClose(hNetTx);

  // Close the API
  NT_Done();

  return 0;
}