deal with NIC hardware offloads

Currently OPTE assumes there are no NIC hardware offloads in play. This was in issue in the lab env where the underlying NIC was providing partial Tx checksum support (OS provides pseudo-header sum, NIC does the rest) and OPTE was erroneously modifying the TCP header checksum to include the NAT port modifications (not part of the pseudo-header).

The offload flags are defined in pattr.h and stored in the db_struioun.cksum.flags field.

rpz@buskin ~ $ pfexec dtrace -qn 'sdt:opte::tx { this->mp = (mblk_t *)arg0; this->rptr = this->mp->b_rptr; this->eh = (struct ether_header *)this->rptr; this->iph = (ipha_t *)(this->rptr + 14); this->tcph = (tcpha_t *)(this->rptr + 14 + 20); print(*this->mp->b_datap); print(*this->eh); print(*this->iph); print(*this->tcph); }'
struct datab {
    frtn_t *db_frtnp = 0xfffffeb259c70030
    unsigned char *db_base = 0xfffffeb291234c90
    unsigned char *db_lim = 0xfffffeb291234d1a
    unsigned char db_ref = 0x1
    unsigned char db_type = 0
    unsigned char db_flags = 0
    unsigned char db_struioflag = 0
    pid_t db_cpid = 0xffffffff
    void *db_cache = 0xfffffeb1876ee008
    struct msgb *db_mblk = 0xfffffeb1cf92ba80
    void (*)() db_free = genunix`dblk_lastfree_desb
    void (*)() db_lastfree = genunix`dblk_lastfree_desb
    intptr_t db_cksumstart = 0x14
    intptr_t db_cksumend = 0x3c
    intptr_t db_cksumstuff = 0x24
    union db_struioun = {
        double enforce_alignment = -3.7481675e-25
        unsigned char [8] data = [ 0, 0, 0xdd, 0xba, 0x2, 0, 0xdd, 0xba ]
        struct cksum = {
            union cksum_val = {
                uint32_t u32 = 0xbadd0000
                uint16_t u16 = 0
            }
            uint16_t flags = 0x2
            uint16_t pad = 0xbadd
        }
    }
    struct fthdr *db_fthdr = 0
    cred_t *db_credp = 0

In this case the uint16_t flags = 0x2 indicates partial ULP checksum.

/*
 * Values for hcksum_flags
 */
#define	HCK_IPV4_HDRCKSUM	0x01	/* On Transmit: Compute IP header */
					/* checksum in hardware. */

#define	HCK_IPV4_HDRCKSUM_OK	0x01	/* On Receive: IP header checksum */
					/* was verified by h/w and is */
					/* correct. */

#define	HCK_PARTIALCKSUM	0x02	/* On Transmit: Compute partial 1's */
					/* complement checksum based on */
					/* start, stuff and end offsets. */
					/* On Receive : Partial checksum */
					/* computed and attached. */

#define	HCK_FULLCKSUM		0x04	/* On Transmit: Compute full(in case */
					/* of TCP/UDP, full is pseudo-header */
					/* + header + payload) checksum for */
					/* this packet. */
					/* On Receive : Full checksum  */
					/* computed in h/w and is attached */

#define	HCK_FULLCKSUM_OK	0x08	/* On Transmit: N/A */
					/* On Receive: Full checksum status */
					/* If set, implies full checksum */
					/* computation was successful */
					/* i.e. checksum was correct. */
					/* If it is not set, IP will also */
					/* check the attached h/w computed */
					/* checksum value to determine if */
					/* checksum was bad */

In the meantime, if you are trying to run OPTE, make sure to disable Tx checksum offloads.

Start with the offload flag definitions in pattr.h and trace how db_struioun.cksum.flags is handled during transmit and NAT port changes. Reproduce the partial ULP checksum case described in the issue, then verify that OPTE no longer modifies checksum fields incorrectly when Tx hardware offload flags are present.