em_txrx.c 22.5 KB
Newer Older
1
/*-
2
3
 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org>
 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io>
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

28
29
30
/* $FreeBSD$ */
#include "if_em.h"

Sean Bruno's avatar
Sean Bruno committed
31
#ifdef RSS
32
33
34
35
36
37
38
39
40
41
42
43
44
#include <net/rss_config.h>
#include <netinet/in_rss.h>
#endif

#ifdef VERBOSE_DEBUG
#define DPRINTF device_printf
#else
#define DPRINTF(...)
#endif

/*********************************************************************
 *  Local Function prototypes
 *********************************************************************/
45
static int em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, u32 *txd_upper,
Sean Bruno's avatar
Sean Bruno committed
46
    u32 *txd_lower);
47
static int em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi,
Sean Bruno's avatar
Sean Bruno committed
48
    u32 *txd_upper, u32 *txd_lower);
49
static int em_isc_txd_encap(void *arg, if_pkt_info_t pi);
Sean Bruno's avatar
Sean Bruno committed
50
51
52
53
54
55
56
static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear);
static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru);
static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
    qidx_t pidx);
static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
    qidx_t budget);
57
58
static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);

Sean Bruno's avatar
Sean Bruno committed
59
static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru);
60

Sean Bruno's avatar
Sean Bruno committed
61
62
static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
   qidx_t budget);
63
64
static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);

65
static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t);
66
static int em_determine_rsstype(u32 pkt_info);
67
68
extern int em_intr(void *arg);

Sean Bruno's avatar
Sean Bruno committed
69
struct if_txrx em_txrx = {
70
71
72
73
74
75
76
77
	.ift_txd_encap = em_isc_txd_encap,
	.ift_txd_flush = em_isc_txd_flush,
	.ift_txd_credits_update = em_isc_txd_credits_update,
	.ift_rxd_available = em_isc_rxd_available,
	.ift_rxd_pkt_get = em_isc_rxd_pkt_get,
	.ift_rxd_refill = em_isc_rxd_refill,
	.ift_rxd_flush = em_isc_rxd_flush,
	.ift_legacy_intr = em_intr
78
79
};

Sean Bruno's avatar
Sean Bruno committed
80
struct if_txrx lem_txrx = {
81
82
83
84
85
86
87
88
	.ift_txd_encap = em_isc_txd_encap,
	.ift_txd_flush = em_isc_txd_flush,
	.ift_txd_credits_update = em_isc_txd_credits_update,
	.ift_rxd_available = lem_isc_rxd_available,
	.ift_rxd_pkt_get = lem_isc_rxd_pkt_get,
	.ift_rxd_refill = lem_isc_rxd_refill,
	.ift_rxd_flush = em_isc_rxd_flush,
	.ift_legacy_intr = em_intr
89
90
};

Sean Bruno's avatar
Sean Bruno committed
91
92
93
extern if_shared_ctx_t em_sctx;

void
94
em_dump_rs(struct e1000_softc *sc)
Sean Bruno's avatar
Sean Bruno committed
95
{
96
	if_softc_ctx_t scctx = sc->shared;
Sean Bruno's avatar
Sean Bruno committed
97
98
99
100
101
102
103
104
	struct em_tx_queue *que;
	struct tx_ring *txr;
	qidx_t i, ntxd, qid, cur;
	int16_t rs_cidx;
	uint8_t status;

	printf("\n");
	ntxd = scctx->isc_ntxd[0];
105
106
	for (qid = 0; qid < sc->tx_num_queues; qid++) {
		que = &sc->tx_queues[qid];
Sean Bruno's avatar
Sean Bruno committed
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
		txr =  &que->txr;
		rs_cidx = txr->tx_rs_cidx;
		if (rs_cidx != txr->tx_rs_pidx) {
			cur = txr->tx_rsq[rs_cidx];
			status = txr->tx_base[cur].upper.fields.status;
			if (!(status & E1000_TXD_STAT_DD))
				printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur);
		} else {
			rs_cidx = (rs_cidx-1)&(ntxd-1);
			cur = txr->tx_rsq[rs_cidx];
			printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d  ", qid, rs_cidx, cur);
		}
		printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, txr->tx_rs_pidx);
		for (i = 0; i < ntxd; i++) {
			if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD)
				printf("%d set ", i);
		}
		printf("\n");
	}
}
127
128
129
130
131
132
133
134

/**********************************************************************
 *
 *  Setup work for hardware segmentation offload (TSO) on
 *  adapters using advanced tx descriptors
 *
 **********************************************************************/
static int
135
em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower)
136
{
137
138
	if_softc_ctx_t scctx = sc->shared;
	struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
Sean Bruno's avatar
Sean Bruno committed
139
	struct tx_ring *txr = &que->txr;
140
	struct e1000_context_desc *TXD;
Sean Bruno's avatar
Sean Bruno committed
141
	int cur, hdr_len;
142
143

	hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
Sean Bruno's avatar
Sean Bruno committed
144
	*txd_lower = (E1000_TXD_CMD_DEXT |	/* Extended descr type */
145
146
147
148
149
150
151
		      E1000_TXD_DTYP_D |	/* Data descr type */
		      E1000_TXD_CMD_TSE);	/* Do TSE on this packet */

	/* IP and/or TCP header checksum calculation and insertion. */
	*txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;

	cur = pi->ipi_pidx;
Sean Bruno's avatar
Sean Bruno committed
152
153
154
	TXD = (struct e1000_context_desc *)&txr->tx_base[cur];

	/*
155
156
157
158
159
160
161
162
163
	 * Start offset for header checksum calculation.
	 * End offset for header checksum calculation.
	 * Offset of place put the checksum.
	 */
	TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
	TXD->lower_setup.ip_fields.ipcse =
	    htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1);
	TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);

Sean Bruno's avatar
Sean Bruno committed
164
	/*
165
166
167
168
169
170
171
172
	 * Start offset for payload checksum calculation.
	 * End offset for payload checksum calculation.
	 * Offset of place to put the checksum.
	 */
	TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
	TXD->upper_setup.tcp_fields.tucse = 0;
	TXD->upper_setup.tcp_fields.tucso =
	    pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum);
Sean Bruno's avatar
Sean Bruno committed
173
174

	/*
175
176
177
178
179
180
	 * Payload size per packet w/o any headers.
	 * Length of all headers up to payload.
	 */
	TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz);
	TXD->tcp_seg_setup.fields.hdr_len = hdr_len;

181
	TXD->cmd_and_length = htole32(sc->txd_cmd |
182
183
184
185
186
				E1000_TXD_CMD_DEXT |	/* Extended descr */
				E1000_TXD_CMD_TSE |	/* TSE context */
				E1000_TXD_CMD_IP |	/* Do IP csum */
				E1000_TXD_CMD_TCP |	/* Do TCP checksum */
				      (pi->ipi_len - hdr_len)); /* Total len */
Kevin Bowling's avatar
Kevin Bowling committed
187
	txr->tx_tso = true;
188
189
190
191

	if (++cur == scctx->isc_ntxd[0]) {
		cur = 0;
	}
192
	DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur);
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
	return (cur);
}

#define TSO_WORKAROUND 4
#define DONT_FORCE_CTX 1


/*********************************************************************
 *  The offload context is protocol specific (TCP/UDP) and thus
 *  only needs to be set when the protocol changes. The occasion
 *  of a context change can be a performance detriment, and
 *  might be better just disabled. The reason arises in the way
 *  in which the controller supports pipelined requests from the
 *  Tx data DMA. Up to four requests can be pipelined, and they may
 *  belong to the same packet or to multiple packets. However all
 *  requests for one packet are issued before a request is issued
 *  for a subsequent packet and if a request for the next packet
 *  requires a context change, that request will be stalled
 *  until the previous request completes. This means setting up
 *  a new context effectively disables pipelined Tx data DMA which
 *  in turn greatly slow down performance to send small sized
Sean Bruno's avatar
Sean Bruno committed
214
 *  frames.
215
216
217
 **********************************************************************/

static int
218
em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower)
219
{
Sean Bruno's avatar
Sean Bruno committed
220
	 struct e1000_context_desc *TXD = NULL;
221
222
	if_softc_ctx_t scctx = sc->shared;
	struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
Sean Bruno's avatar
Sean Bruno committed
223
224
225
226
227
	struct tx_ring *txr = &que->txr;
	int csum_flags = pi->ipi_csum_flags;
	int cur, hdr_len;
	u32 cmd;

228
229
	cur = pi->ipi_pidx;
	hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
230
	cmd = sc->txd_cmd;
231
232
233
234
235
236
237
238
239

	/*
	 * The 82574L can only remember the *last* context used
	 * regardless of queue that it was use for.  We cannot reuse
	 * contexts on this hardware platform and must generate a new
	 * context every time.  82574L hardware spec, section 7.2.6,
	 * second note.
	 */
	if (DONT_FORCE_CTX &&
240
	    sc->tx_num_queues == 1 &&
241
242
243
244
245
246
247
248
249
250
251
252
253
254
	    txr->csum_lhlen == pi->ipi_ehdrlen &&
	    txr->csum_iphlen == pi->ipi_ip_hlen &&
	    txr->csum_flags == csum_flags) {
		/*
		 * Same csum offload context as the previous packets;
		 * just return.
		 */
		*txd_upper = txr->csum_txd_upper;
		*txd_lower = txr->csum_txd_lower;
		return (cur);
	}

	TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
	if (csum_flags & CSUM_IP) {
Sean Bruno's avatar
Sean Bruno committed
255
		*txd_upper |= E1000_TXD_POPTS_IXSM << 8;
256
257
258
259
260
261
262
263
264
265
266
267
268
269
		/*
		 * Start offset for header checksum calculation.
		 * End offset for header checksum calculation.
		 * Offset of place to put the checksum.
		 */
		TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
		TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
		TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
		cmd |= E1000_TXD_CMD_IP;
	}

	if (csum_flags & (CSUM_TCP|CSUM_UDP)) {
		uint8_t tucso;

Sean Bruno's avatar
Sean Bruno committed
270
		*txd_upper |= E1000_TXD_POPTS_TXSM << 8;
271
272
273
274
275
276
277
		*txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;

		if (csum_flags & CSUM_TCP) {
			tucso = hdr_len + offsetof(struct tcphdr, th_sum);
			cmd |= E1000_TXD_CMD_TCP;
		} else
			tucso = hdr_len + offsetof(struct udphdr, uh_sum);
Sean Bruno's avatar
Sean Bruno committed
278
279
280
		TXD->upper_setup.tcp_fields.tucss = hdr_len;
		TXD->upper_setup.tcp_fields.tucse = htole16(0);
		TXD->upper_setup.tcp_fields.tucso = tucso;
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
	}

	txr->csum_lhlen = pi->ipi_ehdrlen;
	txr->csum_iphlen = pi->ipi_ip_hlen;
	txr->csum_flags = csum_flags;
	txr->csum_txd_upper = *txd_upper;
	txr->csum_txd_lower = *txd_lower;

	TXD->tcp_seg_setup.data = htole32(0);
	TXD->cmd_and_length =
		htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);

	if (++cur == scctx->isc_ntxd[0]) {
		cur = 0;
	}
296
	DPRINTF(iflib_get_dev(sc->ctx), "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n",
297
298
299
300
301
302
303
		      csum_flags, *txd_upper, *txd_lower, hdr_len, cmd);
	return (cur);
}

static int
em_isc_txd_encap(void *arg, if_pkt_info_t pi)
{
304
	struct e1000_softc *sc = arg;
Sean Bruno's avatar
Sean Bruno committed
305
306
307
308
309
310
311
312
313
	if_softc_ctx_t scctx = sc->shared;
	struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
	struct tx_ring *txr = &que->txr;
	bus_dma_segment_t *segs = pi->ipi_segs;
	int nsegs = pi->ipi_nsegs;
	int csum_flags = pi->ipi_csum_flags;
	int i, j, first, pidx_last;
	u32 txd_flags, txd_upper = 0, txd_lower = 0;

314
	struct e1000_tx_desc *ctxd = NULL;
Sean Bruno's avatar
Sean Bruno committed
315
316
317
318
319
	bool do_tso, tso_desc;
	qidx_t ntxd;

	txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0;
	i = first = pi->ipi_pidx;
320
	do_tso = (csum_flags & CSUM_TSO);
Kevin Bowling's avatar
Kevin Bowling committed
321
	tso_desc = false;
Sean Bruno's avatar
Sean Bruno committed
322
323
	ntxd = scctx->isc_ntxd[0];
	/*
324
325
326
327
328
	 * TSO Hardware workaround, if this packet is not
	 * TSO, and is only a single descriptor long, and
	 * it follows a TSO burst, then we need to add a
	 * sentinel descriptor to prevent premature writeback.
	 */
Kevin Bowling's avatar
Kevin Bowling committed
329
	if ((!do_tso) && (txr->tx_tso == true)) {
330
		if (nsegs == 1)
Kevin Bowling's avatar
Kevin Bowling committed
331
332
			tso_desc = true;
		txr->tx_tso = false;
333
334
335
336
337
	}

	/* Do hardware assists */
	if (do_tso) {
		i = em_tso_setup(sc, pi, &txd_upper, &txd_lower);
Kevin Bowling's avatar
Kevin Bowling committed
338
		tso_desc = true;
339
	} else if (csum_flags & EM_CSUM_OFFLOAD) {
340
341
342
343
		i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower);
	}

	if (pi->ipi_mflags & M_VLANTAG) {
Sean Bruno's avatar
Sean Bruno committed
344
		/* Set the vlan id. */
345
		txd_upper |= htole16(pi->ipi_vtag) << 16;
Sean Bruno's avatar
Sean Bruno committed
346
347
		/* Tell hardware to add tag */
		txd_lower |= htole32(E1000_TXD_CMD_VLE);
348
349
350
	}

	DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i);
351
	/* XXX sc->pcix_82544 -- lem_fill_descriptors */
352
353
354
355
356
357
358
359
360
361
362
363
364

	/* Set up our transmit descriptors */
	for (j = 0; j < nsegs; j++) {
		bus_size_t seg_len;
		bus_addr_t seg_addr;
		uint32_t cmd;

		ctxd = &txr->tx_base[i];
		seg_addr = segs[j].ds_addr;
		seg_len = segs[j].ds_len;
		cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd;

		/*
Sean Bruno's avatar
Sean Bruno committed
365
366
367
368
		 * TSO Workaround:
		 * If this is the last descriptor, we want to
		 * split it so we have a small final sentinel
		 */
369
370
371
372
373
374
		if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
			seg_len -= TSO_WORKAROUND;
			ctxd->buffer_addr = htole64(seg_addr);
			ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
			ctxd->upper.data = htole32(txd_upper);

Sean Bruno's avatar
Sean Bruno committed
375
			if (++i == scctx->isc_ntxd[0])
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
				i = 0;

			/* Now make the sentinel */
			ctxd = &txr->tx_base[i];
			ctxd->buffer_addr = htole64(seg_addr + seg_len);
			ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND);
			ctxd->upper.data = htole32(txd_upper);
			pidx_last = i;
			if (++i == scctx->isc_ntxd[0])
				i = 0;
			DPRINTF(iflib_get_dev(sc->ctx), "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]);
		} else {
			ctxd->buffer_addr = htole64(seg_addr);
			ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
			ctxd->upper.data = htole32(txd_upper);
			pidx_last = i;
			if (++i == scctx->isc_ntxd[0])
				i = 0;
			DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]);
		}
	}

	/*
Sean Bruno's avatar
Sean Bruno committed
399
	 * Last Descriptor of Packet
400
401
	 * needs End Of Packet (EOP)
	 * and Report Status (RS)
Sean Bruno's avatar
Sean Bruno committed
402
	 */
403
	if (txd_flags && nsegs) {
Sean Bruno's avatar
Sean Bruno committed
404
405
406
407
408
409
		txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
		DPRINTF(iflib_get_dev(sc->ctx), "setting to RS on %d rs_pidx %d first: %d\n", pidx_last, txr->tx_rs_pidx, first);
		txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1);
		MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
	}
	ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags);
410
411
412
	DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i);
	pi->ipi_new_pidx = i;

Sean Bruno's avatar
Sean Bruno committed
413
	return (0);
414
415
416
}

static void
Sean Bruno's avatar
Sean Bruno committed
417
em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
418
{
419
420
	struct e1000_softc *sc = arg;
	struct em_tx_queue *que = &sc->tx_queues[txqid];
421
422
	struct tx_ring *txr = &que->txr;

423
	E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx);
424
425
426
}

static int
Sean Bruno's avatar
Sean Bruno committed
427
em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
428
{
429
430
431
	struct e1000_softc *sc = arg;
	if_softc_ctx_t scctx = sc->shared;
	struct em_tx_queue *que = &sc->tx_queues[txqid];
432
433
	struct tx_ring *txr = &que->txr;

Sean Bruno's avatar
Sean Bruno committed
434
435
436
437
438
439
440
441
442
443
444
445
446
447
	qidx_t processed = 0;
	int updated;
	qidx_t cur, prev, ntxd, rs_cidx;
	int32_t delta;
	uint8_t status;

	rs_cidx = txr->tx_rs_cidx;
	if (rs_cidx == txr->tx_rs_pidx)
		return (0);
	cur = txr->tx_rsq[rs_cidx];
	MPASS(cur != QIDX_INVALID);
	status = txr->tx_base[cur].upper.fields.status;
	updated = !!(status & E1000_TXD_STAT_DD);

448
449
450
451
452
453
454
	if (!updated)
		return (0);

	/* If clear is false just let caller know that there
	 * are descriptors to reclaim */
	if (!clear)
		return (1);
Sean Bruno's avatar
Sean Bruno committed
455
456
457
458

	prev = txr->tx_cidx_processed;
	ntxd = scctx->isc_ntxd[0];
	do {
459
		MPASS(prev != cur);
Sean Bruno's avatar
Sean Bruno committed
460
461
462
		delta = (int32_t)cur - (int32_t)prev;
		if (delta < 0)
			delta += ntxd;
463
		MPASS(delta > 0);
464
		DPRINTF(iflib_get_dev(sc->ctx),
Sean Bruno's avatar
Sean Bruno committed
465
466
467
468
469
470
471
			      "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n",
			      __FUNCTION__, prev, cur, clear, delta);

		processed += delta;
		prev  = cur;
		rs_cidx = (rs_cidx + 1) & (ntxd-1);
		if (rs_cidx  == txr->tx_rs_pidx)
472
			break;
Sean Bruno's avatar
Sean Bruno committed
473
474
475
476
		cur = txr->tx_rsq[rs_cidx];
		MPASS(cur != QIDX_INVALID);
		status = txr->tx_base[cur].upper.fields.status;
	} while ((status & E1000_TXD_STAT_DD));
477

Sean Bruno's avatar
Sean Bruno committed
478
479
	txr->tx_rs_cidx = rs_cidx;
	txr->tx_cidx_processed = prev;
480
481
482
483
	return(processed);
}

static void
Sean Bruno's avatar
Sean Bruno committed
484
lem_isc_rxd_refill(void *arg, if_rxd_update_t iru)
485
{
486
	struct e1000_softc *sc = arg;
487
	if_softc_ctx_t scctx = sc->shared;
Sean Bruno's avatar
Sean Bruno committed
488
	struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx];
489
490
	struct rx_ring *rxr = &que->rxr;
	struct e1000_rx_desc *rxd;
Sean Bruno's avatar
Sean Bruno committed
491
492
493
	uint64_t *paddrs;
	uint32_t next_pidx, pidx;
	uint16_t count;
494
	int i;
Sean Bruno's avatar
Sean Bruno committed
495
496
497
498

	paddrs = iru->iru_paddrs;
	pidx = iru->iru_pidx;
	count = iru->iru_count;
499
500
501
502
503
504
505
506
507
508
509
510
511

	for (i = 0, next_pidx = pidx; i < count; i++) {
		rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx];
		rxd->buffer_addr = htole64(paddrs[i]);
		/* status bits must be cleared */
		rxd->status = 0;

		if (++next_pidx == scctx->isc_nrxd[0])
			next_pidx = 0;
	}
}

static void
Sean Bruno's avatar
Sean Bruno committed
512
em_isc_rxd_refill(void *arg, if_rxd_update_t iru)
513
{
514
	struct e1000_softc *sc = arg;
515
	if_softc_ctx_t scctx = sc->shared;
Sean Bruno's avatar
Sean Bruno committed
516
	uint16_t rxqid = iru->iru_qsidx;
517
518
519
	struct em_rx_queue *que = &sc->rx_queues[rxqid];
	struct rx_ring *rxr = &que->rxr;
	union e1000_rx_desc_extended *rxd;
Sean Bruno's avatar
Sean Bruno committed
520
521
522
	uint64_t *paddrs;
	uint32_t next_pidx, pidx;
	uint16_t count;
523
	int i;
Sean Bruno's avatar
Sean Bruno committed
524
525
526
527

	paddrs = iru->iru_paddrs;
	pidx = iru->iru_pidx;
	count = iru->iru_count;
528
529
530
531

	for (i = 0, next_pidx = pidx; i < count; i++) {
		rxd = &rxr->rx_base[next_pidx];
		rxd->read.buffer_addr = htole64(paddrs[i]);
Stephen Hurd's avatar
Stephen Hurd committed
532
533
		/* DD bits must be cleared */
		rxd->wb.upper.status_error = 0;
Sean Bruno's avatar
Sean Bruno committed
534

535
536
537
538
539
540
		if (++next_pidx == scctx->isc_nrxd[0])
			next_pidx = 0;
	}
}

static void
Sean Bruno's avatar
Sean Bruno committed
541
em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx)
542
{
543
	struct e1000_softc *sc = arg;
Sean Bruno's avatar
Sean Bruno committed
544
545
	struct em_rx_queue *que = &sc->rx_queues[rxqid];
	struct rx_ring *rxr = &que->rxr;
546

Sean Bruno's avatar
Sean Bruno committed
547
	E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
548
549
550
}

static int
Sean Bruno's avatar
Sean Bruno committed
551
lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
552
{
553
	struct e1000_softc *sc = arg;
554
	if_softc_ctx_t scctx = sc->shared;
Sean Bruno's avatar
Sean Bruno committed
555
556
	struct em_rx_queue *que = &sc->rx_queues[rxqid];
	struct rx_ring *rxr = &que->rxr;
557
	struct e1000_rx_desc *rxd;
Sean Bruno's avatar
Sean Bruno committed
558
559
560
	u32 staterr = 0;
	int cnt, i;

Stephen Hurd's avatar
Stephen Hurd committed
561
	for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
562
563
564
565
566
567
568
569
570
571
572
573
574
575
		rxd = (struct e1000_rx_desc *)&rxr->rx_base[i];
		staterr = rxd->status;

		if ((staterr & E1000_RXD_STAT_DD) == 0)
			break;
		if (++i == scctx->isc_nrxd[0])
			i = 0;
		if (staterr & E1000_RXD_STAT_EOP)
			cnt++;
	}
	return (cnt);
}

static int
Sean Bruno's avatar
Sean Bruno committed
576
em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
577
{
578
	struct e1000_softc *sc = arg;
579
	if_softc_ctx_t scctx = sc->shared;
Sean Bruno's avatar
Sean Bruno committed
580
581
	struct em_rx_queue *que = &sc->rx_queues[rxqid];
	struct rx_ring *rxr = &que->rxr;
582
	union e1000_rx_desc_extended *rxd;
Sean Bruno's avatar
Sean Bruno committed
583
584
585
	u32 staterr = 0;
	int cnt, i;

Stephen Hurd's avatar
Stephen Hurd committed
586
	for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
587
588
589
590
591
		rxd = &rxr->rx_base[i];
		staterr = le32toh(rxd->wb.upper.status_error);

		if ((staterr & E1000_RXD_STAT_DD) == 0)
			break;
592
		if (++i == scctx->isc_nrxd[0])
593
594
595
596
597
598
599
600
601
602
			i = 0;
		if (staterr & E1000_RXD_STAT_EOP)
			cnt++;
	}
	return (cnt);
}

static int
lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
{
603
604
605
	struct e1000_softc *sc = arg;
	if_softc_ctx_t scctx = sc->shared;
	struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
Sean Bruno's avatar
Sean Bruno committed
606
	struct rx_ring *rxr = &que->rxr;
607
	struct e1000_rx_desc *rxd;
Sean Bruno's avatar
Sean Bruno committed
608
609
610
611
	u16 len;
	u32 status, errors;
	bool eop;
	int i, cidx;
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630

	status = errors = i = 0;
	cidx = ri->iri_cidx;

	do {
		rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx];
		status = rxd->status;
		errors = rxd->errors;

		/* Error Checking then decrement count */
		MPASS ((status & E1000_RXD_STAT_DD) != 0);

		len = le16toh(rxd->length);
		ri->iri_len += len;

		eop = (status & E1000_RXD_STAT_EOP) != 0;

		/* Make sure bad packets are discarded */
		if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
631
			sc->dropped_pkts++;
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
			/* XXX fixup if common */
			return (EBADMSG);
		}

		ri->iri_frags[i].irf_flid = 0;
		ri->iri_frags[i].irf_idx = cidx;
		ri->iri_frags[i].irf_len = len;
		/* Zero out the receive descriptors status. */
		rxd->status = 0;

		if (++cidx == scctx->isc_nrxd[0])
			cidx = 0;
		i++;
	} while (!eop);

	/* XXX add a faster way to look this up */
648
	if (sc->hw.mac.type >= e1000_82543)
649
		em_receive_checksum(status, errors, ri);
650
651
652
653
654
655
656
657
658
659
660
661
662
663

	if (status & E1000_RXD_STAT_VP) {
		ri->iri_vtag = le16toh(rxd->special);
		ri->iri_flags |= M_VLANTAG;
	}

	ri->iri_nfrags = i;

	return (0);
}

static int
em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
{
664
665
666
	struct e1000_softc *sc = arg;
	if_softc_ctx_t scctx = sc->shared;
	struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
Sean Bruno's avatar
Sean Bruno committed
667
	struct rx_ring *rxr = &que->rxr;
668
669
	union e1000_rx_desc_extended *rxd;

Sean Bruno's avatar
Sean Bruno committed
670
671
672
673
674
	u16 len;
	u32 pkt_info;
	u32 staterr = 0;
	bool eop;
	int i, cidx, vtag;
675
676
677
678
679
680

	i = vtag = 0;
	cidx = ri->iri_cidx;

	do {
		rxd = &rxr->rx_base[cidx];
Sean Bruno's avatar
Sean Bruno committed
681
		staterr = le32toh(rxd->wb.upper.status_error);
682
		pkt_info = le32toh(rxd->wb.lower.mrq);
Sean Bruno's avatar
Sean Bruno committed
683

684
		/* Error Checking then decrement count */
Stephen Hurd's avatar
Stephen Hurd committed
685
		MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
686
687

		len = le16toh(rxd->wb.upper.length);
Sean Bruno's avatar
Sean Bruno committed
688
		ri->iri_len += len;
689
690
691
692
693

		eop = (staterr & E1000_RXD_STAT_EOP) != 0;

		/* Make sure bad packets are discarded */
		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
694
			sc->dropped_pkts++;
695
696
697
698
699
700
701
702
703
704
705
706
707
708
			return EBADMSG;
		}

		ri->iri_frags[i].irf_flid = 0;
		ri->iri_frags[i].irf_idx = cidx;
		ri->iri_frags[i].irf_len = len;
		/* Zero out the receive descriptors status. */
		rxd->wb.upper.status_error &= htole32(~0xFF);

		if (++cidx == scctx->isc_nrxd[0])
			cidx = 0;
		i++;
	} while (!eop);

709
	em_receive_checksum(staterr, staterr >> 24, ri);
710
711
712

	if (staterr & E1000_RXD_STAT_VP) {
		vtag = le16toh(rxd->wb.upper.vlan);
Sean Bruno's avatar
Sean Bruno committed
713
714
	}

715
716
717
	ri->iri_vtag = vtag;
	if (vtag)
		ri->iri_flags |= M_VLANTAG;
Sean Bruno's avatar
Sean Bruno committed
718
719
720

	ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss);
	ri->iri_rsstype = em_determine_rsstype(pkt_info);
721
722

	ri->iri_nfrags = i;
723
724
725
726
727
728
729
730
731
732
733
	return (0);
}

/*********************************************************************
 *
 *  Verify that the hardware indicated that the checksum is valid.
 *  Inform the stack about the status of checksum so that stack
 *  doesn't spend time verifying the checksum.
 *
 *********************************************************************/
static void
734
em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri)
735
{
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
	if (__predict_false(status & E1000_RXD_STAT_IXSM))
		return;

	/* If there is a layer 3 or 4 error we are done */
	if (__predict_false(errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE)))
		return;

	/* IP Checksum Good */
	if (status & E1000_RXD_STAT_IPCS)
		ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);

	/* Valid L4E checksum */
	if (__predict_true(status &
	    (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) {
		ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
		ri->iri_csum_data = htons(0xffff);
752
753
754
	}
}

755
756
757
758
759
760
761
762
/********************************************************************
 *
 *  Parse the packet type to determine the appropriate hash
 *
 ******************************************************************/
static int
em_determine_rsstype(u32 pkt_info)
{
Sean Bruno's avatar
Sean Bruno committed
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
	switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
	case E1000_RXDADV_RSSTYPE_IPV4_TCP:
		return M_HASHTYPE_RSS_TCP_IPV4;
	case E1000_RXDADV_RSSTYPE_IPV4:
		return M_HASHTYPE_RSS_IPV4;
	case E1000_RXDADV_RSSTYPE_IPV6_TCP:
		return M_HASHTYPE_RSS_TCP_IPV6;
	case E1000_RXDADV_RSSTYPE_IPV6_EX: 
		return M_HASHTYPE_RSS_IPV6_EX;
	case E1000_RXDADV_RSSTYPE_IPV6:
		return M_HASHTYPE_RSS_IPV6;
	case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
		return M_HASHTYPE_RSS_TCP_IPV6_EX;
	default:
		return M_HASHTYPE_OPAQUE;
	}
779
}