1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
|
// SPDX-License-Identifier: GPL-2.0-or-later
/* Broadcom NetXtreme-C/E network driver.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation.
*/
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <net/netdev_queues.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/udp.h>
#include <net/tso.h>
#include <linux/bnxt/hsi.h>
#include "bnxt.h"
#include "bnxt_gso.h"
static u32 bnxt_sw_gso_lhint(unsigned int len)
{
if (len <= 512)
return TX_BD_FLAGS_LHINT_512_AND_SMALLER;
else if (len <= 1023)
return TX_BD_FLAGS_LHINT_512_TO_1023;
else if (len <= 2047)
return TX_BD_FLAGS_LHINT_1024_TO_2047;
else
return TX_BD_FLAGS_LHINT_2048_AND_LARGER;
}
netdev_tx_t bnxt_sw_udp_gso_xmit(struct bnxt *bp,
struct bnxt_tx_ring_info *txr,
struct netdev_queue *txq,
struct sk_buff *skb)
{
unsigned int last_unmap_len __maybe_unused = 0;
dma_addr_t last_unmap_addr __maybe_unused = 0;
struct bnxt_sw_tx_bd *last_unmap_buf = NULL;
unsigned int hdr_len, mss, num_segs;
struct pci_dev *pdev = bp->pdev;
unsigned int total_payload;
struct tso_dma_map map;
u32 vlan_tag_flags = 0;
int i, bds_needed;
struct tso_t tso;
u16 cfa_action;
__le32 csum;
u16 prod;
hdr_len = tso_start(skb, &tso);
mss = skb_shinfo(skb)->gso_size;
total_payload = skb->len - hdr_len;
num_segs = DIV_ROUND_UP(total_payload, mss);
if (unlikely(num_segs <= 1))
goto drop;
/* Upper bound on the number of descriptors needed.
*
* Each segment uses 1 long BD + 1 ext BD + payload BDs, which is
* at most num_segs + nr_frags (each frag boundary crossing adds at
* most 1 extra BD).
*/
bds_needed = 3 * num_segs + skb_shinfo(skb)->nr_frags + 1;
if (unlikely(bnxt_tx_avail(bp, txr) < bds_needed)) {
netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
bp->tx_wake_thresh);
return NETDEV_TX_BUSY;
}
/* BD backpressure alone cannot prevent overwriting in-flight
* headers in the inline buffer. Check slot availability directly.
*/
if (!netif_txq_maybe_stop(txq, bnxt_inline_avail(txr),
num_segs, num_segs))
return NETDEV_TX_BUSY;
if (unlikely(tso_dma_map_init(&map, &pdev->dev, skb, hdr_len)))
goto drop;
cfa_action = bnxt_xmit_get_cfa_action(skb);
if (skb_vlan_tag_present(skb)) {
vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
skb_vlan_tag_get(skb);
if (skb->vlan_proto == htons(ETH_P_8021Q))
vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
}
csum = cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
if (!tso.ipv6)
csum |= cpu_to_le32(TX_BD_FLAGS_IP_CKSUM);
prod = txr->tx_prod;
for (i = 0; i < num_segs; i++) {
unsigned int seg_payload = min_t(unsigned int, mss,
total_payload - i * mss);
u16 slot = (txr->tx_inline_prod + i) &
(BNXT_SW_USO_MAX_SEGS - 1);
struct bnxt_sw_tx_bd *tx_buf;
unsigned int mapping_len;
dma_addr_t this_hdr_dma;
unsigned int chunk_len;
unsigned int offset;
dma_addr_t dma_addr;
struct tx_bd *txbd;
struct udphdr *uh;
void *this_hdr;
int bd_count;
bool last;
u32 flags;
last = (i == num_segs - 1);
offset = slot * TSO_HEADER_SIZE;
this_hdr = txr->tx_inline_buf + offset;
this_hdr_dma = txr->tx_inline_dma + offset;
tso_build_hdr(skb, this_hdr, &tso, seg_payload, last);
/* Zero stale csum fields copied from the original skb;
* HW offload recomputes from scratch.
*/
uh = this_hdr + skb_transport_offset(skb);
uh->check = 0;
if (!tso.ipv6) {
struct iphdr *iph = this_hdr + skb_network_offset(skb);
iph->check = 0;
}
dma_sync_single_for_device(&pdev->dev, this_hdr_dma,
hdr_len, DMA_TO_DEVICE);
bd_count = tso_dma_map_count(&map, seg_payload);
tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
tx_buf->skb = skb;
tx_buf->nr_frags = bd_count;
tx_buf->is_push = 0;
tx_buf->is_ts_pkt = 0;
dma_unmap_addr_set(tx_buf, mapping, this_hdr_dma);
dma_unmap_len_set(tx_buf, len, 0);
if (last) {
tx_buf->is_sw_gso = BNXT_SW_GSO_LAST;
tso_dma_map_completion_save(&map, &tx_buf->sw_gso_cstate);
} else {
tx_buf->is_sw_gso = BNXT_SW_GSO_MID;
}
flags = (hdr_len << TX_BD_LEN_SHIFT) |
TX_BD_TYPE_LONG_TX_BD |
TX_BD_CNT(2 + bd_count);
flags |= bnxt_sw_gso_lhint(hdr_len + seg_payload);
txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
txbd->tx_bd_haddr = cpu_to_le64(this_hdr_dma);
txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod,
2 + bd_count);
prod = NEXT_TX(prod);
bnxt_init_ext_bd(bp, txr, prod, csum,
vlan_tag_flags, cfa_action);
/* set dma_unmap_len on the LAST BD touching each
* region. Since completions are in-order, the last segment
* completes after all earlier ones, so the unmap is safe.
*/
while (tso_dma_map_next(&map, &dma_addr, &chunk_len,
&mapping_len, seg_payload)) {
prod = NEXT_TX(prod);
txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
txbd->tx_bd_haddr = cpu_to_le64(dma_addr);
dma_unmap_addr_set(tx_buf, mapping, dma_addr);
dma_unmap_len_set(tx_buf, len, 0);
tx_buf->skb = NULL;
tx_buf->is_sw_gso = 0;
if (mapping_len) {
if (last_unmap_buf) {
dma_unmap_addr_set(last_unmap_buf,
mapping,
last_unmap_addr);
dma_unmap_len_set(last_unmap_buf,
len,
last_unmap_len);
}
last_unmap_addr = dma_addr;
last_unmap_len = mapping_len;
}
last_unmap_buf = tx_buf;
flags = chunk_len << TX_BD_LEN_SHIFT;
txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
txbd->tx_bd_opaque = 0;
seg_payload -= chunk_len;
}
txbd->tx_bd_len_flags_type |=
cpu_to_le32(TX_BD_FLAGS_PACKET_END);
prod = NEXT_TX(prod);
}
if (last_unmap_buf) {
dma_unmap_addr_set(last_unmap_buf, mapping, last_unmap_addr);
dma_unmap_len_set(last_unmap_buf, len, last_unmap_len);
}
txr->tx_inline_prod += num_segs;
netdev_tx_sent_queue(txq, skb->len);
WRITE_ONCE(txr->tx_prod, prod);
/* Sync BDs before doorbell */
wmb();
bnxt_db_write(bp, &txr->tx_db, prod);
if (unlikely(bnxt_tx_avail(bp, txr) <= bp->tx_wake_thresh))
netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
bp->tx_wake_thresh);
return NETDEV_TX_OK;
drop:
dev_kfree_skb_any(skb);
dev_core_stats_tx_dropped_inc(bp->dev);
return NETDEV_TX_OK;
}
|
