Commit 2e27230f authored by Gleb Smirnoff's avatar Gleb Smirnoff
Browse files

tcp_hpts: rewrite inpcb synchronization

Just trust the pcb database, that if we did in_pcbref(), no way
an inpcb can go away.  And if we never put a dropped inpcb on
our queue, and tcp_discardcb() always removes an inpcb to be
dropped from the queue, then any inpcb on the queue is valid.

Now, to solve LOR between inpcb lock and HPTS queue lock do the
following trick.  When we are about to process a certain time
slot, take the full queue of the head list into on stack list,
drop the HPTS lock and work on our queue.  This of course opens
a race when an inpcb is being removed from the on stack queue,
which was already mentioned in comments.  To address this race
introduce generation count into queues.  If we want to remove
an inpcb with generation count mismatch, we can't do that, we
can only mark it with desired new time slot or -1 for remove.

Reviewed by:		rrs
Differential revision:	https://reviews.freebsd.org/D33026
parent f971e791
......@@ -221,7 +221,7 @@ struct inpcb {
#define inp_zero_size (sizeof(struct inpcb) - \
offsetof(struct inpcb, inp_start_zero))
TAILQ_ENTRY(inpcb) inp_hpts; /* pacing out queue next lock(b) */
uint32_t inp_hpts_gencnt; /* XXXGL */
uint32_t inp_hpts_request; /* Current hpts request, zero if
* fits in the pacing window (i&b). */
/*
......@@ -254,7 +254,7 @@ struct inpcb {
uint8_t inp_numa_domain; /* numa domain */
void *inp_ppcb; /* (i) pointer to per-protocol pcb */
struct socket *inp_socket; /* (i) back pointer to socket */
uint32_t inp_hptsslot; /* Hpts wheel slot this tcb is Lock(i&b) */
int32_t inp_hptsslot; /* Hpts wheel slot this tcb is Lock(i&b) */
uint32_t inp_hpts_drop_reas; /* reason we are dropping the PCB (lock i&b) */
uint32_t inp_dropq_gencnt;
TAILQ_ENTRY(inpcb) inp_dropq; /* hpts drop queue next lock(b) */
......
This diff is collapsed.
......@@ -119,13 +119,7 @@ void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line);
#define HPTS_REMOVE_DROPQ 0x01
#define HPTS_REMOVE_OUTPUT 0x02
#define HPTS_REMOVE_ALL (HPTS_REMOVE_DROPQ | HPTS_REMOVE_OUTPUT)
static inline bool
tcp_in_hpts(struct inpcb *inp)
{
return (inp->inp_in_hpts > 0);
}
bool tcp_in_hpts(struct inpcb *);
/*
* To insert a TCB on the hpts you *must* be holding the
......@@ -151,11 +145,10 @@ tcp_in_hpts(struct inpcb *inp)
* that INP_WLOCK() or from destroying your TCB where again
* you should already have the INP_WLOCK().
*/
uint32_t __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line);
#define tcp_hpts_insert(a, b) __tcp_hpts_insert(a, b, __LINE__)
uint32_t
tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag);
uint32_t tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line,
struct hpts_diag *diag);
#define tcp_hpts_insert(inp, slot) \
tcp_hpts_insert_diag((inp), (slot), __LINE__, NULL)
void __tcp_set_hpts(struct inpcb *inp, int32_t line);
#define tcp_set_hpts(a) __tcp_set_hpts(a, __LINE__)
......@@ -164,6 +157,8 @@ void tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason);
void tcp_run_hpts(void);
uint16_t hpts_random_cpu(struct inpcb *inp);
extern int32_t tcp_min_hptsi_time;
#endif /* _KERNEL */
......
......@@ -2587,6 +2587,9 @@ tcp_close(struct tcpcb *tp)
tcp_fastopen_decrement_counter(tp->t_tfo_pending);
tp->t_tfo_pending = NULL;
}
#ifdef TCPHPTS
tcp_hpts_remove(inp, HPTS_REMOVE_ALL);
#endif
in_pcbdrop(inp);
TCPSTAT_INC(tcps_closed);
if (tp->t_state != TCPS_CLOSED)
......
......@@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$");
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_hpts.h>
#ifdef INET6
#include <netinet6/tcp6_var.h>
#endif
......@@ -343,6 +344,9 @@ tcp_twstart(struct tcpcb *tp)
* Note: soisdisconnected() call used to be made in tcp_discardcb(),
* and might not be needed here any longer.
*/
#ifdef TCPHPTS
tcp_hpts_remove(inp, HPTS_REMOVE_ALL);
#endif
tcp_discardcb(tp);
soisdisconnected(so);
tw->tw_so_options = so->so_options;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment