Commit 2e27230f authored by Gleb Smirnoff's avatar Gleb Smirnoff
Browse files

tcp_hpts: rewrite inpcb synchronization

Just trust the pcb database, that if we did in_pcbref(), no way
an inpcb can go away.  And if we never put a dropped inpcb on
our queue, and tcp_discardcb() always removes an inpcb to be
dropped from the queue, then any inpcb on the queue is valid.

Now, to solve LOR between inpcb lock and HPTS queue lock do the
following trick.  When we are about to process a certain time
slot, take the full queue of the head list into on stack list,
drop the HPTS lock and work on our queue.  This of course opens
a race when an inpcb is being removed from the on stack queue,
which was already mentioned in comments.  To address this race
introduce generation count into queues.  If we want to remove
an inpcb with generation count mismatch, we can't do that, we
can only mark it with desired new time slot or -1 for remove.

Reviewed by:		rrs
Differential revision:	https://reviews.freebsd.org/D33026
parent f971e791
...@@ -221,7 +221,7 @@ struct inpcb { ...@@ -221,7 +221,7 @@ struct inpcb {
#define inp_zero_size (sizeof(struct inpcb) - \ #define inp_zero_size (sizeof(struct inpcb) - \
offsetof(struct inpcb, inp_start_zero)) offsetof(struct inpcb, inp_start_zero))
TAILQ_ENTRY(inpcb) inp_hpts; /* pacing out queue next lock(b) */ TAILQ_ENTRY(inpcb) inp_hpts; /* pacing out queue next lock(b) */
uint32_t inp_hpts_gencnt; /* XXXGL */
uint32_t inp_hpts_request; /* Current hpts request, zero if uint32_t inp_hpts_request; /* Current hpts request, zero if
* fits in the pacing window (i&b). */ * fits in the pacing window (i&b). */
/* /*
...@@ -254,7 +254,7 @@ struct inpcb { ...@@ -254,7 +254,7 @@ struct inpcb {
uint8_t inp_numa_domain; /* numa domain */ uint8_t inp_numa_domain; /* numa domain */
void *inp_ppcb; /* (i) pointer to per-protocol pcb */ void *inp_ppcb; /* (i) pointer to per-protocol pcb */
struct socket *inp_socket; /* (i) back pointer to socket */ struct socket *inp_socket; /* (i) back pointer to socket */
uint32_t inp_hptsslot; /* Hpts wheel slot this tcb is Lock(i&b) */ int32_t inp_hptsslot; /* Hpts wheel slot this tcb is Lock(i&b) */
uint32_t inp_hpts_drop_reas; /* reason we are dropping the PCB (lock i&b) */ uint32_t inp_hpts_drop_reas; /* reason we are dropping the PCB (lock i&b) */
uint32_t inp_dropq_gencnt; uint32_t inp_dropq_gencnt;
TAILQ_ENTRY(inpcb) inp_dropq; /* hpts drop queue next lock(b) */ TAILQ_ENTRY(inpcb) inp_dropq; /* hpts drop queue next lock(b) */
......
This diff is collapsed.
...@@ -119,13 +119,7 @@ void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line); ...@@ -119,13 +119,7 @@ void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line);
#define HPTS_REMOVE_DROPQ 0x01 #define HPTS_REMOVE_DROPQ 0x01
#define HPTS_REMOVE_OUTPUT 0x02 #define HPTS_REMOVE_OUTPUT 0x02
#define HPTS_REMOVE_ALL (HPTS_REMOVE_DROPQ | HPTS_REMOVE_OUTPUT) #define HPTS_REMOVE_ALL (HPTS_REMOVE_DROPQ | HPTS_REMOVE_OUTPUT)
bool tcp_in_hpts(struct inpcb *);
static inline bool
tcp_in_hpts(struct inpcb *inp)
{
return (inp->inp_in_hpts > 0);
}
/* /*
* To insert a TCB on the hpts you *must* be holding the * To insert a TCB on the hpts you *must* be holding the
...@@ -151,11 +145,10 @@ tcp_in_hpts(struct inpcb *inp) ...@@ -151,11 +145,10 @@ tcp_in_hpts(struct inpcb *inp)
* that INP_WLOCK() or from destroying your TCB where again * that INP_WLOCK() or from destroying your TCB where again
* you should already have the INP_WLOCK(). * you should already have the INP_WLOCK().
*/ */
uint32_t __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line); uint32_t tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line,
#define tcp_hpts_insert(a, b) __tcp_hpts_insert(a, b, __LINE__) struct hpts_diag *diag);
#define tcp_hpts_insert(inp, slot) \
uint32_t tcp_hpts_insert_diag((inp), (slot), __LINE__, NULL)
tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag);
void __tcp_set_hpts(struct inpcb *inp, int32_t line); void __tcp_set_hpts(struct inpcb *inp, int32_t line);
#define tcp_set_hpts(a) __tcp_set_hpts(a, __LINE__) #define tcp_set_hpts(a) __tcp_set_hpts(a, __LINE__)
...@@ -164,6 +157,8 @@ void tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason); ...@@ -164,6 +157,8 @@ void tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason);
void tcp_run_hpts(void); void tcp_run_hpts(void);
uint16_t hpts_random_cpu(struct inpcb *inp);
extern int32_t tcp_min_hptsi_time; extern int32_t tcp_min_hptsi_time;
#endif /* _KERNEL */ #endif /* _KERNEL */
......
...@@ -2587,6 +2587,9 @@ tcp_close(struct tcpcb *tp) ...@@ -2587,6 +2587,9 @@ tcp_close(struct tcpcb *tp)
tcp_fastopen_decrement_counter(tp->t_tfo_pending); tcp_fastopen_decrement_counter(tp->t_tfo_pending);
tp->t_tfo_pending = NULL; tp->t_tfo_pending = NULL;
} }
#ifdef TCPHPTS
tcp_hpts_remove(inp, HPTS_REMOVE_ALL);
#endif
in_pcbdrop(inp); in_pcbdrop(inp);
TCPSTAT_INC(tcps_closed); TCPSTAT_INC(tcps_closed);
if (tp->t_state != TCPS_CLOSED) if (tp->t_state != TCPS_CLOSED)
......
...@@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$"); ...@@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$");
#include <netinet/tcp_seq.h> #include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h> #include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h> #include <netinet/tcp_var.h>
#include <netinet/tcp_hpts.h>
#ifdef INET6 #ifdef INET6
#include <netinet6/tcp6_var.h> #include <netinet6/tcp6_var.h>
#endif #endif
...@@ -343,6 +344,9 @@ tcp_twstart(struct tcpcb *tp) ...@@ -343,6 +344,9 @@ tcp_twstart(struct tcpcb *tp)
* Note: soisdisconnected() call used to be made in tcp_discardcb(), * Note: soisdisconnected() call used to be made in tcp_discardcb(),
* and might not be needed here any longer. * and might not be needed here any longer.
*/ */
#ifdef TCPHPTS
tcp_hpts_remove(inp, HPTS_REMOVE_ALL);
#endif
tcp_discardcb(tp); tcp_discardcb(tp);
soisdisconnected(so); soisdisconnected(so);
tw->tw_so_options = so->so_options; tw->tw_so_options = so->so_options;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment