Skip to content
GitLab
Commit 50f081ec authored by Gleb Smirnoff's avatar Gleb Smirnoff
Browse files

tcp_hpts: provide tcp_in_hpts(). 

It will hide some internal HPTS knowledge from the consumers.

Reviewed by:		rrs
Differential revision:	https://reviews.freebsd.org/D33023
parent de2d4784
Hide whitespace changes
Inline Side-by-side
sys/netinet/tcp_hpts.c
View file @ 50f081ec
......@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$");
* its top a check to prevent unwanted calls (from user land or the
* arrival of incoming ack's). So it would add something like:
*
* if (inp->inp_in_hpts)
* if (tcp_in_hpts(inp))
* return;
*
* to prevent output processing until the time alotted has gone by.
......
sys/netinet/tcp_hpts.h
View file @ 50f081ec
......@@ -201,6 +201,13 @@ struct tcp_hpts_entry *tcp_cur_hpts(struct inpcb *inp);
#define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__)
void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line);
static inline bool
tcp_in_hpts(struct inpcb *inp)
{
return (inp->inp_in_hpts > 0);
}
/*
* To insert a TCB on the hpts you *must* be holding the
* INP_WLOCK(). The hpts insert code will then acqurire
......
sys/netinet/tcp_stacks/bbr.c
View file @ 50f081ec
......@@ -747,7 +747,7 @@ bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_
uint32_t prev_delay = 0;
inp = tp->t_inpcb;
if (inp->inp_in_hpts) {
if (tcp_in_hpts(inp)) {
/* A previous call is already set up */
return;
}
......@@ -1058,7 +1058,7 @@ bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sock
*/
wrong_timer:
if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {
if (inp->inp_in_hpts)
if (tcp_in_hpts(inp))
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
bbr_timer_cancel(bbr, __LINE__, cts);
bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,
......@@ -1883,7 +1883,7 @@ bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t ct
l->lt_epoch = bbr->r_ctl.rc_lt_epoch;
l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;
l->inhpts = bbr->rc_inp->inp_in_hpts;
l->inhpts = tcp_in_hpts(bbr->rc_inp);
l->ininput = bbr->rc_inp->inp_in_input;
l->use_lt_bw = bbr->rc_lt_use_bw;
l->pkts_out = bbr->r_ctl.rc_flight_at_input;
......@@ -3969,7 +3969,7 @@ bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_s
bbr->rc_tlp_rtx_out = 0;
bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;
tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
if (bbr->rc_inp->inp_in_hpts &&
if (tcp_in_hpts(bbr->rc_inp) &&
((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {
/*
* When we enter recovery, we need to restart
......@@ -5264,7 +5264,7 @@ bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts)
if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
uint8_t hpts_removed = 0;
if (bbr->rc_inp->inp_in_hpts &&
if (tcp_in_hpts(bbr->rc_inp) &&
(bbr->rc_timer_first == 1)) {
/*
* If we are canceling timer's when we have the
......@@ -7972,7 +7972,7 @@ bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t li
* don't want to transfer forward the time
* for our sum's calculations.
*/
if (bbr->rc_inp->inp_in_hpts) {
if (tcp_in_hpts(bbr->rc_inp)) {
tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
bbr->rc_timer_first = 0;
bbr->r_ctl.rc_hpts_flags = 0;
......@@ -11653,7 +11653,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
*/
if ((tp->snd_max == tp->snd_una) &&
((tp->t_flags & TF_DELACK) == 0) &&
(bbr->rc_inp->inp_in_hpts) &&
(tcp_in_hpts(bbr->rc_inp)) &&
(bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
/*
* keep alive not needed if we are hptsi
......@@ -11661,7 +11661,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
*/
;
} else {
if (bbr->rc_inp->inp_in_hpts) {
if (tcp_in_hpts(bbr->rc_inp)) {
tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
(TSTMP_GT(lcts, bbr->rc_pacer_started))) {
......@@ -12011,7 +12011,7 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
}
#endif
if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
inp->inp_in_hpts) {
tcp_in_hpts(inp)) {
/*
* We are on the hpts for some timer but not hptsi output.
* Possibly remove from the hpts so we can send/recv etc.
......@@ -12056,9 +12056,9 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
if ((bbr->r_timer_override) ||
(tp->t_state < TCPS_ESTABLISHED)) {
/* Timeouts or early states are exempt */
if (inp->inp_in_hpts)
if (tcp_in_hpts(inp))
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
} else if (inp->inp_in_hpts) {
} else if (tcp_in_hpts(inp)) {
if ((bbr->r_ctl.rc_last_delay_val) &&
(bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
delay_calc) {
......
sys/netinet/tcp_stacks/rack.c
View file @ 50f081ec
......@@ -2294,7 +2294,7 @@ rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t
log.u_bbr.flex5 = rsm->r_start;
log.u_bbr.flex6 = rsm->r_end;
log.u_bbr.flex8 = mod;
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
......@@ -2329,7 +2329,7 @@ rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t slot
log.u_bbr.pkts_out = 0;
else
log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
......@@ -2354,7 +2354,7 @@ rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rs
struct timeval tv;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex8 = to_num;
log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt;
......@@ -2393,7 +2393,7 @@ rack_log_map_chg(struct tcpcb *tp, struct tcp_rack *rack,
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.flex8 = flag;
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.cur_del_rate = (uint64_t)prev;
log.u_bbr.delRate = (uint64_t)rsm;
......@@ -2438,7 +2438,7 @@ rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, uint32_t t, uint32_t l
union tcp_log_stackspecific log;
struct timeval tv;
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = t;
log.u_bbr.flex2 = len;
......@@ -2588,7 +2588,7 @@ rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick,
struct timeval tv;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = line;
log.u_bbr.flex2 = tick;
......@@ -2615,7 +2615,7 @@ rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_
union tcp_log_stackspecific log;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = slot;
if (rack->rack_no_prr)
......@@ -2661,7 +2661,7 @@ rack_log_doseg_done(struct tcp_rack *rack, uint32_t cts, int32_t nxt_pkt, int32_
log.u_bbr.flex7 <<= 1;
log.u_bbr.flex7 |= rack->r_wanted_output; /* Do we want output */
log.u_bbr.flex8 = rack->rc_in_persist;
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
......@@ -2717,7 +2717,7 @@ rack_log_type_just_return(struct tcp_rack *rack, uint32_t cts, uint32_t tlen, ui
struct timeval tv;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = slot;
log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags;
......@@ -2750,7 +2750,7 @@ rack_log_to_cancel(struct tcp_rack *rack, int32_t hpts_removed, int line, uint32
union tcp_log_stackspecific log;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = line;
log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to;
......@@ -4466,7 +4466,7 @@ rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack,
rack->r_ctl.rc_app_limited_cnt,
0, 0, 10, __LINE__, NULL, quality);
}
if (rack->rc_inp->inp_in_hpts &&
if (tcp_in_hpts(rack->rc_inp) &&
(rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
/*
* Ok we can't trust the pacer in this case
......@@ -5678,7 +5678,7 @@ rack_enter_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
static void
rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
{
if (rack->rc_inp->inp_in_hpts) {
if (tcp_in_hpts(rack->rc_inp)) {
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
rack->r_ctl.rc_hpts_flags = 0;
}
......@@ -5811,7 +5811,7 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
(tp->t_state == TCPS_LISTEN)) {
return;
}
if (inp->inp_in_hpts) {
if (tcp_in_hpts(inp)) {
/* Already on the pacer */
return;
}
......@@ -7244,7 +7244,7 @@ rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int lin
}
if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
rack->rc_tmr_stopped = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
if (rack->rc_inp->inp_in_hpts &&
if (tcp_in_hpts(rack->rc_inp) &&
((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)) {
/*
* Canceling timer's when we have no output being
......@@ -9983,7 +9983,7 @@ rack_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, int entered
(entered_recovery == 0)) {
rack_update_prr(tp, rack, changed, th_ack);
if ((rsm && (rack->r_ctl.rc_prr_sndcnt >= ctf_fixed_maxseg(tp)) &&
((rack->rc_inp->inp_in_hpts == 0) &&
((tcp_in_hpts(rack->rc_inp) == 0) &&
((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)))) {
/*
* If you are pacing output you don't want
......@@ -13232,7 +13232,7 @@ rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb)
* We will force the hpts to be stopped if any, and restart
* with the slot set to what was in the saved slot.
*/
if (rack->rc_inp->inp_in_hpts) {
if (tcp_in_hpts(rack->rc_inp)) {
if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
uint32_t us_cts;
 
......@@ -13328,7 +13328,7 @@ rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent
}
#endif
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr == 0)
log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
......@@ -14320,7 +14320,7 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
}
#endif
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr == 0)
log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
......@@ -14679,13 +14679,13 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
/* We could not send (probably in the hpts but stopped the timer earlier)? */
if ((tp->snd_max == tp->snd_una) &&
((tp->t_flags & TF_DELACK) == 0) &&
(rack->rc_inp->inp_in_hpts) &&
(tcp_in_hpts(rack->rc_inp)) &&
(rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
/* keep alive not needed if we are hptsi output yet */
;
} else {
int late = 0;
if (rack->rc_inp->inp_in_hpts) {
if (tcp_in_hpts(rack->rc_inp)) {
if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
us_cts = tcp_get_usecs(NULL);
if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) {
......@@ -15611,7 +15611,7 @@ rack_log_fsb(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_
struct timeval tv;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = error;
log.u_bbr.flex2 = flags;
......@@ -16127,7 +16127,7 @@ rack_fast_rsm_output(struct tcpcb *tp, struct tcp_rack *rack, struct rack_sendma
union tcp_log_stackspecific log;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr)
log.u_bbr.flex1 = 0;
......@@ -16628,7 +16628,7 @@ rack_fast_output(struct tcpcb *tp, struct tcp_rack *rack, uint64_t ts_val,
union tcp_log_stackspecific log;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr)
log.u_bbr.flex1 = 0;
......@@ -16867,7 +16867,7 @@ rack_output(struct tcpcb *tp)
cts = tcp_get_usecs(&tv);
ms_cts = tcp_tv_to_mssectick(&tv);
if (((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
rack->rc_inp->inp_in_hpts) {
tcp_in_hpts(rack->rc_inp)) {
/*
* We are on the hpts for some timer but not hptsi output.
* Remove from the hpts unconditionally.
......@@ -16893,7 +16893,7 @@ rack_output(struct tcpcb *tp)
}
}
if (rack->rc_in_persist) {
if (rack->rc_inp->inp_in_hpts == 0) {
if (tcp_in_hpts(rack->rc_inp) == 0) {
/* Timer is not running */
rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
}
......@@ -16907,9 +16907,9 @@ rack_output(struct tcpcb *tp)
(delayed) ||
(tp->t_state < TCPS_ESTABLISHED)) {
rack->rc_ack_can_sendout_data = 0;
if (rack->rc_inp->inp_in_hpts)
if (tcp_in_hpts(rack->rc_inp))
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
} else if (rack->rc_inp->inp_in_hpts) {
} else if (tcp_in_hpts(rack->rc_inp)) {
/*
* On the hpts you can't pass even if ACKNOW is on, we will
* when the hpts fires.
......@@ -18800,7 +18800,7 @@ rack_output(struct tcpcb *tp)
union tcp_log_stackspecific log;
 
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts;
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr)
log.u_bbr.flex1 = 0;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment