Commit 50f081ec authored by Gleb Smirnoff's avatar Gleb Smirnoff
Browse files

tcp_hpts: provide tcp_in_hpts().

It will hide some internal HPTS knowledge from the consumers.

Reviewed by:		rrs
Differential revision:	https://reviews.freebsd.org/D33023
parent de2d4784
...@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$"); ...@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$");
* its top a check to prevent unwanted calls (from user land or the * its top a check to prevent unwanted calls (from user land or the
* arrival of incoming ack's). So it would add something like: * arrival of incoming ack's). So it would add something like:
* *
* if (inp->inp_in_hpts) * if (tcp_in_hpts(inp))
* return; * return;
* *
* to prevent output processing until the time alotted has gone by. * to prevent output processing until the time alotted has gone by.
......
...@@ -201,6 +201,13 @@ struct tcp_hpts_entry *tcp_cur_hpts(struct inpcb *inp); ...@@ -201,6 +201,13 @@ struct tcp_hpts_entry *tcp_cur_hpts(struct inpcb *inp);
#define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__) #define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__)
void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line); void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line);
static inline bool
tcp_in_hpts(struct inpcb *inp)
{
return (inp->inp_in_hpts > 0);
}
/* /*
* To insert a TCB on the hpts you *must* be holding the * To insert a TCB on the hpts you *must* be holding the
* INP_WLOCK(). The hpts insert code will then acqurire * INP_WLOCK(). The hpts insert code will then acqurire
......
...@@ -747,7 +747,7 @@ bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_ ...@@ -747,7 +747,7 @@ bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_
uint32_t prev_delay = 0; uint32_t prev_delay = 0;
inp = tp->t_inpcb; inp = tp->t_inpcb;
if (inp->inp_in_hpts) { if (tcp_in_hpts(inp)) {
/* A previous call is already set up */ /* A previous call is already set up */
return; return;
} }
...@@ -1058,7 +1058,7 @@ bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sock ...@@ -1058,7 +1058,7 @@ bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sock
*/ */
wrong_timer: wrong_timer:
if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) { if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {
if (inp->inp_in_hpts) if (tcp_in_hpts(inp))
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
bbr_timer_cancel(bbr, __LINE__, cts); bbr_timer_cancel(bbr, __LINE__, cts);
bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val, bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,
...@@ -1883,7 +1883,7 @@ bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t ct ...@@ -1883,7 +1883,7 @@ bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t ct
l->lt_epoch = bbr->r_ctl.rc_lt_epoch; l->lt_epoch = bbr->r_ctl.rc_lt_epoch;
l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain; l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain; l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;
l->inhpts = bbr->rc_inp->inp_in_hpts; l->inhpts = tcp_in_hpts(bbr->rc_inp);
l->ininput = bbr->rc_inp->inp_in_input; l->ininput = bbr->rc_inp->inp_in_input;
l->use_lt_bw = bbr->rc_lt_use_bw; l->use_lt_bw = bbr->rc_lt_use_bw;
l->pkts_out = bbr->r_ctl.rc_flight_at_input; l->pkts_out = bbr->r_ctl.rc_flight_at_input;
...@@ -3969,7 +3969,7 @@ bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_s ...@@ -3969,7 +3969,7 @@ bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_s
bbr->rc_tlp_rtx_out = 0; bbr->rc_tlp_rtx_out = 0;
bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate; bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;
tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
if (bbr->rc_inp->inp_in_hpts && if (tcp_in_hpts(bbr->rc_inp) &&
((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) { ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {
/* /*
* When we enter recovery, we need to restart * When we enter recovery, we need to restart
...@@ -5264,7 +5264,7 @@ bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts) ...@@ -5264,7 +5264,7 @@ bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts)
if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
uint8_t hpts_removed = 0; uint8_t hpts_removed = 0;
if (bbr->rc_inp->inp_in_hpts && if (tcp_in_hpts(bbr->rc_inp) &&
(bbr->rc_timer_first == 1)) { (bbr->rc_timer_first == 1)) {
/* /*
* If we are canceling timer's when we have the * If we are canceling timer's when we have the
...@@ -7972,7 +7972,7 @@ bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t li ...@@ -7972,7 +7972,7 @@ bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t li
* don't want to transfer forward the time * don't want to transfer forward the time
* for our sum's calculations. * for our sum's calculations.
*/ */
if (bbr->rc_inp->inp_in_hpts) { if (tcp_in_hpts(bbr->rc_inp)) {
tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
bbr->rc_timer_first = 0; bbr->rc_timer_first = 0;
bbr->r_ctl.rc_hpts_flags = 0; bbr->r_ctl.rc_hpts_flags = 0;
...@@ -11653,7 +11653,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, ...@@ -11653,7 +11653,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
*/ */
if ((tp->snd_max == tp->snd_una) && if ((tp->snd_max == tp->snd_una) &&
((tp->t_flags & TF_DELACK) == 0) && ((tp->t_flags & TF_DELACK) == 0) &&
(bbr->rc_inp->inp_in_hpts) && (tcp_in_hpts(bbr->rc_inp)) &&
(bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
/* /*
* keep alive not needed if we are hptsi * keep alive not needed if we are hptsi
...@@ -11661,7 +11661,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, ...@@ -11661,7 +11661,7 @@ bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
*/ */
; ;
} else { } else {
if (bbr->rc_inp->inp_in_hpts) { if (tcp_in_hpts(bbr->rc_inp)) {
tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
(TSTMP_GT(lcts, bbr->rc_pacer_started))) { (TSTMP_GT(lcts, bbr->rc_pacer_started))) {
...@@ -12011,7 +12011,7 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) ...@@ -12011,7 +12011,7 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
} }
#endif #endif
if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
inp->inp_in_hpts) { tcp_in_hpts(inp)) {
/* /*
* We are on the hpts for some timer but not hptsi output. * We are on the hpts for some timer but not hptsi output.
* Possibly remove from the hpts so we can send/recv etc. * Possibly remove from the hpts so we can send/recv etc.
...@@ -12056,9 +12056,9 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) ...@@ -12056,9 +12056,9 @@ bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
if ((bbr->r_timer_override) || if ((bbr->r_timer_override) ||
(tp->t_state < TCPS_ESTABLISHED)) { (tp->t_state < TCPS_ESTABLISHED)) {
/* Timeouts or early states are exempt */ /* Timeouts or early states are exempt */
if (inp->inp_in_hpts) if (tcp_in_hpts(inp))
tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
} else if (inp->inp_in_hpts) { } else if (tcp_in_hpts(inp)) {
if ((bbr->r_ctl.rc_last_delay_val) && if ((bbr->r_ctl.rc_last_delay_val) &&
(bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
delay_calc) { delay_calc) {
......
...@@ -2294,7 +2294,7 @@ rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t ...@@ -2294,7 +2294,7 @@ rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t
log.u_bbr.flex5 = rsm->r_start; log.u_bbr.flex5 = rsm->r_start;
log.u_bbr.flex6 = rsm->r_end; log.u_bbr.flex6 = rsm->r_end;
log.u_bbr.flex8 = mod; log.u_bbr.flex8 = mod;
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
...@@ -2329,7 +2329,7 @@ rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t slot ...@@ -2329,7 +2329,7 @@ rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t slot
log.u_bbr.pkts_out = 0; log.u_bbr.pkts_out = 0;
else else
log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt; log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
...@@ -2354,7 +2354,7 @@ rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rs ...@@ -2354,7 +2354,7 @@ rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rs
struct timeval tv; struct timeval tv;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex8 = to_num; log.u_bbr.flex8 = to_num;
log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt; log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt;
...@@ -2393,7 +2393,7 @@ rack_log_map_chg(struct tcpcb *tp, struct tcp_rack *rack, ...@@ -2393,7 +2393,7 @@ rack_log_map_chg(struct tcpcb *tp, struct tcp_rack *rack,
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.flex8 = flag; log.u_bbr.flex8 = flag;
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.cur_del_rate = (uint64_t)prev; log.u_bbr.cur_del_rate = (uint64_t)prev;
log.u_bbr.delRate = (uint64_t)rsm; log.u_bbr.delRate = (uint64_t)rsm;
...@@ -2438,7 +2438,7 @@ rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, uint32_t t, uint32_t l ...@@ -2438,7 +2438,7 @@ rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, uint32_t t, uint32_t l
union tcp_log_stackspecific log; union tcp_log_stackspecific log;
struct timeval tv; struct timeval tv;
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = t; log.u_bbr.flex1 = t;
log.u_bbr.flex2 = len; log.u_bbr.flex2 = len;
...@@ -2588,7 +2588,7 @@ rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, ...@@ -2588,7 +2588,7 @@ rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick,
struct timeval tv; struct timeval tv;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = line; log.u_bbr.flex1 = line;
log.u_bbr.flex2 = tick; log.u_bbr.flex2 = tick;
...@@ -2615,7 +2615,7 @@ rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_ ...@@ -2615,7 +2615,7 @@ rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_
union tcp_log_stackspecific log; union tcp_log_stackspecific log;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = slot; log.u_bbr.flex1 = slot;
if (rack->rack_no_prr) if (rack->rack_no_prr)
...@@ -2661,7 +2661,7 @@ rack_log_doseg_done(struct tcp_rack *rack, uint32_t cts, int32_t nxt_pkt, int32_ ...@@ -2661,7 +2661,7 @@ rack_log_doseg_done(struct tcp_rack *rack, uint32_t cts, int32_t nxt_pkt, int32_
log.u_bbr.flex7 <<= 1; log.u_bbr.flex7 <<= 1;
log.u_bbr.flex7 |= rack->r_wanted_output; /* Do we want output */ log.u_bbr.flex7 |= rack->r_wanted_output; /* Do we want output */
log.u_bbr.flex8 = rack->rc_in_persist; log.u_bbr.flex8 = rack->rc_in_persist;
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
log.u_bbr.use_lt_bw = rack->r_ent_rec_ns; log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
...@@ -2717,7 +2717,7 @@ rack_log_type_just_return(struct tcp_rack *rack, uint32_t cts, uint32_t tlen, ui ...@@ -2717,7 +2717,7 @@ rack_log_type_just_return(struct tcp_rack *rack, uint32_t cts, uint32_t tlen, ui
struct timeval tv; struct timeval tv;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = slot; log.u_bbr.flex1 = slot;
log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags; log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags;
...@@ -2750,7 +2750,7 @@ rack_log_to_cancel(struct tcp_rack *rack, int32_t hpts_removed, int line, uint32 ...@@ -2750,7 +2750,7 @@ rack_log_to_cancel(struct tcp_rack *rack, int32_t hpts_removed, int line, uint32
union tcp_log_stackspecific log; union tcp_log_stackspecific log;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = line; log.u_bbr.flex1 = line;
log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to; log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to;
...@@ -4466,7 +4466,7 @@ rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack, ...@@ -4466,7 +4466,7 @@ rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack,
rack->r_ctl.rc_app_limited_cnt, rack->r_ctl.rc_app_limited_cnt,
0, 0, 10, __LINE__, NULL, quality); 0, 0, 10, __LINE__, NULL, quality);
} }
if (rack->rc_inp->inp_in_hpts && if (tcp_in_hpts(rack->rc_inp) &&
(rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
/* /*
* Ok we can't trust the pacer in this case * Ok we can't trust the pacer in this case
...@@ -5678,7 +5678,7 @@ rack_enter_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) ...@@ -5678,7 +5678,7 @@ rack_enter_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
static void static void
rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
{ {
if (rack->rc_inp->inp_in_hpts) { if (tcp_in_hpts(rack->rc_inp)) {
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT); tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
rack->r_ctl.rc_hpts_flags = 0; rack->r_ctl.rc_hpts_flags = 0;
} }
...@@ -5811,7 +5811,7 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts, ...@@ -5811,7 +5811,7 @@ rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
(tp->t_state == TCPS_LISTEN)) { (tp->t_state == TCPS_LISTEN)) {
return; return;
} }
if (inp->inp_in_hpts) { if (tcp_in_hpts(inp)) {
/* Already on the pacer */ /* Already on the pacer */
return; return;
} }
...@@ -7244,7 +7244,7 @@ rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int lin ...@@ -7244,7 +7244,7 @@ rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int lin
} }
if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
rack->rc_tmr_stopped = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK; rack->rc_tmr_stopped = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
if (rack->rc_inp->inp_in_hpts && if (tcp_in_hpts(rack->rc_inp) &&
((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)) { ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)) {
/* /*
* Canceling timer's when we have no output being * Canceling timer's when we have no output being
...@@ -9983,7 +9983,7 @@ rack_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, int entered ...@@ -9983,7 +9983,7 @@ rack_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, int entered
(entered_recovery == 0)) { (entered_recovery == 0)) {
rack_update_prr(tp, rack, changed, th_ack); rack_update_prr(tp, rack, changed, th_ack);
if ((rsm && (rack->r_ctl.rc_prr_sndcnt >= ctf_fixed_maxseg(tp)) && if ((rsm && (rack->r_ctl.rc_prr_sndcnt >= ctf_fixed_maxseg(tp)) &&
((rack->rc_inp->inp_in_hpts == 0) && ((tcp_in_hpts(rack->rc_inp) == 0) &&
((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)))) { ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)))) {
/* /*
* If you are pacing output you don't want * If you are pacing output you don't want
...@@ -13232,7 +13232,7 @@ rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb) ...@@ -13232,7 +13232,7 @@ rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb)
* We will force the hpts to be stopped if any, and restart * We will force the hpts to be stopped if any, and restart
* with the slot set to what was in the saved slot. * with the slot set to what was in the saved slot.
*/ */
if (rack->rc_inp->inp_in_hpts) { if (tcp_in_hpts(rack->rc_inp)) {
if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
uint32_t us_cts; uint32_t us_cts;
   
...@@ -13328,7 +13328,7 @@ rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent ...@@ -13328,7 +13328,7 @@ rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent
} }
#endif #endif
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr == 0) if (rack->rack_no_prr == 0)
log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
...@@ -14320,7 +14320,7 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, ...@@ -14320,7 +14320,7 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
} }
#endif #endif
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr == 0) if (rack->rack_no_prr == 0)
log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
...@@ -14679,13 +14679,13 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, ...@@ -14679,13 +14679,13 @@ rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
/* We could not send (probably in the hpts but stopped the timer earlier)? */ /* We could not send (probably in the hpts but stopped the timer earlier)? */
if ((tp->snd_max == tp->snd_una) && if ((tp->snd_max == tp->snd_una) &&
((tp->t_flags & TF_DELACK) == 0) && ((tp->t_flags & TF_DELACK) == 0) &&
(rack->rc_inp->inp_in_hpts) && (tcp_in_hpts(rack->rc_inp)) &&
(rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
/* keep alive not needed if we are hptsi output yet */ /* keep alive not needed if we are hptsi output yet */
; ;
} else { } else {
int late = 0; int late = 0;
if (rack->rc_inp->inp_in_hpts) { if (tcp_in_hpts(rack->rc_inp)) {
if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
us_cts = tcp_get_usecs(NULL); us_cts = tcp_get_usecs(NULL);
if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) { if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) {
...@@ -15611,7 +15611,7 @@ rack_log_fsb(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_ ...@@ -15611,7 +15611,7 @@ rack_log_fsb(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_
struct timeval tv; struct timeval tv;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
log.u_bbr.flex1 = error; log.u_bbr.flex1 = error;
log.u_bbr.flex2 = flags; log.u_bbr.flex2 = flags;
...@@ -16127,7 +16127,7 @@ rack_fast_rsm_output(struct tcpcb *tp, struct tcp_rack *rack, struct rack_sendma ...@@ -16127,7 +16127,7 @@ rack_fast_rsm_output(struct tcpcb *tp, struct tcp_rack *rack, struct rack_sendma
union tcp_log_stackspecific log; union tcp_log_stackspecific log;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr) if (rack->rack_no_prr)
log.u_bbr.flex1 = 0; log.u_bbr.flex1 = 0;
...@@ -16628,7 +16628,7 @@ rack_fast_output(struct tcpcb *tp, struct tcp_rack *rack, uint64_t ts_val, ...@@ -16628,7 +16628,7 @@ rack_fast_output(struct tcpcb *tp, struct tcp_rack *rack, uint64_t ts_val,
union tcp_log_stackspecific log; union tcp_log_stackspecific log;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr) if (rack->rack_no_prr)
log.u_bbr.flex1 = 0; log.u_bbr.flex1 = 0;
...@@ -16867,7 +16867,7 @@ rack_output(struct tcpcb *tp) ...@@ -16867,7 +16867,7 @@ rack_output(struct tcpcb *tp)
cts = tcp_get_usecs(&tv); cts = tcp_get_usecs(&tv);
ms_cts = tcp_tv_to_mssectick(&tv); ms_cts = tcp_tv_to_mssectick(&tv);
if (((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && if (((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
rack->rc_inp->inp_in_hpts) { tcp_in_hpts(rack->rc_inp)) {
/* /*
* We are on the hpts for some timer but not hptsi output. * We are on the hpts for some timer but not hptsi output.
* Remove from the hpts unconditionally. * Remove from the hpts unconditionally.
...@@ -16893,7 +16893,7 @@ rack_output(struct tcpcb *tp) ...@@ -16893,7 +16893,7 @@ rack_output(struct tcpcb *tp)
} }
} }
if (rack->rc_in_persist) { if (rack->rc_in_persist) {
if (rack->rc_inp->inp_in_hpts == 0) { if (tcp_in_hpts(rack->rc_inp) == 0) {
/* Timer is not running */ /* Timer is not running */
rack_start_hpts_timer(rack, tp, cts, 0, 0, 0); rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
} }
...@@ -16907,9 +16907,9 @@ rack_output(struct tcpcb *tp) ...@@ -16907,9 +16907,9 @@ rack_output(struct tcpcb *tp)
(delayed) || (delayed) ||
(tp->t_state < TCPS_ESTABLISHED)) { (tp->t_state < TCPS_ESTABLISHED)) {
rack->rc_ack_can_sendout_data = 0; rack->rc_ack_can_sendout_data = 0;
if (rack->rc_inp->inp_in_hpts) if (tcp_in_hpts(rack->rc_inp))
tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT); tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT);
} else if (rack->rc_inp->inp_in_hpts) { } else if (tcp_in_hpts(rack->rc_inp)) {
/* /*
* On the hpts you can't pass even if ACKNOW is on, we will * On the hpts you can't pass even if ACKNOW is on, we will
* when the hpts fires. * when the hpts fires.
...@@ -18800,7 +18800,7 @@ rack_output(struct tcpcb *tp) ...@@ -18800,7 +18800,7 @@ rack_output(struct tcpcb *tp)
union tcp_log_stackspecific log; union tcp_log_stackspecific log;
   
memset(&log.u_bbr, 0, sizeof(log.u_bbr)); memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.ininput = rack->rc_inp->inp_in_input;
if (rack->rack_no_prr) if (rack->rack_no_prr)
log.u_bbr.flex1 = 0; log.u_bbr.flex1 = 0;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment