local_apic.c 57.5 KB
Newer Older
John Baldwin's avatar
John Baldwin committed
1
/*-
2
3
 * SPDX-License-Identifier: BSD-3-Clause
 *
John Baldwin's avatar
John Baldwin committed
4
5
 * Copyright (c) 1996, by Steve Passe
 * All rights reserved.
6
 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
John Baldwin's avatar
John Baldwin committed
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. The name of the developer may NOT be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 * 3. Neither the name of the author nor the names of any co-contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * Local APIC support on Pentium and later processors.
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

39
#include "opt_atpic.h"
40
41
#include "opt_hwpmc_hooks.h"

42
43
#include "opt_ddb.h"

John Baldwin's avatar
John Baldwin committed
44
45
#include <sys/param.h>
#include <sys/systm.h>
Mark Johnston's avatar
Mark Johnston committed
46
#include <sys/asan.h>
John Baldwin's avatar
John Baldwin committed
47
48
#include <sys/bus.h>
#include <sys/kernel.h>
49
#include <sys/lock.h>
50
#include <sys/malloc.h>
Mark Johnston's avatar
Mark Johnston committed
51
#include <sys/msan.h>
52
#include <sys/mutex.h>
John Baldwin's avatar
John Baldwin committed
53
#include <sys/pcpu.h>
54
55
#include <sys/proc.h>
#include <sys/sched.h>
56
#include <sys/smp.h>
57
#include <sys/sysctl.h>
58
#include <sys/timeet.h>
59
#include <sys/timetc.h>
John Baldwin's avatar
John Baldwin committed
60
61
62
63

#include <vm/vm.h>
#include <vm/pmap.h>

64
#include <x86/apicreg.h>
65
#include <machine/clock.h>
66
#include <machine/cpufunc.h>
John Baldwin's avatar
John Baldwin committed
67
#include <machine/cputypes.h>
68
#include <machine/fpu.h>
John Baldwin's avatar
John Baldwin committed
69
70
#include <machine/frame.h>
#include <machine/intr_machdep.h>
71
#include <x86/apicvar.h>
72
#include <x86/mca.h>
John Baldwin's avatar
John Baldwin committed
73
74
75
#include <machine/md_var.h>
#include <machine/smp.h>
#include <machine/specialreg.h>
76
#include <x86/init.h>
John Baldwin's avatar
John Baldwin committed
77

78
79
80
81
82
#ifdef DDB
#include <sys/interrupt.h>
#include <ddb/ddb.h>
#endif

83
84
85
86
87
88
89
90
#ifdef __amd64__
#define	SDT_APIC	SDT_SYSIGT
#define	GSEL_APIC	0
#else
#define	SDT_APIC	SDT_SYS386IGT
#define	GSEL_APIC	GSEL(GCODE_SEL, SEL_KPL)
#endif

91
92
static MALLOC_DEFINE(M_LAPIC, "local_apic", "Local APIC items");

93
/* Sanity checks on IDT vectors. */
94
95
96
CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
CTASSERT(APIC_LOCAL_INTS == 240);
97
98
CTASSERT(IPI_STOP < APIC_SPURIOUS_INT);

99
100
101
102
103
104
105
106
107
108
/*
 * I/O interrupts use non-negative IRQ values.  These values are used
 * to mark unused IDT entries or IDT entries reserved for a non-I/O
 * interrupt.
 */
#define	IRQ_FREE	-1
#define	IRQ_TIMER	-2
#define	IRQ_SYSCALL	-3
#define	IRQ_DTRACE_RET	-4
#define	IRQ_EVTCHN	-5
109

110
111
112
113
114
115
116
enum lat_timer_mode {
	LAT_MODE_UNDEF =	0,
	LAT_MODE_PERIODIC =	1,
	LAT_MODE_ONESHOT =	2,
	LAT_MODE_DEADLINE =	3,
};

John Baldwin's avatar
John Baldwin committed
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
/*
 * Support for local APICs.  Local APICs manage interrupts on each
 * individual processor as opposed to I/O APICs which receive interrupts
 * from I/O devices and then forward them on to the local APICs.
 *
 * Local APICs can also send interrupts to each other thus providing the
 * mechanism for IPIs.
 */

struct lvt {
	u_int lvt_edgetrigger:1;
	u_int lvt_activehi:1;
	u_int lvt_masked:1;
	u_int lvt_active:1;
	u_int lvt_mode:16;
	u_int lvt_vector:8;
};

struct lapic {
136
	struct lvt la_lvts[APIC_LVT_MAX + 1];
137
	struct lvt la_elvts[APIC_ELVT_MAX + 1];
John Baldwin's avatar
John Baldwin committed
138
139
140
141
	u_int la_id:8;
	u_int la_cluster:4;
	u_int la_cluster_id:2;
	u_int la_present:1;
142
	u_long *la_timer_count;
143
144
145
146
	uint64_t la_timer_period;
	enum lat_timer_mode la_timer_mode;
	uint32_t lvt_timer_base;
	uint32_t lvt_timer_last;
147
	/* Include IDT_SYSCALL to make indexing easier. */
148
	int la_ioint_irqs[APIC_NUM_IOINTS + 1];
149
} static *lapics;
John Baldwin's avatar
John Baldwin committed
150
151

/* Global defaults for local APIC LVT entries. */
152
static struct lvt lvts[APIC_LVT_MAX + 1] = {
John Baldwin's avatar
John Baldwin committed
153
154
	{ 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 },	/* LINT0: masked ExtINT */
	{ 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 },	/* LINT1: NMI */
155
	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT },	/* Timer */
156
	{ 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT },	/* Error */
157
	{ 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 },	/* PMC */
158
	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT },	/* Thermal */
159
	{ 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT },	/* CMCI */
John Baldwin's avatar
John Baldwin committed
160
161
};

162
163
164
165
166
167
168
169
/* Global defaults for AMD local APIC ELVT entries. */
static struct lvt elvts[APIC_ELVT_MAX + 1] = {
	{ 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
	{ 1, 1, 1, 0, APIC_LVT_DM_FIXED, APIC_CMC_INT },
	{ 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
	{ 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 },
};

John Baldwin's avatar
John Baldwin committed
170
171
172
173
174
175
176
static inthand_t *ioint_handlers[] = {
	NULL,			/* 0 - 31 */
	IDTVEC(apic_isr1),	/* 32 - 63 */
	IDTVEC(apic_isr2),	/* 64 - 95 */
	IDTVEC(apic_isr3),	/* 96 - 127 */
	IDTVEC(apic_isr4),	/* 128 - 159 */
	IDTVEC(apic_isr5),	/* 160 - 191 */
177
178
	IDTVEC(apic_isr6),	/* 192 - 223 */
	IDTVEC(apic_isr7),	/* 224 - 255 */
John Baldwin's avatar
John Baldwin committed
179
180
};

Konstantin Belousov's avatar
Konstantin Belousov committed
181
182
183
184
185
186
187
188
189
190
static inthand_t *ioint_pti_handlers[] = {
	NULL,			/* 0 - 31 */
	IDTVEC(apic_isr1_pti),	/* 32 - 63 */
	IDTVEC(apic_isr2_pti),	/* 64 - 95 */
	IDTVEC(apic_isr3_pti),	/* 96 - 127 */
	IDTVEC(apic_isr4_pti),	/* 128 - 159 */
	IDTVEC(apic_isr5_pti),	/* 160 - 191 */
	IDTVEC(apic_isr6_pti),	/* 192 - 223 */
	IDTVEC(apic_isr7_pti),	/* 224 - 255 */
};
191

192
static u_int32_t lapic_timer_divisors[] = {
193
194
195
196
	APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
	APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
};

Konstantin Belousov's avatar
Konstantin Belousov committed
197
extern inthand_t IDTVEC(rsvd_pti), IDTVEC(rsvd);
198

199
volatile char *lapic_map;
200
vm_paddr_t lapic_paddr = DEFAULT_APIC_BASE;
201
int x2apic_mode;
202
int lapic_eoi_suppression;
203
static int lapic_timer_tsc_deadline;
204
static u_long lapic_timer_divisor, count_freq;
205
static struct eventtimer lapic_et;
206
#ifdef SMP
207
static uint64_t lapic_ipi_wait_mult;
208
static int __read_mostly lapic_ds_idle_timeout = 1000000;
209
#endif
210
unsigned int max_apic_id;
John Baldwin's avatar
John Baldwin committed
211

212
213
SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
    "APIC options");
214
SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
215
216
SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
    &lapic_eoi_suppression, 0, "");
217
218
SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
    &lapic_timer_tsc_deadline, 0, "");
219
220
221
222
223
#ifdef SMP
SYSCTL_INT(_hw_apic, OID_AUTO, ds_idle_timeout, CTLFLAG_RWTUN,
    &lapic_ds_idle_timeout, 0,
    "timeout (in us) for APIC Delivery Status to become Idle (xAPIC only)");
#endif
224

225
226
static void lapic_calibrate_initcount(struct lapic *la);

227
228
229
230
231
232
233
234
235
236
237
/*
 * Use __nosanitizethread to exempt the LAPIC I/O accessors from KCSan
 * instrumentation.  Otherwise, if x2APIC is not available, use of the global
 * lapic_map will generate a KCSan false positive.  While the mapping is
 * shared among all CPUs, the physical access will always take place on the
 * local CPU's APIC, so there isn't in fact a race here.  Furthermore, the
 * KCSan warning printf can cause a panic if issued during LAPIC access,
 * due to attempted recursive use of event timer resources.
 */

static uint32_t __nosanitizethread
238
239
240
241
242
243
244
245
246
247
248
249
lapic_read32(enum LAPIC_REGISTERS reg)
{
	uint32_t res;

	if (x2apic_mode) {
		res = rdmsr32(MSR_APIC_000 + reg);
	} else {
		res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL);
	}
	return (res);
}

250
static void __nosanitizethread
251
252
253
254
255
lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val)
{

	if (x2apic_mode) {
		mfence();
256
		lfence();
257
258
259
260
261
262
		wrmsr(MSR_APIC_000 + reg, val);
	} else {
		*(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
	}
}

263
static void __nosanitizethread
264
265
266
267
268
269
270
271
272
273
lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val)
{

	if (x2apic_mode) {
		wrmsr(MSR_APIC_000 + reg, val);
	} else {
		*(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
	}
}

274
#ifdef SMP
275
276
277
278
279
280
281
282
283
284
static uint64_t
lapic_read_icr_lo(void)
{

	return (lapic_read32(LAPIC_ICR_LO));
}

static void
lapic_write_icr(uint32_t vhi, uint32_t vlo)
{
285
	register_t saveintr;
286
287
288
289
290
291
292
	uint64_t v;

	if (x2apic_mode) {
		v = ((uint64_t)vhi << 32) | vlo;
		mfence();
		wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
	} else {
293
		saveintr = intr_disable();
294
295
		lapic_write32(LAPIC_ICR_HI, vhi);
		lapic_write32(LAPIC_ICR_LO, vlo);
296
		intr_restore(saveintr);
297
298
	}
}
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318

static void
lapic_write_icr_lo(uint32_t vlo)
{

	if (x2apic_mode) {
		mfence();
		wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, vlo);
	} else {
		lapic_write32(LAPIC_ICR_LO, vlo);
	}
}

static void
lapic_write_self_ipi(uint32_t vector)
{

	KASSERT(x2apic_mode, ("SELF IPI write in xAPIC mode"));
	wrmsr(MSR_APIC_000 + LAPIC_SELF_IPI, vector);
}
319
#endif /* SMP */
320
321
322
323
324
325
326
327
328
329
330

static void
native_lapic_enable_x2apic(void)
{
	uint64_t apic_base;

	apic_base = rdmsr(MSR_APICBASE);
	apic_base |= APICBASE_X2APIC | APICBASE_ENABLED;
	wrmsr(MSR_APICBASE, apic_base);
}

331
332
333
334
335
336
337
338
339
340
static bool
native_lapic_is_x2apic(void)
{
	uint64_t apic_base;

	apic_base = rdmsr(MSR_APICBASE);
	return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) ==
	    (APICBASE_X2APIC | APICBASE_ENABLED));
}

341
static void	lapic_enable(void);
342
static void	lapic_resume(struct pic *pic, bool suspend_cancelled);
343
344
345
346
static void	lapic_timer_oneshot(struct lapic *);
static void	lapic_timer_oneshot_nointr(struct lapic *, uint32_t);
static void	lapic_timer_periodic(struct lapic *);
static void	lapic_timer_deadline(struct lapic *);
347
static void	lapic_timer_stop(struct lapic *);
348
static void	lapic_timer_set_divisor(u_int divisor);
349
static uint32_t	lvt_mode(struct lapic *la, u_int pin, uint32_t value);
350
static int	lapic_et_start(struct eventtimer *et,
351
		    sbintime_t first, sbintime_t period);
352
static int	lapic_et_stop(struct eventtimer *et);
353
354
static u_int	apic_idt_to_irq(u_int apic_id, u_int vector);
static void	lapic_set_tpr(u_int vector);
355

356
357
struct pic lapic_pic = { .pic_resume = lapic_resume };

358
359
360
/* Forward declarations for apic_ops */
static void	native_lapic_create(u_int apic_id, int boot_cpu);
static void	native_lapic_init(vm_paddr_t addr);
361
static void	native_lapic_xapic_mode(void);
362
363
364
static void	native_lapic_setup(int boot);
static void	native_lapic_dump(const char *str);
static void	native_lapic_disable(void);
365
static void	native_lapic_eoi(void);
366
367
368
369
370
371
372
373
374
375
376
static int	native_lapic_id(void);
static int	native_lapic_intr_pending(u_int vector);
static u_int	native_apic_cpuid(u_int apic_id);
static u_int	native_apic_alloc_vector(u_int apic_id, u_int irq);
static u_int	native_apic_alloc_vectors(u_int apic_id, u_int *irqs,
		    u_int count, u_int align);
static void 	native_apic_disable_vector(u_int apic_id, u_int vector);
static void 	native_apic_enable_vector(u_int apic_id, u_int vector);
static void 	native_apic_free_vector(u_int apic_id, u_int vector, u_int irq);
static void 	native_lapic_set_logical_id(u_int apic_id, u_int cluster,
		    u_int cluster_id);
377
static void	native_lapic_calibrate_timer(void);
378
379
380
381
static int 	native_lapic_enable_pmc(void);
static void 	native_lapic_disable_pmc(void);
static void 	native_lapic_reenable_pmc(void);
static void 	native_lapic_enable_cmc(void);
382
static int 	native_lapic_enable_mca_elvt(void);
383
384
385
386
387
388
389
390
static int 	native_lapic_set_lvt_mask(u_int apic_id, u_int lvt,
		    u_char masked);
static int 	native_lapic_set_lvt_mode(u_int apic_id, u_int lvt,
		    uint32_t mode);
static int 	native_lapic_set_lvt_polarity(u_int apic_id, u_int lvt,
		    enum intr_polarity pol);
static int 	native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
		    enum intr_trigger trigger);
391
392
393
394
#ifdef SMP
static void 	native_lapic_ipi_raw(register_t icrlo, u_int dest);
static void 	native_lapic_ipi_vectored(u_int vector, int dest);
static int 	native_lapic_ipi_wait(int delay);
395
#endif /* SMP */
396
397
static int	native_lapic_ipi_alloc(inthand_t *ipifunc);
static void	native_lapic_ipi_free(int vector);
398
399
400
401

struct apic_ops apic_ops = {
	.create			= native_lapic_create,
	.init			= native_lapic_init,
402
	.xapic_mode		= native_lapic_xapic_mode,
403
	.is_x2apic		= native_lapic_is_x2apic,
404
405
406
407
408
409
410
411
412
413
414
415
416
	.setup			= native_lapic_setup,
	.dump			= native_lapic_dump,
	.disable		= native_lapic_disable,
	.eoi			= native_lapic_eoi,
	.id			= native_lapic_id,
	.intr_pending		= native_lapic_intr_pending,
	.set_logical_id		= native_lapic_set_logical_id,
	.cpuid			= native_apic_cpuid,
	.alloc_vector		= native_apic_alloc_vector,
	.alloc_vectors		= native_apic_alloc_vectors,
	.enable_vector		= native_apic_enable_vector,
	.disable_vector		= native_apic_disable_vector,
	.free_vector		= native_apic_free_vector,
417
	.calibrate_timer	= native_lapic_calibrate_timer,
418
419
420
421
	.enable_pmc		= native_lapic_enable_pmc,
	.disable_pmc		= native_lapic_disable_pmc,
	.reenable_pmc		= native_lapic_reenable_pmc,
	.enable_cmc		= native_lapic_enable_cmc,
422
	.enable_mca_elvt	= native_lapic_enable_mca_elvt,
423
#ifdef SMP
424
425
426
	.ipi_raw		= native_lapic_ipi_raw,
	.ipi_vectored		= native_lapic_ipi_vectored,
	.ipi_wait		= native_lapic_ipi_wait,
427
#endif
428
429
	.ipi_alloc		= native_lapic_ipi_alloc,
	.ipi_free		= native_lapic_ipi_free,
430
431
432
433
434
435
	.set_lvt_mask		= native_lapic_set_lvt_mask,
	.set_lvt_mode		= native_lapic_set_lvt_mode,
	.set_lvt_polarity	= native_lapic_set_lvt_polarity,
	.set_lvt_triggermode	= native_lapic_set_lvt_triggermode,
};

John Baldwin's avatar
John Baldwin committed
436
static uint32_t
437
lvt_mode_impl(struct lapic *la, struct lvt *lvt, u_int pin, uint32_t value)
John Baldwin's avatar
John Baldwin committed
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
{

	value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM |
	    APIC_LVT_VECTOR);
	if (lvt->lvt_edgetrigger == 0)
		value |= APIC_LVT_TM;
	if (lvt->lvt_activehi == 0)
		value |= APIC_LVT_IIPP_INTALO;
	if (lvt->lvt_masked)
		value |= APIC_LVT_M;
	value |= lvt->lvt_mode;
	switch (lvt->lvt_mode) {
	case APIC_LVT_DM_NMI:
	case APIC_LVT_DM_SMI:
	case APIC_LVT_DM_INIT:
	case APIC_LVT_DM_EXTINT:
454
		if (!lvt->lvt_edgetrigger && bootverbose) {
John Baldwin's avatar
John Baldwin committed
455
456
			printf("lapic%u: Forcing LINT%u to edge trigger\n",
			    la->la_id, pin);
457
			value &= ~APIC_LVT_TM;
John Baldwin's avatar
John Baldwin committed
458
459
460
461
462
463
464
465
466
467
468
469
		}
		/* Use a vector of 0. */
		break;
	case APIC_LVT_DM_FIXED:
		value |= lvt->lvt_vector;
		break;
	default:
		panic("bad APIC LVT delivery mode: %#x\n", value);
	}
	return (value);
}

470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
static uint32_t
lvt_mode(struct lapic *la, u_int pin, uint32_t value)
{
	struct lvt *lvt;

	KASSERT(pin <= APIC_LVT_MAX,
	    ("%s: pin %u out of range", __func__, pin));
	if (la->la_lvts[pin].lvt_active)
		lvt = &la->la_lvts[pin];
	else
		lvt = &lvts[pin];

	return (lvt_mode_impl(la, lvt, pin, value));
}

static uint32_t
elvt_mode(struct lapic *la, u_int idx, uint32_t value)
{
	struct lvt *elvt;

	KASSERT(idx <= APIC_ELVT_MAX,
	    ("%s: idx %u out of range", __func__, idx));

	elvt = &la->la_elvts[idx];
	KASSERT(elvt->lvt_active, ("%s: ELVT%u is not active", __func__, idx));
	KASSERT(elvt->lvt_edgetrigger,
	    ("%s: ELVT%u is not edge triggered", __func__, idx));
	KASSERT(elvt->lvt_activehi,
	    ("%s: ELVT%u is not active high", __func__, idx));
	return (lvt_mode_impl(la, elvt, idx, value));
}

John Baldwin's avatar
John Baldwin committed
502
503
504
/*
 * Map the local APIC and setup necessary interrupt vectors.
 */
505
506
static void
native_lapic_init(vm_paddr_t addr)
John Baldwin's avatar
John Baldwin committed
507
{
508
509
510
#ifdef SMP
	uint64_t r, r1, r2, rx;
#endif
511
	uint32_t ver;
512
513
	int i;
	bool arat;
John Baldwin's avatar
John Baldwin committed
514

515
	/*
516
517
518
519
520
521
522
	 * Enable x2APIC mode if possible. Map the local APIC
	 * registers page.
	 *
	 * Keep the LAPIC registers page mapped uncached for x2APIC
	 * mode too, to have direct map page attribute set to
	 * uncached.  This is needed to work around CPU errata present
	 * on all Intel processors.
523
	 */
John Baldwin's avatar
John Baldwin committed
524
525
	KASSERT(trunc_page(addr) == addr,
	    ("local APIC not aligned on a page boundary"));
526
527
	lapic_paddr = addr;
	lapic_map = pmap_mapdev(addr, PAGE_SIZE);
528
529
	if (x2apic_mode) {
		native_lapic_enable_x2apic();
530
		lapic_map = NULL;
531
532
533
	}

	/* Setup the spurious interrupt handler. */
534
535
	setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
	    GSEL_APIC);
John Baldwin's avatar
John Baldwin committed
536
537

	/* Perform basic initialization of the BSP's local APIC. */
538
	lapic_enable();
John Baldwin's avatar
John Baldwin committed
539
540
541
542

	/* Set BSP's per-CPU local APIC ID. */
	PCPU_SET(apic_id, lapic_id());

543
	/* Local APIC timer interrupt. */
Konstantin Belousov's avatar
Konstantin Belousov committed
544
545
	setidt(APIC_TIMER_INT, pti ? IDTVEC(timerint_pti) : IDTVEC(timerint),
	    SDT_APIC, SEL_KPL, GSEL_APIC);
546

547
	/* Local APIC error interrupt. */
Konstantin Belousov's avatar
Konstantin Belousov committed
548
549
	setidt(APIC_ERROR_INT, pti ? IDTVEC(errorint_pti) : IDTVEC(errorint),
	    SDT_APIC, SEL_KPL, GSEL_APIC);
550
551

	/* XXX: Thermal interrupt */
552
553

	/* Local APIC CMCI. */
Konstantin Belousov's avatar
Konstantin Belousov committed
554
	setidt(APIC_CMC_INT, pti ? IDTVEC(cmcint_pti) : IDTVEC(cmcint),
Konstantin Belousov's avatar
Konstantin Belousov committed
555
	    SDT_APIC, SEL_KPL, GSEL_APIC);
556
557

	if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) {
558
559
560
		/* Set if APIC timer runs in C3. */
		arat = (cpu_power_eax & CPUTPM1_ARAT);

561
562
563
564
565
566
567
		bzero(&lapic_et, sizeof(lapic_et));
		lapic_et.et_name = "LAPIC";
		lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
		    ET_FLAGS_PERCPU;
		lapic_et.et_quality = 600;
		if (!arat) {
			lapic_et.et_flags |= ET_FLAGS_C3STOP;
568
569
570
			lapic_et.et_quality = 100;
		}
		if ((cpu_feature & CPUID_TSC) != 0 &&
571
572
573
574
575
		    (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
		    tsc_is_invariant && tsc_freq != 0) {
			lapic_timer_tsc_deadline = 1;
			TUNABLE_INT_FETCH("hw.lapic_tsc_deadline",
			    &lapic_timer_tsc_deadline);
576
		}
577

578
		lapic_et.et_frequency = 0;
579
		/* We don't know frequency yet, so trying to guess. */
Alexander Motin's avatar
Alexander Motin committed
580
581
		lapic_et.et_min_period = 0x00001000LL;
		lapic_et.et_max_period = SBT_1S;
582
583
584
585
586
		lapic_et.et_start = lapic_et_start;
		lapic_et.et_stop = lapic_et_stop;
		lapic_et.et_priv = NULL;
		et_register(&lapic_et);
	}
587
588
589
590
591
592
593

	/*
	 * Set lapic_eoi_suppression after lapic_enable(), to not
	 * enable suppression in the hardware prematurely.  Note that
	 * we by default enable suppression even when system only has
	 * one IO-APIC, since EOI is broadcasted to all APIC agents,
	 * including CPUs, otherwise.
594
595
596
	 *
	 * It seems that at least some KVM versions report
	 * EOI_SUPPRESSION bit, but auto-EOI does not work.
597
598
599
	 */
	ver = lapic_read32(LAPIC_VERSION);
	if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
600
		lapic_eoi_suppression = 1;
601
		if (vm_guest == VM_GUEST_KVM) {
602
603
604
			if (bootverbose)
				printf(
		       "KVM -- disabling lapic eoi suppression\n");
605
			lapic_eoi_suppression = 0;
606
		}
607
		TUNABLE_INT_FETCH("hw.lapic_eoi_suppression",
608
		    &lapic_eoi_suppression);
609
	}
610

611
#ifdef SMP
612
#define	LOOPS	100000
613
614
615
616
617
618
619
620
621
622
623
624
625
	/*
	 * Calibrate the busy loop waiting for IPI ack in xAPIC mode.
	 * lapic_ipi_wait_mult contains the number of iterations which
	 * approximately delay execution for 1 microsecond (the
	 * argument to native_lapic_ipi_wait() is in microseconds).
	 *
	 * We assume that TSC is present and already measured.
	 * Possible TSC frequency jumps are irrelevant to the
	 * calibration loop below, the CPU clock management code is
	 * not yet started, and we do not enter sleep states.
	 */
	KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0,
	    ("TSC not initialized"));
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
	if (!x2apic_mode) {
		r = rdtsc();
		for (rx = 0; rx < LOOPS; rx++) {
			(void)lapic_read_icr_lo();
			ia32_pause();
		}
		r = rdtsc() - r;
		r1 = tsc_freq * LOOPS;
		r2 = r * 1000000;
		lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1;
		if (bootverbose) {
			printf("LAPIC: ipi_wait() us multiplier %ju (r %ju "
			    "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult,
			    (uintmax_t)r, (uintmax_t)tsc_freq);
		}
641
642
	}
#undef LOOPS
643
#endif /* SMP */
John Baldwin's avatar
John Baldwin committed
644
645
646
647
648
}

/*
 * Create a local APIC instance.
 */
649
650
static void
native_lapic_create(u_int apic_id, int boot_cpu)
John Baldwin's avatar
John Baldwin committed
651
652
653
{
	int i;

654
	if (apic_id > max_apic_id) {
John Baldwin's avatar
John Baldwin committed
655
656
657
658
659
660
661
662
663
664
665
666
667
668
		printf("APIC: Ignoring local APIC with ID %d\n", apic_id);
		if (boot_cpu)
			panic("Can't ignore BSP");
		return;
	}
	KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u",
	    apic_id));

	/*
	 * Assume no local LVT overrides and a cluster of 0 and
	 * intra-cluster ID of 0.
	 */
	lapics[apic_id].la_present = 1;
	lapics[apic_id].la_id = apic_id;
669
	for (i = 0; i <= APIC_LVT_MAX; i++) {
John Baldwin's avatar
John Baldwin committed
670
671
672
		lapics[apic_id].la_lvts[i] = lvts[i];
		lapics[apic_id].la_lvts[i].lvt_active = 0;
	}
673
674
675
676
	for (i = 0; i <= APIC_ELVT_MAX; i++) {
		lapics[apic_id].la_elvts[i] = elvts[i];
		lapics[apic_id].la_elvts[i].lvt_active = 0;
	}
677
	for (i = 0; i <= APIC_NUM_IOINTS; i++)
678
	    lapics[apic_id].la_ioint_irqs[i] = IRQ_FREE;
679
680
681
	lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
	lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
	    IRQ_TIMER;
682
#ifdef KDTRACE_HOOKS
683
684
	lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] =
	    IRQ_DTRACE_RET;
685
#endif
686
687
688
#ifdef XENHVM
	lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN;
#endif
689

John Baldwin's avatar
John Baldwin committed
690
691
692
693
694
#ifdef SMP
	cpu_add(apic_id, boot_cpu);
#endif
}

695
696
697
698
699
static inline uint32_t
amd_read_ext_features(void)
{
	uint32_t version;

700
701
	if (cpu_vendor_id != CPU_VENDOR_AMD &&
	    cpu_vendor_id != CPU_VENDOR_HYGON)
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
		return (0);
	version = lapic_read32(LAPIC_VERSION);
	if ((version & APIC_VER_AMD_EXT_SPACE) != 0)
		return (lapic_read32(LAPIC_EXT_FEATURES));
	else
		return (0);
}

static inline uint32_t
amd_read_elvt_count(void)
{
	uint32_t extf;
	uint32_t count;

	extf = amd_read_ext_features();
	count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT;
	count = min(count, APIC_ELVT_MAX + 1);
	return (count);
}

John Baldwin's avatar
John Baldwin committed
722
723
724
/*
 * Dump contents of local APIC registers
 */
725
726
static void
native_lapic_dump(const char* str)
John Baldwin's avatar
John Baldwin committed
727
{
728
	uint32_t version;
729
	uint32_t maxlvt;
730
731
732
	uint32_t extf;
	int elvt_count;
	int i;
John Baldwin's avatar
John Baldwin committed
733

734
735
	version = lapic_read32(LAPIC_VERSION);
	maxlvt = (version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
John Baldwin's avatar
John Baldwin committed
736
	printf("cpu%d %s:\n", PCPU_GET(cpuid), str);
737
	printf("     ID: 0x%08x   VER: 0x%08x LDR: 0x%08x DFR: 0x%08x",
738
	    lapic_read32(LAPIC_ID), version,
739
740
741
742
743
744
	    lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR));
	if ((cpu_feature2 & CPUID2_X2APIC) != 0)
		printf(" x2APIC: %d", x2apic_mode);
	printf("\n  lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n",
	    lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1),
	    lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR));
745
	printf("  timer: 0x%08x therm: 0x%08x err: 0x%08x",
746
747
	    lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL),
	    lapic_read32(LAPIC_LVT_ERROR));
748
	if (maxlvt >= APIC_LVT_PMC)
749
		printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT));
750
	printf("\n");
751
	if (maxlvt >= APIC_LVT_CMCI)
752
		printf("   cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI));
753
754
	extf = amd_read_ext_features();
	if (extf != 0) {
755
		printf("   AMD ext features: 0x%08x", extf);
756
757
		elvt_count = amd_read_elvt_count();
		for (i = 0; i < elvt_count; i++)
758
			printf("%s elvt%d: 0x%08x", (i % 4) ? "" : "\n ", i,
759
			    lapic_read32(LAPIC_EXT_LVT0 + i));
760
		printf("\n");
761
	}
762
763
764
765
766
767
768
769
770
771
772
}

static void
native_lapic_xapic_mode(void)
{
	register_t saveintr;

	saveintr = intr_disable();
	if (x2apic_mode)
		native_lapic_enable_x2apic();
	intr_restore(saveintr);
John Baldwin's avatar
John Baldwin committed
773
774
}

775
776
static void
native_lapic_setup(int boot)
John Baldwin's avatar
John Baldwin committed
777
778
{
	struct lapic *la;
779
	uint32_t version;
780
	uint32_t maxlvt;
781
	register_t saveintr;
782
783
	int elvt_count;
	int i;
John Baldwin's avatar
John Baldwin committed
784

785
786
	saveintr = intr_disable();

John Baldwin's avatar
John Baldwin committed
787
788
	la = &lapics[lapic_id()];
	KASSERT(la->la_present, ("missing APIC structure"));
789
790
	version = lapic_read32(LAPIC_VERSION);
	maxlvt = (version & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
John Baldwin's avatar
John Baldwin committed
791

792
793
	/* Initialize the TPR to allow all interrupts. */
	lapic_set_tpr(0);
John Baldwin's avatar
John Baldwin committed
794
795

	/* Setup spurious vector and enable the local APIC. */
796
797
798
	lapic_enable();

	/* Program LINT[01] LVT entries. */
799
800
801
802
	lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0,
	    lapic_read32(LAPIC_LVT_LINT0)));
	lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1,
	    lapic_read32(LAPIC_LVT_LINT1)));
803

804
	/* Program the PMC LVT entry if present. */
805
806
807
808
	if (maxlvt >= APIC_LVT_PMC) {
		lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
		    LAPIC_LVT_PCINT));
	}
809

810
811
812
813
	/*
	 * Program the timer LVT.  Calibration is deferred until it is certain
	 * that we have a reliable timecounter.
	 */
814
	la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
815
	    lapic_read32(LAPIC_LVT_TIMER));
816
817
	la->lvt_timer_last = la->lvt_timer_base;
	lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
818

819
820
821
	if (boot)
		la->la_timer_mode = LAT_MODE_UNDEF;
	else if (la->la_timer_mode != LAT_MODE_UNDEF) {
822
		KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
823
		    lapic_id()));
824
825
		switch (la->la_timer_mode) {
		case LAT_MODE_PERIODIC:
826
			lapic_timer_set_divisor(lapic_timer_divisor);
827
828
829
			lapic_timer_periodic(la);
			break;
		case LAT_MODE_ONESHOT:
830
			lapic_timer_set_divisor(lapic_timer_divisor);
831
832
833
834
835
836
837
838
839
			lapic_timer_oneshot(la);
			break;
		case LAT_MODE_DEADLINE:
			lapic_timer_deadline(la);
			break;
		default:
			panic("corrupted la_timer_mode %p %d", la,
			    la->la_timer_mode);
		}
840
841
	}

842
	/* Program error LVT and clear any existing errors. */
843
844
845
	lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR,
	    lapic_read32(LAPIC_LVT_ERROR)));
	lapic_write32(LAPIC_ESR, 0);
846
847

	/* XXX: Thermal LVT */
848

849
	/* Program the CMCI LVT entry if present. */
850
851
852
853
	if (maxlvt >= APIC_LVT_CMCI) {
		lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI,
		    lapic_read32(LAPIC_LVT_CMCI)));
	}
854

855
856
857
858
859
860
861
	elvt_count = amd_read_elvt_count();
	for (i = 0; i < elvt_count; i++) {
		if (la->la_elvts[i].lvt_active)
			lapic_write32(LAPIC_EXT_LVT0 + i,
			    elvt_mode(la, i, lapic_read32(LAPIC_EXT_LVT0 + i)));
	}

862
	intr_restore(saveintr);
John Baldwin's avatar
John Baldwin committed
863
864
}

865
866
867
868
869
870
871
872
873
874
875
876
877
static void
native_lapic_intrcnt(void *dummy __unused)
{
	struct pcpu *pc;
	struct lapic *la;
	char buf[MAXCOMLEN + 1];

	/* If there are no APICs, skip this function. */
	if (lapics == NULL)
		return;

	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
		la = &lapics[pc->pc_apic_id];
878
879
		if (!la->la_present)
		    continue;
880
881
882
883
884
885
886
887

		snprintf(buf, sizeof(buf), "cpu%d:timer", pc->pc_cpuid);
		intrcnt_add(buf, &la->la_timer_count);
	}
}
SYSINIT(native_lapic_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, native_lapic_intrcnt,
    NULL);

888
889
static void
native_lapic_reenable_pmc(void)
890
891
892
893
{
#ifdef HWPMC_HOOKS
	uint32_t value;

894
	value = lapic_read32(LAPIC_LVT_PCINT);
895
	value &= ~APIC_LVT_M;
896
	lapic_write32(LAPIC_LVT_PCINT, value);
897
898
899
900
901
902
903
904
905
906
#endif
}

#ifdef HWPMC_HOOKS
static void
lapic_update_pmc(void *dummy)
{
	struct lapic *la;

	la = &lapics[lapic_id()];
907
908
	lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC,
	    lapic_read32(LAPIC_LVT_PCINT)));
909
910
911
}
#endif

912
913
914
915
916
917
static void
native_lapic_calibrate_timer(void)
{
	struct lapic *la;
	register_t intr;

918
919
920
921
922
923
#ifdef DEV_ATPIC
	/* Fail if the local APIC is not present. */
	if (!x2apic_mode && lapic_map == NULL)
		return;
#endif

924
925
926
927
928
929
930
931
932
933
934
935
936
	intr = intr_disable();
	la = &lapics[lapic_id()];

	lapic_calibrate_initcount(la);

	intr_restore(intr);

	if (lapic_timer_tsc_deadline && bootverbose) {
		printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
		    (uintmax_t)tsc_freq);
	}
}

937
938
static int
native_lapic_enable_pmc(void)
939
940
941
942
{
#ifdef HWPMC_HOOKS
	u_int32_t maxlvt;

943
#ifdef DEV_ATPIC
944
	/* Fail if the local APIC is not present. */
945
	if (!x2apic_mode && lapic_map == NULL)
946
		return (0);
947
#endif
948
949

	/* Fail if the PMC LVT is not present. */
950
	maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
951
	if (maxlvt < APIC_LVT_PMC)
952
953
		return (0);

954
	lvts[APIC_LVT_PMC].lvt_masked = 0;
955

956
957
958
959
#ifdef EARLY_AP_STARTUP
	MPASS(mp_ncpus == 1 || smp_started);
	smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
#else
960
961
962
963
964
965
966
967
968
969
970
#ifdef SMP
	/*
	 * If hwpmc was loaded at boot time then the APs may not be
	 * started yet.  In that case, don't forward the request to
	 * them as they will program the lvt when they start.
	 */
	if (smp_started)
		smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
	else
#endif
		lapic_update_pmc(NULL);
971
#endif
972
973
974
975
976
977
	return (1);
#else
	return (0);
#endif
}

978
979
static void
native_lapic_disable_pmc(void)
980
981
982
983
{
#ifdef HWPMC_HOOKS
	u_int32_t maxlvt;

984
#ifdef DEV_ATPIC
985
	/* Fail if the local APIC is not present. */
986
	if (!x2apic_mode && lapic_map == NULL)
987
		return;
988
#endif
989
990

	/* Fail if the PMC LVT is not present. */
991
	maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT;
992
	if (maxlvt < APIC_LVT_PMC)
993
994
		return;

995
	lvts[APIC_LVT_PMC].lvt_masked = 1;
996
997
998
999
1000
1001
1002
1003
1004

#ifdef SMP
	/* The APs should always be started when hwpmc is unloaded. */
	KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early"));
#endif
	smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL);
#endif
}

1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
static int
lapic_calibrate_initcount_cpuid_vm(void)
{
	u_int regs[4];
	uint64_t freq;

	/* Get value from CPUID leaf if possible. */
	if (vm_guest == VM_GUEST_NO)
		return (false);
	if (hv_high < 0x40000010)
		return (false);
	do_cpuid(0x40000010, regs);
	freq = (uint64_t)(regs[1]) * 1000;

	/* Pick timer divisor. */
	lapic_timer_divisor = 2;
	do {
		if (freq / lapic_timer_divisor < APIC_TIMER_MAX_COUNT)
			break;
		lapic_timer_divisor <<= 1;
	} while (lapic_timer_divisor <= 128);
	if (lapic_timer_divisor > 128)
		return (false);

	/* Record divided frequency. */
	count_freq = freq / lapic_timer_divisor;
	return (true);
}

1034
1035
1036
1037
1038
1039
1040
static uint64_t
cb_lapic_getcount(void)
{

	return (APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER));
}

1041
static void
1042
lapic_calibrate_initcount(struct lapic *la)
1043
{
1044
1045
	uint64_t freq;

1046
1047
1048
	if (lapic_calibrate_initcount_cpuid_vm())
		goto done;

1049
1050
1051
1052
1053
1054
	/* Calibrate the APIC timer frequency. */
	lapic_timer_set_divisor(2);
	lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT);
	fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
	freq = clockcalib(cb_lapic_getcount, "lapic");
	fpu_kern_leave(curthread, NULL);
1055

1056
	/* Pick a different divisor if necessary. */
1057
1058
	lapic_timer_divisor = 2;
	do {
1059
		if (freq * 2 / lapic_timer_divisor < APIC_TIMER_MAX_COUNT)
1060
1061
1062
1063
1064
			break;
		lapic_timer_divisor <<= 1;
	} while (lapic_timer_divisor <= 128);
	if (lapic_timer_divisor > 128)
		panic("lapic: Divisor too big");
1065
	count_freq = freq * 2 / lapic_timer_divisor;
1066
done:
1067
1068
	if (bootverbose) {
		printf("lapic: Divisor %lu, Frequency %lu Hz\n",
1069
		    lapic_timer_divisor, count_freq);
1070
1071
1072
	}
}

1073
1074
1075
1076
static void
lapic_change_mode(struct eventtimer *et, struct lapic *la,
    enum lat_timer_mode newmode)
{
1077
	if (la->la_timer_mode == newmode)
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
		return;
	switch (newmode) {
	case LAT_MODE_PERIODIC:
		lapic_timer_set_divisor(lapic_timer_divisor);
		et->et_frequency = count_freq;
		break;
	case LAT_MODE_DEADLINE:
		et->et_frequency = tsc_freq;
		break;
	case LAT_MODE_ONESHOT:
		lapic_timer_set_divisor(lapic_timer_divisor);
		et->et_frequency = count_freq;
		break;
	default:
		panic("lapic_change_mode %d", newmode);
1093
	}
1094
1095
1096
	la->la_timer_mode = newmode;
	et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
	et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
1097
1098
}

1099
static int
Alexander Motin's avatar
Alexander Motin committed
1100
lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
1101
{
1102
	struct lapic *la;
1103

1104
	la = &lapics[PCPU_GET(apic_id)];
Alexander Motin's avatar
Alexander Motin committed
1105
	if (period != 0) {
1106
		lapic_change_mode(et, la, LAT_MODE_PERIODIC);
1107
1108
1109
1110
		la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
		    32;
		lapic_timer_periodic(la);
	} else if (lapic_timer_tsc_deadline) {
1111
		lapic_change_mode(et, la, LAT_MODE_DEADLINE);
1112
1113
		la->la_timer_period = (et->et_frequency * first) >> 32;
		lapic_timer_deadline(la);
1114
	} else {
1115
		lapic_change_mode(et, la, LAT_MODE_ONESHOT);
1116
1117
1118
		la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
		    32;
		lapic_timer_oneshot(la);
1119
1120
1121
1122
1123
1124
1125
	}
	return (0);
}

static int
lapic_et_stop(struct eventtimer *et)
{
1126
	struct lapic *la;
1127

1128
	la = &lapics[PCPU_GET(apic_id)];
1129
	lapic_timer_stop(la);
1130
	la->la_timer_mode = LAT_MODE_UNDEF;
1131
	return (0);
1132
1133
}

1134
1135
static void
native_lapic_disable(void)
John Baldwin's avatar
John Baldwin committed
1136
1137
1138
1139
{
	uint32_t value;

	/* Software disable the local APIC. */
1140
	value = lapic_read32(LAPIC_SVR);
John Baldwin's avatar
John Baldwin committed
1141
	value &= ~APIC_SVR_SWEN;
1142
	lapic_write32(LAPIC_SVR, value);
John Baldwin's avatar
John Baldwin committed
1143
1144
}

1145
1146
1147
static void
lapic_enable(void)
{
1148
	uint32_t value;
1149
1150

	/* Program the spurious vector to enable the local APIC. */
1151
	value