aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-01-29 19:50:58 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2018-01-29 19:50:58 -0500
commita46d3f9b1c9888a244ed1ce8da0eca98c3f378e2 (patch)
tree65eeac9c5b491e7260e4b56bcd09f1529c93cccd
parent7bcd34259466413b477d85d12a48789b5e90e0f4 (diff)
parent303c146df1c4574db3495d9acc5c440dd46c6b0f (diff)
Merge branch 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull timer updates from Thomas Gleixner: "The timer departement presents: - A rather large rework of the hrtimer infrastructure which introduces softirq based hrtimers to replace the spread of hrtimer/tasklet combos which force the actual callback execution into softirq context. The approach is completely different from the initial implementation which you cursed at 10 years ago rightfully. The softirq based timers have their own queues and there is no nasty indirection and list reshuffling in the hard interrupt anymore. This comes with conversion of some of the hrtimer/tasklet users, the rest and the final removal of that horrible interface will come towards the end of the merge window or go through the relevant maintainer trees. Note: The top commit merged the last minute bugfix for the 10 years old CPU hotplug bug as I wanted to make sure that I fatfinger the merge conflict resolution myself. - The overhaul of the STM32 clocksource/clockevents driver - A new driver for the Spreadtrum SC9860 timer - A new driver dor the Actions Semi S700 timer - The usual set of fixes and updates all over the place" * 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (53 commits) usb/gadget/NCM: Replace tasklet with softirq hrtimer ALSA/dummy: Replace tasklet with softirq hrtimer hrtimer: Implement SOFT/HARD clock base selection hrtimer: Implement support for softirq based hrtimers hrtimer: Prepare handling of hard and softirq based hrtimers hrtimer: Add clock bases and hrtimer mode for softirq context hrtimer: Use irqsave/irqrestore around __run_hrtimer() hrtimer: Factor out __hrtimer_next_event_base() hrtimer: Factor out __hrtimer_start_range_ns() hrtimer: Remove the 'base' parameter from hrtimer_reprogram() hrtimer: Make remote enqueue decision less restrictive hrtimer: Unify remote enqueue handling hrtimer: Unify hrtimer removal handling hrtimer: Make hrtimer_force_reprogramm() unconditionally available hrtimer: Make hrtimer_reprogramm() unconditional hrtimer: Make hrtimer_cpu_base.next_timer handling unconditional hrtimer: Make the remote enqueue check unconditional hrtimer: Use accesor functions instead of direct access hrtimer: Make the hrtimer_cpu_base::hres_active field unconditional, to simplify the code hrtimer: Make room in 'struct hrtimer_cpu_base' ...
-rw-r--r--Documentation/devicetree/bindings/timer/actions,owl-timer.txt1
-rw-r--r--Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt20
-rw-r--r--drivers/clocksource/Kconfig8
-rw-r--r--drivers/clocksource/Makefile1
-rw-r--r--drivers/clocksource/owl-timer.c5
-rw-r--r--drivers/clocksource/tcb_clksrc.c2
-rw-r--r--drivers/clocksource/timer-of.c84
-rw-r--r--drivers/clocksource/timer-of.h1
-rw-r--r--drivers/clocksource/timer-sprd.c159
-rw-r--r--drivers/clocksource/timer-stm32.c358
-rw-r--r--drivers/usb/gadget/function/f_ncm.c30
-rw-r--r--include/linux/hrtimer.h113
-rw-r--r--include/linux/posix-timers.h25
-rw-r--r--include/trace/events/timer.h37
-rw-r--r--kernel/time/hrtimer.c657
-rw-r--r--kernel/time/posix-clock.c2
-rw-r--r--kernel/time/posix-cpu-timers.c7
-rw-r--r--kernel/time/tick-internal.h13
-rw-r--r--kernel/time/tick-sched.c2
-rw-r--r--kernel/time/timer.c90
-rw-r--r--sound/drivers/dummy.c27
-rw-r--r--tools/testing/selftests/ptp/testptp.c4
22 files changed, 1123 insertions, 523 deletions
diff --git a/Documentation/devicetree/bindings/timer/actions,owl-timer.txt b/Documentation/devicetree/bindings/timer/actions,owl-timer.txt
index e3c28da80cb2..977054f87563 100644
--- a/Documentation/devicetree/bindings/timer/actions,owl-timer.txt
+++ b/Documentation/devicetree/bindings/timer/actions,owl-timer.txt
@@ -2,6 +2,7 @@ Actions Semi Owl Timer
2 2
3Required properties: 3Required properties:
4- compatible : "actions,s500-timer" for S500 4- compatible : "actions,s500-timer" for S500
5 "actions,s700-timer" for S700
5 "actions,s900-timer" for S900 6 "actions,s900-timer" for S900
6- reg : Offset and length of the register set for the device. 7- reg : Offset and length of the register set for the device.
7- interrupts : Should contain the interrupts. 8- interrupts : Should contain the interrupts.
diff --git a/Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt b/Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt
new file mode 100644
index 000000000000..6d97e7d0f6e8
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt
@@ -0,0 +1,20 @@
1Spreadtrum timers
2
3The Spreadtrum SC9860 platform provides 3 general-purpose timers.
4These timers can support 32bit or 64bit counter, as well as supporting
5period mode or one-shot mode, and they are can be wakeup source
6during deep sleep.
7
8Required properties:
9- compatible: should be "sprd,sc9860-timer" for SC9860 platform.
10- reg: The register address of the timer device.
11- interrupts: Should contain the interrupt for the timer device.
12- clocks: The phandle to the source clock (usually a 32.768 KHz fixed clock).
13
14Example:
15 timer@40050000 {
16 compatible = "sprd,sc9860-timer";
17 reg = <0 0x40050000 0 0x20>;
18 interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
19 clocks = <&ext_32k>;
20 };
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index c729a88007d0..b3b4ed9b6874 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -269,6 +269,7 @@ config CLKSRC_STM32
269 bool "Clocksource for STM32 SoCs" if !ARCH_STM32 269 bool "Clocksource for STM32 SoCs" if !ARCH_STM32
270 depends on OF && ARM && (ARCH_STM32 || COMPILE_TEST) 270 depends on OF && ARM && (ARCH_STM32 || COMPILE_TEST)
271 select CLKSRC_MMIO 271 select CLKSRC_MMIO
272 select TIMER_OF
272 273
273config CLKSRC_MPS2 274config CLKSRC_MPS2
274 bool "Clocksource for MPS2 SoCs" if COMPILE_TEST 275 bool "Clocksource for MPS2 SoCs" if COMPILE_TEST
@@ -441,6 +442,13 @@ config MTK_TIMER
441 help 442 help
442 Support for Mediatek timer driver. 443 Support for Mediatek timer driver.
443 444
445config SPRD_TIMER
446 bool "Spreadtrum timer driver" if COMPILE_TEST
447 depends on HAS_IOMEM
448 select TIMER_OF
449 help
450 Enables support for the Spreadtrum timer driver.
451
444config SYS_SUPPORTS_SH_MTU2 452config SYS_SUPPORTS_SH_MTU2
445 bool 453 bool
446 454
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index 72711f1491e3..d6dec4489d66 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -54,6 +54,7 @@ obj-$(CONFIG_CLKSRC_TI_32K) += timer-ti-32k.o
54obj-$(CONFIG_CLKSRC_NPS) += timer-nps.o 54obj-$(CONFIG_CLKSRC_NPS) += timer-nps.o
55obj-$(CONFIG_OXNAS_RPS_TIMER) += timer-oxnas-rps.o 55obj-$(CONFIG_OXNAS_RPS_TIMER) += timer-oxnas-rps.o
56obj-$(CONFIG_OWL_TIMER) += owl-timer.o 56obj-$(CONFIG_OWL_TIMER) += owl-timer.o
57obj-$(CONFIG_SPRD_TIMER) += timer-sprd.o
57 58
58obj-$(CONFIG_ARC_TIMERS) += arc_timer.o 59obj-$(CONFIG_ARC_TIMERS) += arc_timer.o
59obj-$(CONFIG_ARM_ARCH_TIMER) += arm_arch_timer.o 60obj-$(CONFIG_ARM_ARCH_TIMER) += arm_arch_timer.o
diff --git a/drivers/clocksource/owl-timer.c b/drivers/clocksource/owl-timer.c
index c68630565079..ea00a5e8f95d 100644
--- a/drivers/clocksource/owl-timer.c
+++ b/drivers/clocksource/owl-timer.c
@@ -168,5 +168,6 @@ static int __init owl_timer_init(struct device_node *node)
168 168
169 return 0; 169 return 0;
170} 170}
171CLOCKSOURCE_OF_DECLARE(owl_s500, "actions,s500-timer", owl_timer_init); 171TIMER_OF_DECLARE(owl_s500, "actions,s500-timer", owl_timer_init);
172CLOCKSOURCE_OF_DECLARE(owl_s900, "actions,s900-timer", owl_timer_init); 172TIMER_OF_DECLARE(owl_s700, "actions,s700-timer", owl_timer_init);
173TIMER_OF_DECLARE(owl_s900, "actions,s900-timer", owl_timer_init);
diff --git a/drivers/clocksource/tcb_clksrc.c b/drivers/clocksource/tcb_clksrc.c
index 9de47d4d2d9e..43f4d5c4d6fa 100644
--- a/drivers/clocksource/tcb_clksrc.c
+++ b/drivers/clocksource/tcb_clksrc.c
@@ -384,7 +384,7 @@ static int __init tcb_clksrc_init(void)
384 384
385 printk(bootinfo, clksrc.name, CONFIG_ATMEL_TCB_CLKSRC_BLOCK, 385 printk(bootinfo, clksrc.name, CONFIG_ATMEL_TCB_CLKSRC_BLOCK,
386 divided_rate / 1000000, 386 divided_rate / 1000000,
387 ((divided_rate + 500000) % 1000000) / 1000); 387 ((divided_rate % 1000000) + 500) / 1000);
388 388
389 if (tc->tcb_config && tc->tcb_config->counter_width == 32) { 389 if (tc->tcb_config && tc->tcb_config->counter_width == 32) {
390 /* use apropriate function to read 32 bit counter */ 390 /* use apropriate function to read 32 bit counter */
diff --git a/drivers/clocksource/timer-of.c b/drivers/clocksource/timer-of.c
index a31990408153..06ed88a2a8a0 100644
--- a/drivers/clocksource/timer-of.c
+++ b/drivers/clocksource/timer-of.c
@@ -24,7 +24,13 @@
24 24
25#include "timer-of.h" 25#include "timer-of.h"
26 26
27static __init void timer_irq_exit(struct of_timer_irq *of_irq) 27/**
28 * timer_of_irq_exit - Release the interrupt
29 * @of_irq: an of_timer_irq structure pointer
30 *
31 * Free the irq resource
32 */
33static __init void timer_of_irq_exit(struct of_timer_irq *of_irq)
28{ 34{
29 struct timer_of *to = container_of(of_irq, struct timer_of, of_irq); 35 struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
30 36
@@ -34,8 +40,24 @@ static __init void timer_irq_exit(struct of_timer_irq *of_irq)
34 free_irq(of_irq->irq, clkevt); 40 free_irq(of_irq->irq, clkevt);
35} 41}
36 42
37static __init int timer_irq_init(struct device_node *np, 43/**
38 struct of_timer_irq *of_irq) 44 * timer_of_irq_init - Request the interrupt
45 * @np: a device tree node pointer
46 * @of_irq: an of_timer_irq structure pointer
47 *
48 * Get the interrupt number from the DT from its definition and
49 * request it. The interrupt is gotten by falling back the following way:
50 *
51 * - Get interrupt number by name
52 * - Get interrupt number by index
53 *
54 * When the interrupt is per CPU, 'request_percpu_irq()' is called,
55 * otherwise 'request_irq()' is used.
56 *
57 * Returns 0 on success, < 0 otherwise
58 */
59static __init int timer_of_irq_init(struct device_node *np,
60 struct of_timer_irq *of_irq)
39{ 61{
40 int ret; 62 int ret;
41 struct timer_of *to = container_of(of_irq, struct timer_of, of_irq); 63 struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
@@ -72,15 +94,30 @@ static __init int timer_irq_init(struct device_node *np,
72 return 0; 94 return 0;
73} 95}
74 96
75static __init void timer_clk_exit(struct of_timer_clk *of_clk) 97/**
98 * timer_of_clk_exit - Release the clock resources
99 * @of_clk: a of_timer_clk structure pointer
100 *
101 * Disables and releases the refcount on the clk
102 */
103static __init void timer_of_clk_exit(struct of_timer_clk *of_clk)
76{ 104{
77 of_clk->rate = 0; 105 of_clk->rate = 0;
78 clk_disable_unprepare(of_clk->clk); 106 clk_disable_unprepare(of_clk->clk);
79 clk_put(of_clk->clk); 107 clk_put(of_clk->clk);
80} 108}
81 109
82static __init int timer_clk_init(struct device_node *np, 110/**
83 struct of_timer_clk *of_clk) 111 * timer_of_clk_init - Initialize the clock resources
112 * @np: a device tree node pointer
113 * @of_clk: a of_timer_clk structure pointer
114 *
115 * Get the clock by name or by index, enable it and get the rate
116 *
117 * Returns 0 on success, < 0 otherwise
118 */
119static __init int timer_of_clk_init(struct device_node *np,
120 struct of_timer_clk *of_clk)
84{ 121{
85 int ret; 122 int ret;
86 123
@@ -116,19 +153,19 @@ out_clk_put:
116 goto out; 153 goto out;
117} 154}
118 155
119static __init void timer_base_exit(struct of_timer_base *of_base) 156static __init void timer_of_base_exit(struct of_timer_base *of_base)
120{ 157{
121 iounmap(of_base->base); 158 iounmap(of_base->base);
122} 159}
123 160
124static __init int timer_base_init(struct device_node *np, 161static __init int timer_of_base_init(struct device_node *np,
125 struct of_timer_base *of_base) 162 struct of_timer_base *of_base)
126{ 163{
127 const char *name = of_base->name ? of_base->name : np->full_name; 164 of_base->base = of_base->name ?
128 165 of_io_request_and_map(np, of_base->index, of_base->name) :
129 of_base->base = of_io_request_and_map(np, of_base->index, name); 166 of_iomap(np, of_base->index);
130 if (IS_ERR(of_base->base)) { 167 if (IS_ERR(of_base->base)) {
131 pr_err("Failed to iomap (%s)\n", name); 168 pr_err("Failed to iomap (%s)\n", of_base->name);
132 return PTR_ERR(of_base->base); 169 return PTR_ERR(of_base->base);
133 } 170 }
134 171
@@ -141,21 +178,21 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
141 int flags = 0; 178 int flags = 0;
142 179
143 if (to->flags & TIMER_OF_BASE) { 180 if (to->flags & TIMER_OF_BASE) {
144 ret = timer_base_init(np, &to->of_base); 181 ret = timer_of_base_init(np, &to->of_base);
145 if (ret) 182 if (ret)
146 goto out_fail; 183 goto out_fail;
147 flags |= TIMER_OF_BASE; 184 flags |= TIMER_OF_BASE;
148 } 185 }
149 186
150 if (to->flags & TIMER_OF_CLOCK) { 187 if (to->flags & TIMER_OF_CLOCK) {
151 ret = timer_clk_init(np, &to->of_clk); 188 ret = timer_of_clk_init(np, &to->of_clk);
152 if (ret) 189 if (ret)
153 goto out_fail; 190 goto out_fail;
154 flags |= TIMER_OF_CLOCK; 191 flags |= TIMER_OF_CLOCK;
155 } 192 }
156 193
157 if (to->flags & TIMER_OF_IRQ) { 194 if (to->flags & TIMER_OF_IRQ) {
158 ret = timer_irq_init(np, &to->of_irq); 195 ret = timer_of_irq_init(np, &to->of_irq);
159 if (ret) 196 if (ret)
160 goto out_fail; 197 goto out_fail;
161 flags |= TIMER_OF_IRQ; 198 flags |= TIMER_OF_IRQ;
@@ -163,17 +200,20 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
163 200
164 if (!to->clkevt.name) 201 if (!to->clkevt.name)
165 to->clkevt.name = np->name; 202 to->clkevt.name = np->name;
203
204 to->np = np;
205
166 return ret; 206 return ret;
167 207
168out_fail: 208out_fail:
169 if (flags & TIMER_OF_IRQ) 209 if (flags & TIMER_OF_IRQ)
170 timer_irq_exit(&to->of_irq); 210 timer_of_irq_exit(&to->of_irq);
171 211
172 if (flags & TIMER_OF_CLOCK) 212 if (flags & TIMER_OF_CLOCK)
173 timer_clk_exit(&to->of_clk); 213 timer_of_clk_exit(&to->of_clk);
174 214
175 if (flags & TIMER_OF_BASE) 215 if (flags & TIMER_OF_BASE)
176 timer_base_exit(&to->of_base); 216 timer_of_base_exit(&to->of_base);
177 return ret; 217 return ret;
178} 218}
179 219
@@ -187,11 +227,11 @@ out_fail:
187void __init timer_of_cleanup(struct timer_of *to) 227void __init timer_of_cleanup(struct timer_of *to)
188{ 228{
189 if (to->flags & TIMER_OF_IRQ) 229 if (to->flags & TIMER_OF_IRQ)
190 timer_irq_exit(&to->of_irq); 230 timer_of_irq_exit(&to->of_irq);
191 231
192 if (to->flags & TIMER_OF_CLOCK) 232 if (to->flags & TIMER_OF_CLOCK)
193 timer_clk_exit(&to->of_clk); 233 timer_of_clk_exit(&to->of_clk);
194 234
195 if (to->flags & TIMER_OF_BASE) 235 if (to->flags & TIMER_OF_BASE)
196 timer_base_exit(&to->of_base); 236 timer_of_base_exit(&to->of_base);
197} 237}
diff --git a/drivers/clocksource/timer-of.h b/drivers/clocksource/timer-of.h
index 3f708f1be43d..a5478f3e8589 100644
--- a/drivers/clocksource/timer-of.h
+++ b/drivers/clocksource/timer-of.h
@@ -33,6 +33,7 @@ struct of_timer_clk {
33 33
34struct timer_of { 34struct timer_of {
35 unsigned int flags; 35 unsigned int flags;
36 struct device_node *np;
36 struct clock_event_device clkevt; 37 struct clock_event_device clkevt;
37 struct of_timer_base of_base; 38 struct of_timer_base of_base;
38 struct of_timer_irq of_irq; 39 struct of_timer_irq of_irq;
diff --git a/drivers/clocksource/timer-sprd.c b/drivers/clocksource/timer-sprd.c
new file mode 100644
index 000000000000..ef9ebeafb3ed
--- /dev/null
+++ b/drivers/clocksource/timer-sprd.c
@@ -0,0 +1,159 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2017 Spreadtrum Communications Inc.
4 */
5
6#include <linux/init.h>
7#include <linux/interrupt.h>
8
9#include "timer-of.h"
10
11#define TIMER_NAME "sprd_timer"
12
13#define TIMER_LOAD_LO 0x0
14#define TIMER_LOAD_HI 0x4
15#define TIMER_VALUE_LO 0x8
16#define TIMER_VALUE_HI 0xc
17
18#define TIMER_CTL 0x10
19#define TIMER_CTL_PERIOD_MODE BIT(0)
20#define TIMER_CTL_ENABLE BIT(1)
21#define TIMER_CTL_64BIT_WIDTH BIT(16)
22
23#define TIMER_INT 0x14
24#define TIMER_INT_EN BIT(0)
25#define TIMER_INT_RAW_STS BIT(1)
26#define TIMER_INT_MASK_STS BIT(2)
27#define TIMER_INT_CLR BIT(3)
28
29#define TIMER_VALUE_SHDW_LO 0x18
30#define TIMER_VALUE_SHDW_HI 0x1c
31
32#define TIMER_VALUE_LO_MASK GENMASK(31, 0)
33
34static void sprd_timer_enable(void __iomem *base, u32 flag)
35{
36 u32 val = readl_relaxed(base + TIMER_CTL);
37
38 val |= TIMER_CTL_ENABLE;
39 if (flag & TIMER_CTL_64BIT_WIDTH)
40 val |= TIMER_CTL_64BIT_WIDTH;
41 else
42 val &= ~TIMER_CTL_64BIT_WIDTH;
43
44 if (flag & TIMER_CTL_PERIOD_MODE)
45 val |= TIMER_CTL_PERIOD_MODE;
46 else
47 val &= ~TIMER_CTL_PERIOD_MODE;
48
49 writel_relaxed(val, base + TIMER_CTL);
50}
51
52static void sprd_timer_disable(void __iomem *base)
53{
54 u32 val = readl_relaxed(base + TIMER_CTL);
55
56 val &= ~TIMER_CTL_ENABLE;
57 writel_relaxed(val, base + TIMER_CTL);
58}
59
60static void sprd_timer_update_counter(void __iomem *base, unsigned long cycles)
61{
62 writel_relaxed(cycles & TIMER_VALUE_LO_MASK, base + TIMER_LOAD_LO);
63 writel_relaxed(0, base + TIMER_LOAD_HI);
64}
65
66static void sprd_timer_enable_interrupt(void __iomem *base)
67{
68 writel_relaxed(TIMER_INT_EN, base + TIMER_INT);
69}
70
71static void sprd_timer_clear_interrupt(void __iomem *base)
72{
73 u32 val = readl_relaxed(base + TIMER_INT);
74
75 val |= TIMER_INT_CLR;
76 writel_relaxed(val, base + TIMER_INT);
77}
78
79static int sprd_timer_set_next_event(unsigned long cycles,
80 struct clock_event_device *ce)
81{
82 struct timer_of *to = to_timer_of(ce);
83
84 sprd_timer_disable(timer_of_base(to));
85 sprd_timer_update_counter(timer_of_base(to), cycles);
86 sprd_timer_enable(timer_of_base(to), 0);
87
88 return 0;
89}
90
91static int sprd_timer_set_periodic(struct clock_event_device *ce)
92{
93 struct timer_of *to = to_timer_of(ce);
94
95 sprd_timer_disable(timer_of_base(to));
96 sprd_timer_update_counter(timer_of_base(to), timer_of_period(to));
97 sprd_timer_enable(timer_of_base(to), TIMER_CTL_PERIOD_MODE);
98
99 return 0;
100}
101
102static int sprd_timer_shutdown(struct clock_event_device *ce)
103{
104 struct timer_of *to = to_timer_of(ce);
105
106 sprd_timer_disable(timer_of_base(to));
107 return 0;
108}
109
110static irqreturn_t sprd_timer_interrupt(int irq, void *dev_id)
111{
112 struct clock_event_device *ce = (struct clock_event_device *)dev_id;
113 struct timer_of *to = to_timer_of(ce);
114
115 sprd_timer_clear_interrupt(timer_of_base(to));
116
117 if (clockevent_state_oneshot(ce))
118 sprd_timer_disable(timer_of_base(to));
119
120 ce->event_handler(ce);
121 return IRQ_HANDLED;
122}
123
124static struct timer_of to = {
125 .flags = TIMER_OF_IRQ | TIMER_OF_BASE | TIMER_OF_CLOCK,
126
127 .clkevt = {
128 .name = TIMER_NAME,
129 .rating = 300,
130 .features = CLOCK_EVT_FEAT_DYNIRQ | CLOCK_EVT_FEAT_PERIODIC |
131 CLOCK_EVT_FEAT_ONESHOT,
132 .set_state_shutdown = sprd_timer_shutdown,
133 .set_state_periodic = sprd_timer_set_periodic,
134 .set_next_event = sprd_timer_set_next_event,
135 .cpumask = cpu_possible_mask,
136 },
137
138 .of_irq = {
139 .handler = sprd_timer_interrupt,
140 .flags = IRQF_TIMER | IRQF_IRQPOLL,
141 },
142};
143
144static int __init sprd_timer_init(struct device_node *np)
145{
146 int ret;
147
148 ret = timer_of_init(np, &to);
149 if (ret)
150 return ret;
151
152 sprd_timer_enable_interrupt(timer_of_base(&to));
153 clockevents_config_and_register(&to.clkevt, timer_of_rate(&to),
154 1, UINT_MAX);
155
156 return 0;
157}
158
159TIMER_OF_DECLARE(sc9860_timer, "sprd,sc9860-timer", sprd_timer_init);
diff --git a/drivers/clocksource/timer-stm32.c b/drivers/clocksource/timer-stm32.c
index 8f2423789ba9..e5cdc3af684c 100644
--- a/drivers/clocksource/timer-stm32.c
+++ b/drivers/clocksource/timer-stm32.c
@@ -9,6 +9,7 @@
9#include <linux/kernel.h> 9#include <linux/kernel.h>
10#include <linux/clocksource.h> 10#include <linux/clocksource.h>
11#include <linux/clockchips.h> 11#include <linux/clockchips.h>
12#include <linux/delay.h>
12#include <linux/irq.h> 13#include <linux/irq.h>
13#include <linux/interrupt.h> 14#include <linux/interrupt.h>
14#include <linux/of.h> 15#include <linux/of.h>
@@ -16,175 +17,318 @@
16#include <linux/of_irq.h> 17#include <linux/of_irq.h>
17#include <linux/clk.h> 18#include <linux/clk.h>
18#include <linux/reset.h> 19#include <linux/reset.h>
20#include <linux/sched_clock.h>
21#include <linux/slab.h>
22
23#include "timer-of.h"
19 24
20#define TIM_CR1 0x00 25#define TIM_CR1 0x00
21#define TIM_DIER 0x0c 26#define TIM_DIER 0x0c
22#define TIM_SR 0x10 27#define TIM_SR 0x10
23#define TIM_EGR 0x14 28#define TIM_EGR 0x14
29#define TIM_CNT 0x24
24#define TIM_PSC 0x28 30#define TIM_PSC 0x28
25#define TIM_ARR 0x2c 31#define TIM_ARR 0x2c
32#define TIM_CCR1 0x34
26 33
27#define TIM_CR1_CEN BIT(0) 34#define TIM_CR1_CEN BIT(0)
35#define TIM_CR1_UDIS BIT(1)
28#define TIM_CR1_OPM BIT(3) 36#define TIM_CR1_OPM BIT(3)
29#define TIM_CR1_ARPE BIT(7) 37#define TIM_CR1_ARPE BIT(7)
30 38
31#define TIM_DIER_UIE BIT(0) 39#define TIM_DIER_UIE BIT(0)
40#define TIM_DIER_CC1IE BIT(1)
32 41
33#define TIM_SR_UIF BIT(0) 42#define TIM_SR_UIF BIT(0)
34 43
35#define TIM_EGR_UG BIT(0) 44#define TIM_EGR_UG BIT(0)
36 45
37struct stm32_clock_event_ddata { 46#define TIM_PSC_MAX USHRT_MAX
38 struct clock_event_device evtdev; 47#define TIM_PSC_CLKRATE 10000
39 unsigned periodic_top; 48
40 void __iomem *base; 49struct stm32_timer_private {
50 int bits;
41}; 51};
42 52
43static int stm32_clock_event_shutdown(struct clock_event_device *evtdev) 53/**
54 * stm32_timer_of_bits_set - set accessor helper
55 * @to: a timer_of structure pointer
56 * @bits: the number of bits (16 or 32)
57 *
58 * Accessor helper to set the number of bits in the timer-of private
59 * structure.
60 *
61 */
62static void stm32_timer_of_bits_set(struct timer_of *to, int bits)
44{ 63{
45 struct stm32_clock_event_ddata *data = 64 struct stm32_timer_private *pd = to->private_data;
46 container_of(evtdev, struct stm32_clock_event_ddata, evtdev);
47 void *base = data->base;
48 65
49 writel_relaxed(0, base + TIM_CR1); 66 pd->bits = bits;
50 return 0; 67}
68
69/**
70 * stm32_timer_of_bits_get - get accessor helper
71 * @to: a timer_of structure pointer
72 *
73 * Accessor helper to get the number of bits in the timer-of private
74 * structure.
75 *
76 * Returns an integer corresponding to the number of bits.
77 */
78static int stm32_timer_of_bits_get(struct timer_of *to)
79{
80 struct stm32_timer_private *pd = to->private_data;
81
82 return pd->bits;
83}
84
85static void __iomem *stm32_timer_cnt __read_mostly;
86
87static u64 notrace stm32_read_sched_clock(void)
88{
89 return readl_relaxed(stm32_timer_cnt);
90}
91
92static struct delay_timer stm32_timer_delay;
93
94static unsigned long stm32_read_delay(void)
95{
96 return readl_relaxed(stm32_timer_cnt);
51} 97}
52 98
53static int stm32_clock_event_set_periodic(struct clock_event_device *evtdev) 99static void stm32_clock_event_disable(struct timer_of *to)
54{ 100{
55 struct stm32_clock_event_ddata *data = 101 writel_relaxed(0, timer_of_base(to) + TIM_DIER);
56 container_of(evtdev, struct stm32_clock_event_ddata, evtdev); 102}
57 void *base = data->base; 103
104/**
105 * stm32_timer_start - Start the counter without event
106 * @to: a timer_of structure pointer
107 *
108 * Start the timer in order to have the counter reset and start
109 * incrementing but disable interrupt event when there is a counter
110 * overflow. By default, the counter direction is used as upcounter.
111 */
112static void stm32_timer_start(struct timer_of *to)
113{
114 writel_relaxed(TIM_CR1_UDIS | TIM_CR1_CEN, timer_of_base(to) + TIM_CR1);
115}
116
117static int stm32_clock_event_shutdown(struct clock_event_device *clkevt)
118{
119 struct timer_of *to = to_timer_of(clkevt);
120
121 stm32_clock_event_disable(to);
58 122
59 writel_relaxed(data->periodic_top, base + TIM_ARR);
60 writel_relaxed(TIM_CR1_ARPE | TIM_CR1_CEN, base + TIM_CR1);
61 return 0; 123 return 0;
62} 124}
63 125
64static int stm32_clock_event_set_next_event(unsigned long evt, 126static int stm32_clock_event_set_next_event(unsigned long evt,
65 struct clock_event_device *evtdev) 127 struct clock_event_device *clkevt)
66{ 128{
67 struct stm32_clock_event_ddata *data = 129 struct timer_of *to = to_timer_of(clkevt);
68 container_of(evtdev, struct stm32_clock_event_ddata, evtdev); 130 unsigned long now, next;
131
132 next = readl_relaxed(timer_of_base(to) + TIM_CNT) + evt;
133 writel_relaxed(next, timer_of_base(to) + TIM_CCR1);
134 now = readl_relaxed(timer_of_base(to) + TIM_CNT);
135
136 if ((next - now) > evt)
137 return -ETIME;
69 138
70 writel_relaxed(evt, data->base + TIM_ARR); 139 writel_relaxed(TIM_DIER_CC1IE, timer_of_base(to) + TIM_DIER);
71 writel_relaxed(TIM_CR1_ARPE | TIM_CR1_OPM | TIM_CR1_CEN, 140
72 data->base + TIM_CR1); 141 return 0;
142}
143
144static int stm32_clock_event_set_periodic(struct clock_event_device *clkevt)
145{
146 struct timer_of *to = to_timer_of(clkevt);
147
148 stm32_timer_start(to);
149
150 return stm32_clock_event_set_next_event(timer_of_period(to), clkevt);
151}
152
153static int stm32_clock_event_set_oneshot(struct clock_event_device *clkevt)
154{
155 struct timer_of *to = to_timer_of(clkevt);
156
157 stm32_timer_start(to);
73 158
74 return 0; 159 return 0;
75} 160}
76 161
77static irqreturn_t stm32_clock_event_handler(int irq, void *dev_id) 162static irqreturn_t stm32_clock_event_handler(int irq, void *dev_id)
78{ 163{
79 struct stm32_clock_event_ddata *data = dev_id; 164 struct clock_event_device *clkevt = (struct clock_event_device *)dev_id;
165 struct timer_of *to = to_timer_of(clkevt);
166
167 writel_relaxed(0, timer_of_base(to) + TIM_SR);
80 168
81 writel_relaxed(0, data->base + TIM_SR); 169 if (clockevent_state_periodic(clkevt))
170 stm32_clock_event_set_periodic(clkevt);
171 else
172 stm32_clock_event_shutdown(clkevt);
82 173
83 data->evtdev.event_handler(&data->evtdev); 174 clkevt->event_handler(clkevt);
84 175
85 return IRQ_HANDLED; 176 return IRQ_HANDLED;
86} 177}
87 178
88static struct stm32_clock_event_ddata clock_event_ddata = { 179/**
89 .evtdev = { 180 * stm32_timer_width - Sort out the timer width (32/16)
90 .name = "stm32 clockevent", 181 * @to: a pointer to a timer-of structure
91 .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC, 182 *
92 .set_state_shutdown = stm32_clock_event_shutdown, 183 * Write the 32-bit max value and read/return the result. If the timer
93 .set_state_periodic = stm32_clock_event_set_periodic, 184 * is 32 bits wide, the result will be UINT_MAX, otherwise it will
94 .set_state_oneshot = stm32_clock_event_shutdown, 185 * be truncated by the 16-bit register to USHRT_MAX.
95 .tick_resume = stm32_clock_event_shutdown, 186 *
96 .set_next_event = stm32_clock_event_set_next_event, 187 */
97 .rating = 200, 188static void __init stm32_timer_set_width(struct timer_of *to)
98 }, 189{
99}; 190 u32 width;
191
192 writel_relaxed(UINT_MAX, timer_of_base(to) + TIM_ARR);
193
194 width = readl_relaxed(timer_of_base(to) + TIM_ARR);
195
196 stm32_timer_of_bits_set(to, width == UINT_MAX ? 32 : 16);
197}
100 198
101static int __init stm32_clockevent_init(struct device_node *np) 199/**
200 * stm32_timer_set_prescaler - Compute and set the prescaler register
201 * @to: a pointer to a timer-of structure
202 *
203 * Depending on the timer width, compute the prescaler to always
204 * target a 10MHz timer rate for 16 bits. 32-bit timers are
205 * considered precise and long enough to not use the prescaler.
206 */
207static void __init stm32_timer_set_prescaler(struct timer_of *to)
102{ 208{
103 struct stm32_clock_event_ddata *data = &clock_event_ddata; 209 int prescaler = 1;
104 struct clk *clk; 210
105 struct reset_control *rstc; 211 if (stm32_timer_of_bits_get(to) != 32) {
106 unsigned long rate, max_delta; 212 prescaler = DIV_ROUND_CLOSEST(timer_of_rate(to),
107 int irq, ret, bits, prescaler = 1; 213 TIM_PSC_CLKRATE);
108 214 /*
109 clk = of_clk_get(np, 0); 215 * The prescaler register is an u16, the variable
110 if (IS_ERR(clk)) { 216 * can't be greater than TIM_PSC_MAX, let's cap it in
111 ret = PTR_ERR(clk); 217 * this case.
112 pr_err("failed to get clock for clockevent (%d)\n", ret); 218 */
113 goto err_clk_get; 219 prescaler = prescaler < TIM_PSC_MAX ? prescaler : TIM_PSC_MAX;
114 } 220 }
115 221
116 ret = clk_prepare_enable(clk); 222 writel_relaxed(prescaler - 1, timer_of_base(to) + TIM_PSC);
117 if (ret) { 223 writel_relaxed(TIM_EGR_UG, timer_of_base(to) + TIM_EGR);
118 pr_err("failed to enable timer clock for clockevent (%d)\n", 224 writel_relaxed(0, timer_of_base(to) + TIM_SR);
119 ret);
120 goto err_clk_enable;
121 }
122 225
123 rate = clk_get_rate(clk); 226 /* Adjust rate and period given the prescaler value */
227 to->of_clk.rate = DIV_ROUND_CLOSEST(to->of_clk.rate, prescaler);
228 to->of_clk.period = DIV_ROUND_UP(to->of_clk.rate, HZ);
229}
124 230
125 rstc = of_reset_control_get(np, NULL); 231static int __init stm32_clocksource_init(struct timer_of *to)
126 if (!IS_ERR(rstc)) { 232{
127 reset_control_assert(rstc); 233 u32 bits = stm32_timer_of_bits_get(to);
128 reset_control_deassert(rstc); 234 const char *name = to->np->full_name;
235
236 /*
237 * This driver allows to register several timers and relies on
238 * the generic time framework to select the right one.
239 * However, nothing allows to do the same for the
240 * sched_clock. We are not interested in a sched_clock for the
241 * 16-bit timers but only for the 32-bit one, so if no 32-bit
242 * timer is registered yet, we select this 32-bit timer as a
243 * sched_clock.
244 */
245 if (bits == 32 && !stm32_timer_cnt) {
246
247 /*
248 * Start immediately the counter as we will be using
249 * it right after.
250 */
251 stm32_timer_start(to);
252
253 stm32_timer_cnt = timer_of_base(to) + TIM_CNT;
254 sched_clock_register(stm32_read_sched_clock, bits, timer_of_rate(to));
255 pr_info("%s: STM32 sched_clock registered\n", name);
256
257 stm32_timer_delay.read_current_timer = stm32_read_delay;
258 stm32_timer_delay.freq = timer_of_rate(to);
259 register_current_timer_delay(&stm32_timer_delay);
260 pr_info("%s: STM32 delay timer registered\n", name);
129 } 261 }
130 262
131 data->base = of_iomap(np, 0); 263 return clocksource_mmio_init(timer_of_base(to) + TIM_CNT, name,
132 if (!data->base) { 264 timer_of_rate(to), bits == 32 ? 250 : 100,
133 ret = -ENXIO; 265 bits, clocksource_mmio_readl_up);
134 pr_err("failed to map registers for clockevent\n"); 266}
135 goto err_iomap;
136 }
137 267
138 irq = irq_of_parse_and_map(np, 0); 268static void __init stm32_clockevent_init(struct timer_of *to)
139 if (!irq) { 269{
140 ret = -EINVAL; 270 u32 bits = stm32_timer_of_bits_get(to);
141 pr_err("%pOF: failed to get irq.\n", np);
142 goto err_get_irq;
143 }
144 271
145 /* Detect whether the timer is 16 or 32 bits */ 272 to->clkevt.name = to->np->full_name;
146 writel_relaxed(~0U, data->base + TIM_ARR); 273 to->clkevt.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT;
147 max_delta = readl_relaxed(data->base + TIM_ARR); 274 to->clkevt.set_state_shutdown = stm32_clock_event_shutdown;
148 if (max_delta == ~0U) { 275 to->clkevt.set_state_periodic = stm32_clock_event_set_periodic;
149 prescaler = 1; 276 to->clkevt.set_state_oneshot = stm32_clock_event_set_oneshot;
150 bits = 32; 277 to->clkevt.tick_resume = stm32_clock_event_shutdown;
151 } else { 278 to->clkevt.set_next_event = stm32_clock_event_set_next_event;
152 prescaler = 1024; 279 to->clkevt.rating = bits == 32 ? 250 : 100;
153 bits = 16; 280
154 } 281 clockevents_config_and_register(&to->clkevt, timer_of_rate(to), 0x1,
155 writel_relaxed(0, data->base + TIM_ARR); 282 (1 << bits) - 1);
283
284 pr_info("%pOF: STM32 clockevent driver initialized (%d bits)\n",
285 to->np, bits);
286}
287
288static int __init stm32_timer_init(struct device_node *node)
289{
290 struct reset_control *rstc;
291 struct timer_of *to;
292 int ret;
293
294 to = kzalloc(sizeof(*to), GFP_KERNEL);
295 if (!to)
296 return -ENOMEM;
156 297
157 writel_relaxed(prescaler - 1, data->base + TIM_PSC); 298 to->flags = TIMER_OF_IRQ | TIMER_OF_CLOCK | TIMER_OF_BASE;
158 writel_relaxed(TIM_EGR_UG, data->base + TIM_EGR); 299 to->of_irq.handler = stm32_clock_event_handler;
159 writel_relaxed(TIM_DIER_UIE, data->base + TIM_DIER);
160 writel_relaxed(0, data->base + TIM_SR);
161 300
162 data->periodic_top = DIV_ROUND_CLOSEST(rate, prescaler * HZ); 301 ret = timer_of_init(node, to);
302 if (ret)
303 goto err;
163 304
164 clockevents_config_and_register(&data->evtdev, 305 to->private_data = kzalloc(sizeof(struct stm32_timer_private),
165 DIV_ROUND_CLOSEST(rate, prescaler), 306 GFP_KERNEL);
166 0x1, max_delta); 307 if (!to->private_data)
308 goto deinit;
167 309
168 ret = request_irq(irq, stm32_clock_event_handler, IRQF_TIMER, 310 rstc = of_reset_control_get(node, NULL);
169 "stm32 clockevent", data); 311 if (!IS_ERR(rstc)) {
170 if (ret) { 312 reset_control_assert(rstc);
171 pr_err("%pOF: failed to request irq.\n", np); 313 reset_control_deassert(rstc);
172 goto err_get_irq;
173 } 314 }
174 315
175 pr_info("%pOF: STM32 clockevent driver initialized (%d bits)\n", 316 stm32_timer_set_width(to);
176 np, bits);
177 317
178 return ret; 318 stm32_timer_set_prescaler(to);
319
320 ret = stm32_clocksource_init(to);
321 if (ret)
322 goto deinit;
323
324 stm32_clockevent_init(to);
325 return 0;
179 326
180err_get_irq: 327deinit:
181 iounmap(data->base); 328 timer_of_cleanup(to);
182err_iomap: 329err:
183 clk_disable_unprepare(clk); 330 kfree(to);
184err_clk_enable:
185 clk_put(clk);
186err_clk_get:
187 return ret; 331 return ret;
188} 332}
189 333
190TIMER_OF_DECLARE(stm32, "st,stm32-timer", stm32_clockevent_init); 334TIMER_OF_DECLARE(stm32, "st,stm32-timer", stm32_timer_init);
diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c
index c5bce8e22983..5780fba620ab 100644
--- a/drivers/usb/gadget/function/f_ncm.c
+++ b/drivers/usb/gadget/function/f_ncm.c
@@ -73,9 +73,7 @@ struct f_ncm {
73 struct sk_buff *skb_tx_ndp; 73 struct sk_buff *skb_tx_ndp;
74 u16 ndp_dgram_count; 74 u16 ndp_dgram_count;
75 bool timer_force_tx; 75 bool timer_force_tx;
76 struct tasklet_struct tx_tasklet;
77 struct hrtimer task_timer; 76 struct hrtimer task_timer;
78
79 bool timer_stopping; 77 bool timer_stopping;
80}; 78};
81 79
@@ -1104,7 +1102,7 @@ static struct sk_buff *ncm_wrap_ntb(struct gether *port,
1104 1102
1105 /* Delay the timer. */ 1103 /* Delay the timer. */
1106 hrtimer_start(&ncm->task_timer, TX_TIMEOUT_NSECS, 1104 hrtimer_start(&ncm->task_timer, TX_TIMEOUT_NSECS,
1107 HRTIMER_MODE_REL); 1105 HRTIMER_MODE_REL_SOFT);
1108 1106
1109 /* Add the datagram position entries */ 1107 /* Add the datagram position entries */
1110 ntb_ndp = skb_put_zero(ncm->skb_tx_ndp, dgram_idx_len); 1108 ntb_ndp = skb_put_zero(ncm->skb_tx_ndp, dgram_idx_len);
@@ -1148,17 +1146,15 @@ err:
1148} 1146}
1149 1147
1150/* 1148/*
1151 * This transmits the NTB if there are frames waiting. 1149 * The transmit should only be run if no skb data has been sent
1150 * for a certain duration.
1152 */ 1151 */
1153static void ncm_tx_tasklet(unsigned long data) 1152static enum hrtimer_restart ncm_tx_timeout(struct hrtimer *data)
1154{ 1153{
1155 struct f_ncm *ncm = (void *)data; 1154 struct f_ncm *ncm = container_of(data, struct f_ncm, task_timer);
1156
1157 if (ncm->timer_stopping)
1158 return;
1159 1155
1160 /* Only send if data is available. */ 1156 /* Only send if data is available. */
1161 if (ncm->skb_tx_data) { 1157 if (!ncm->timer_stopping && ncm->skb_tx_data) {
1162 ncm->timer_force_tx = true; 1158 ncm->timer_force_tx = true;
1163 1159
1164 /* XXX This allowance of a NULL skb argument to ndo_start_xmit 1160 /* XXX This allowance of a NULL skb argument to ndo_start_xmit
@@ -1171,16 +1167,6 @@ static void ncm_tx_tasklet(unsigned long data)
1171 1167
1172 ncm->timer_force_tx = false; 1168 ncm->timer_force_tx = false;
1173 } 1169 }
1174}
1175
1176/*
1177 * The transmit should only be run if no skb data has been sent
1178 * for a certain duration.
1179 */
1180static enum hrtimer_restart ncm_tx_timeout(struct hrtimer *data)
1181{
1182 struct f_ncm *ncm = container_of(data, struct f_ncm, task_timer);
1183 tasklet_schedule(&ncm->tx_tasklet);
1184 return HRTIMER_NORESTART; 1170 return HRTIMER_NORESTART;
1185} 1171}
1186 1172
@@ -1513,8 +1499,7 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f)
1513 ncm->port.open = ncm_open; 1499 ncm->port.open = ncm_open;
1514 ncm->port.close = ncm_close; 1500 ncm->port.close = ncm_close;
1515 1501
1516 tasklet_init(&ncm->tx_tasklet, ncm_tx_tasklet, (unsigned long) ncm); 1502 hrtimer_init(&ncm->task_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
1517 hrtimer_init(&ncm->task_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1518 ncm->task_timer.function = ncm_tx_timeout; 1503 ncm->task_timer.function = ncm_tx_timeout;
1519 1504
1520 DBG(cdev, "CDC Network: %s speed IN/%s OUT/%s NOTIFY/%s\n", 1505 DBG(cdev, "CDC Network: %s speed IN/%s OUT/%s NOTIFY/%s\n",
@@ -1623,7 +1608,6 @@ static void ncm_unbind(struct usb_configuration *c, struct usb_function *f)
1623 DBG(c->cdev, "ncm unbind\n"); 1608 DBG(c->cdev, "ncm unbind\n");
1624 1609
1625 hrtimer_cancel(&ncm->task_timer); 1610 hrtimer_cancel(&ncm->task_timer);
1626 tasklet_kill(&ncm->tx_tasklet);
1627 1611
1628 ncm_string_defs[0].id = 0; 1612 ncm_string_defs[0].id = 0;
1629 usb_free_all_descriptors(f); 1613 usb_free_all_descriptors(f);
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 012c37fdb688..c7902ca7c9f4 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -28,13 +28,29 @@ struct hrtimer_cpu_base;
28 28
29/* 29/*
30 * Mode arguments of xxx_hrtimer functions: 30 * Mode arguments of xxx_hrtimer functions:
31 *
32 * HRTIMER_MODE_ABS - Time value is absolute
33 * HRTIMER_MODE_REL - Time value is relative to now
34 * HRTIMER_MODE_PINNED - Timer is bound to CPU (is only considered
35 * when starting the timer)
36 * HRTIMER_MODE_SOFT - Timer callback function will be executed in
37 * soft irq context
31 */ 38 */
32enum hrtimer_mode { 39enum hrtimer_mode {
33 HRTIMER_MODE_ABS = 0x0, /* Time value is absolute */ 40 HRTIMER_MODE_ABS = 0x00,
34 HRTIMER_MODE_REL = 0x1, /* Time value is relative to now */ 41 HRTIMER_MODE_REL = 0x01,
35 HRTIMER_MODE_PINNED = 0x02, /* Timer is bound to CPU */ 42 HRTIMER_MODE_PINNED = 0x02,
36 HRTIMER_MODE_ABS_PINNED = 0x02, 43 HRTIMER_MODE_SOFT = 0x04,
37 HRTIMER_MODE_REL_PINNED = 0x03, 44
45 HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
46 HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
47
48 HRTIMER_MODE_ABS_SOFT = HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT,
49 HRTIMER_MODE_REL_SOFT = HRTIMER_MODE_REL | HRTIMER_MODE_SOFT,
50
51 HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
52 HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
53
38}; 54};
39 55
40/* 56/*
@@ -87,6 +103,7 @@ enum hrtimer_restart {
87 * @base: pointer to the timer base (per cpu and per clock) 103 * @base: pointer to the timer base (per cpu and per clock)
88 * @state: state information (See bit values above) 104 * @state: state information (See bit values above)
89 * @is_rel: Set if the timer was armed relative 105 * @is_rel: Set if the timer was armed relative
106 * @is_soft: Set if hrtimer will be expired in soft interrupt context.
90 * 107 *
91 * The hrtimer structure must be initialized by hrtimer_init() 108 * The hrtimer structure must be initialized by hrtimer_init()
92 */ 109 */
@@ -97,6 +114,7 @@ struct hrtimer {
97 struct hrtimer_clock_base *base; 114 struct hrtimer_clock_base *base;
98 u8 state; 115 u8 state;
99 u8 is_rel; 116 u8 is_rel;
117 u8 is_soft;
100}; 118};
101 119
102/** 120/**
@@ -112,9 +130,9 @@ struct hrtimer_sleeper {
112}; 130};
113 131
114#ifdef CONFIG_64BIT 132#ifdef CONFIG_64BIT
115# define HRTIMER_CLOCK_BASE_ALIGN 64 133# define __hrtimer_clock_base_align ____cacheline_aligned
116#else 134#else
117# define HRTIMER_CLOCK_BASE_ALIGN 32 135# define __hrtimer_clock_base_align
118#endif 136#endif
119 137
120/** 138/**
@@ -123,48 +141,57 @@ struct hrtimer_sleeper {
123 * @index: clock type index for per_cpu support when moving a 141 * @index: clock type index for per_cpu support when moving a
124 * timer to a base on another cpu. 142 * timer to a base on another cpu.
125 * @clockid: clock id for per_cpu support 143 * @clockid: clock id for per_cpu support
144 * @seq: seqcount around __run_hrtimer
145 * @running: pointer to the currently running hrtimer
126 * @active: red black tree root node for the active timers 146 * @active: red black tree root node for the active timers
127 * @get_time: function to retrieve the current time of the clock 147 * @get_time: function to retrieve the current time of the clock
128 * @offset: offset of this clock to the monotonic base 148 * @offset: offset of this clock to the monotonic base
129 */ 149 */
130struct hrtimer_clock_base { 150struct hrtimer_clock_base {
131 struct hrtimer_cpu_base *cpu_base; 151 struct hrtimer_cpu_base *cpu_base;
132 int index; 152 unsigned int index;
133 clockid_t clockid; 153 clockid_t clockid;
154 seqcount_t seq;
155 struct hrtimer *running;
134 struct timerqueue_head active; 156 struct timerqueue_head active;
135 ktime_t (*get_time)(void); 157 ktime_t (*get_time)(void);
136 ktime_t offset; 158 ktime_t offset;
137} __attribute__((__aligned__(HRTIMER_CLOCK_BASE_ALIGN))); 159} __hrtimer_clock_base_align;
138 160
139enum hrtimer_base_type { 161enum hrtimer_base_type {
140 HRTIMER_BASE_MONOTONIC, 162 HRTIMER_BASE_MONOTONIC,
141 HRTIMER_BASE_REALTIME, 163 HRTIMER_BASE_REALTIME,
142 HRTIMER_BASE_BOOTTIME, 164 HRTIMER_BASE_BOOTTIME,
143 HRTIMER_BASE_TAI, 165 HRTIMER_BASE_TAI,
166 HRTIMER_BASE_MONOTONIC_SOFT,
167 HRTIMER_BASE_REALTIME_SOFT,
168 HRTIMER_BASE_BOOTTIME_SOFT,
169 HRTIMER_BASE_TAI_SOFT,
144 HRTIMER_MAX_CLOCK_BASES, 170 HRTIMER_MAX_CLOCK_BASES,
145}; 171};
146 172
147/* 173/**
148 * struct hrtimer_cpu_base - the per cpu clock bases 174 * struct hrtimer_cpu_base - the per cpu clock bases
149 * @lock: lock protecting the base and associated clock bases 175 * @lock: lock protecting the base and associated clock bases
150 * and timers 176 * and timers
151 * @seq: seqcount around __run_hrtimer
152 * @running: pointer to the currently running hrtimer
153 * @cpu: cpu number 177 * @cpu: cpu number
154 * @active_bases: Bitfield to mark bases with active timers 178 * @active_bases: Bitfield to mark bases with active timers
155 * @clock_was_set_seq: Sequence counter of clock was set events 179 * @clock_was_set_seq: Sequence counter of clock was set events
156 * @migration_enabled: The migration of hrtimers to other cpus is enabled
157 * @nohz_active: The nohz functionality is enabled
158 * @expires_next: absolute time of the next event which was scheduled
159 * via clock_set_next_event()
160 * @next_timer: Pointer to the first expiring timer
161 * @in_hrtirq: hrtimer_interrupt() is currently executing
162 * @hres_active: State of high resolution mode 180 * @hres_active: State of high resolution mode
181 * @in_hrtirq: hrtimer_interrupt() is currently executing
163 * @hang_detected: The last hrtimer interrupt detected a hang 182 * @hang_detected: The last hrtimer interrupt detected a hang
183 * @softirq_activated: displays, if the softirq is raised - update of softirq
184 * related settings is not required then.
164 * @nr_events: Total number of hrtimer interrupt events 185 * @nr_events: Total number of hrtimer interrupt events
165 * @nr_retries: Total number of hrtimer interrupt retries 186 * @nr_retries: Total number of hrtimer interrupt retries
166 * @nr_hangs: Total number of hrtimer interrupt hangs 187 * @nr_hangs: Total number of hrtimer interrupt hangs
167 * @max_hang_time: Maximum time spent in hrtimer_interrupt 188 * @max_hang_time: Maximum time spent in hrtimer_interrupt
189 * @expires_next: absolute time of the next event, is required for remote
190 * hrtimer enqueue; it is the total first expiry time (hard
191 * and soft hrtimer are taken into account)
192 * @next_timer: Pointer to the first expiring timer
193 * @softirq_expires_next: Time to check, if soft queues needs also to be expired
194 * @softirq_next_timer: Pointer to the first expiring softirq based timer
168 * @clock_base: array of clock bases for this cpu 195 * @clock_base: array of clock bases for this cpu
169 * 196 *
170 * Note: next_timer is just an optimization for __remove_hrtimer(). 197 * Note: next_timer is just an optimization for __remove_hrtimer().
@@ -173,31 +200,28 @@ enum hrtimer_base_type {
173 */ 200 */
174struct hrtimer_cpu_base { 201struct hrtimer_cpu_base {
175 raw_spinlock_t lock; 202 raw_spinlock_t lock;
176 seqcount_t seq;
177 struct hrtimer *running;
178 unsigned int cpu; 203 unsigned int cpu;
179 unsigned int active_bases; 204 unsigned int active_bases;
180 unsigned int clock_was_set_seq; 205 unsigned int clock_was_set_seq;
181 bool migration_enabled; 206 unsigned int hres_active : 1,
182 bool nohz_active; 207 in_hrtirq : 1,
208 hang_detected : 1,
209 softirq_activated : 1;
183#ifdef CONFIG_HIGH_RES_TIMERS 210#ifdef CONFIG_HIGH_RES_TIMERS
184 unsigned int in_hrtirq : 1,
185 hres_active : 1,
186 hang_detected : 1;
187 ktime_t expires_next;
188 struct hrtimer *next_timer;
189 unsigned int nr_events; 211 unsigned int nr_events;
190 unsigned int nr_retries; 212 unsigned short nr_retries;
191 unsigned int nr_hangs; 213 unsigned short nr_hangs;
192 unsigned int max_hang_time; 214 unsigned int max_hang_time;
193#endif 215#endif
216 ktime_t expires_next;
217 struct hrtimer *next_timer;
218 ktime_t softirq_expires_next;
219 struct hrtimer *softirq_next_timer;
194 struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; 220 struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
195} ____cacheline_aligned; 221} ____cacheline_aligned;
196 222
197static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) 223static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
198{ 224{
199 BUILD_BUG_ON(sizeof(struct hrtimer_clock_base) > HRTIMER_CLOCK_BASE_ALIGN);
200
201 timer->node.expires = time; 225 timer->node.expires = time;
202 timer->_softexpires = time; 226 timer->_softexpires = time;
203} 227}
@@ -266,16 +290,17 @@ static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
266 return timer->base->get_time(); 290 return timer->base->get_time();
267} 291}
268 292
293static inline int hrtimer_is_hres_active(struct hrtimer *timer)
294{
295 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
296 timer->base->cpu_base->hres_active : 0;
297}
298
269#ifdef CONFIG_HIGH_RES_TIMERS 299#ifdef CONFIG_HIGH_RES_TIMERS
270struct clock_event_device; 300struct clock_event_device;
271 301
272extern void hrtimer_interrupt(struct clock_event_device *dev); 302extern void hrtimer_interrupt(struct clock_event_device *dev);
273 303
274static inline int hrtimer_is_hres_active(struct hrtimer *timer)
275{
276 return timer->base->cpu_base->hres_active;
277}
278
279/* 304/*
280 * The resolution of the clocks. The resolution value is returned in 305 * The resolution of the clocks. The resolution value is returned in
281 * the clock_getres() system call to give application programmers an 306 * the clock_getres() system call to give application programmers an
@@ -298,11 +323,6 @@ extern unsigned int hrtimer_resolution;
298 323
299#define hrtimer_resolution (unsigned int)LOW_RES_NSEC 324#define hrtimer_resolution (unsigned int)LOW_RES_NSEC
300 325
301static inline int hrtimer_is_hres_active(struct hrtimer *timer)
302{
303 return 0;
304}
305
306static inline void clock_was_set_delayed(void) { } 326static inline void clock_was_set_delayed(void) { }
307 327
308#endif 328#endif
@@ -365,11 +385,12 @@ extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
365 u64 range_ns, const enum hrtimer_mode mode); 385 u64 range_ns, const enum hrtimer_mode mode);
366 386
367/** 387/**
368 * hrtimer_start - (re)start an hrtimer on the current CPU 388 * hrtimer_start - (re)start an hrtimer
369 * @timer: the timer to be added 389 * @timer: the timer to be added
370 * @tim: expiry time 390 * @tim: expiry time
371 * @mode: expiry mode: absolute (HRTIMER_MODE_ABS) or 391 * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
372 * relative (HRTIMER_MODE_REL) 392 * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
393 * softirq based mode is considered for debug purpose only!
373 */ 394 */
374static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim, 395static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
375 const enum hrtimer_mode mode) 396 const enum hrtimer_mode mode)
@@ -422,7 +443,7 @@ static inline int hrtimer_is_queued(struct hrtimer *timer)
422 */ 443 */
423static inline int hrtimer_callback_running(struct hrtimer *timer) 444static inline int hrtimer_callback_running(struct hrtimer *timer)
424{ 445{
425 return timer->base->cpu_base->running == timer; 446 return timer->base->running == timer;
426} 447}
427 448
428/* Forward a hrtimer so it expires after now: */ 449/* Forward a hrtimer so it expires after now: */
@@ -466,7 +487,7 @@ extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
466extern int schedule_hrtimeout_range_clock(ktime_t *expires, 487extern int schedule_hrtimeout_range_clock(ktime_t *expires,
467 u64 delta, 488 u64 delta,
468 const enum hrtimer_mode mode, 489 const enum hrtimer_mode mode,
469 int clock); 490 clockid_t clock_id);
470extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode); 491extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
471 492
472/* Soft interrupt function to run the hrtimer queues: */ 493/* Soft interrupt function to run the hrtimer queues: */
diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h
index 672c4f32311e..c85704fcdbd2 100644
--- a/include/linux/posix-timers.h
+++ b/include/linux/posix-timers.h
@@ -42,13 +42,26 @@ struct cpu_timer_list {
42#define CLOCKFD CPUCLOCK_MAX 42#define CLOCKFD CPUCLOCK_MAX
43#define CLOCKFD_MASK (CPUCLOCK_PERTHREAD_MASK|CPUCLOCK_CLOCK_MASK) 43#define CLOCKFD_MASK (CPUCLOCK_PERTHREAD_MASK|CPUCLOCK_CLOCK_MASK)
44 44
45#define MAKE_PROCESS_CPUCLOCK(pid, clock) \ 45static inline clockid_t make_process_cpuclock(const unsigned int pid,
46 ((~(clockid_t) (pid) << 3) | (clockid_t) (clock)) 46 const clockid_t clock)
47#define MAKE_THREAD_CPUCLOCK(tid, clock) \ 47{
48 MAKE_PROCESS_CPUCLOCK((tid), (clock) | CPUCLOCK_PERTHREAD_MASK) 48 return ((~pid) << 3) | clock;
49}
50static inline clockid_t make_thread_cpuclock(const unsigned int tid,
51 const clockid_t clock)
52{
53 return make_process_cpuclock(tid, clock | CPUCLOCK_PERTHREAD_MASK);
54}
49 55
50#define FD_TO_CLOCKID(fd) ((~(clockid_t) (fd) << 3) | CLOCKFD) 56static inline clockid_t fd_to_clockid(const int fd)
51#define CLOCKID_TO_FD(clk) ((unsigned int) ~((clk) >> 3)) 57{
58 return make_process_cpuclock((unsigned int) fd, CLOCKFD);
59}
60
61static inline int clockid_to_fd(const clockid_t clk)
62{
63 return ~(clk >> 3);
64}
52 65
53#define REQUEUE_PENDING 1 66#define REQUEUE_PENDING 1
54 67
diff --git a/include/trace/events/timer.h b/include/trace/events/timer.h
index 16e305e69f34..a57e4ee989d6 100644
--- a/include/trace/events/timer.h
+++ b/include/trace/events/timer.h
@@ -136,6 +136,24 @@ DEFINE_EVENT(timer_class, timer_cancel,
136 TP_ARGS(timer) 136 TP_ARGS(timer)
137); 137);
138 138
139#define decode_clockid(type) \
140 __print_symbolic(type, \
141 { CLOCK_REALTIME, "CLOCK_REALTIME" }, \
142 { CLOCK_MONOTONIC, "CLOCK_MONOTONIC" }, \
143 { CLOCK_BOOTTIME, "CLOCK_BOOTTIME" }, \
144 { CLOCK_TAI, "CLOCK_TAI" })
145
146#define decode_hrtimer_mode(mode) \
147 __print_symbolic(mode, \
148 { HRTIMER_MODE_ABS, "ABS" }, \
149 { HRTIMER_MODE_REL, "REL" }, \
150 { HRTIMER_MODE_ABS_PINNED, "ABS|PINNED" }, \
151 { HRTIMER_MODE_REL_PINNED, "REL|PINNED" }, \
152 { HRTIMER_MODE_ABS_SOFT, "ABS|SOFT" }, \
153 { HRTIMER_MODE_REL_SOFT, "REL|SOFT" }, \
154 { HRTIMER_MODE_ABS_PINNED_SOFT, "ABS|PINNED|SOFT" }, \
155 { HRTIMER_MODE_REL_PINNED_SOFT, "REL|PINNED|SOFT" })
156
139/** 157/**
140 * hrtimer_init - called when the hrtimer is initialized 158 * hrtimer_init - called when the hrtimer is initialized
141 * @hrtimer: pointer to struct hrtimer 159 * @hrtimer: pointer to struct hrtimer
@@ -162,10 +180,8 @@ TRACE_EVENT(hrtimer_init,
162 ), 180 ),
163 181
164 TP_printk("hrtimer=%p clockid=%s mode=%s", __entry->hrtimer, 182 TP_printk("hrtimer=%p clockid=%s mode=%s", __entry->hrtimer,
165 __entry->clockid == CLOCK_REALTIME ? 183 decode_clockid(__entry->clockid),
166 "CLOCK_REALTIME" : "CLOCK_MONOTONIC", 184 decode_hrtimer_mode(__entry->mode))
167 __entry->mode == HRTIMER_MODE_ABS ?
168 "HRTIMER_MODE_ABS" : "HRTIMER_MODE_REL")
169); 185);
170 186
171/** 187/**
@@ -174,15 +190,16 @@ TRACE_EVENT(hrtimer_init,
174 */ 190 */
175TRACE_EVENT(hrtimer_start, 191TRACE_EVENT(hrtimer_start,
176 192
177 TP_PROTO(struct hrtimer *hrtimer), 193 TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode),
178 194
179 TP_ARGS(hrtimer), 195 TP_ARGS(hrtimer, mode),
180 196
181 TP_STRUCT__entry( 197 TP_STRUCT__entry(
182 __field( void *, hrtimer ) 198 __field( void *, hrtimer )
183 __field( void *, function ) 199 __field( void *, function )
184 __field( s64, expires ) 200 __field( s64, expires )
185 __field( s64, softexpires ) 201 __field( s64, softexpires )
202 __field( enum hrtimer_mode, mode )
186 ), 203 ),
187 204
188 TP_fast_assign( 205 TP_fast_assign(
@@ -190,12 +207,14 @@ TRACE_EVENT(hrtimer_start,
190 __entry->function = hrtimer->function; 207 __entry->function = hrtimer->function;
191 __entry->expires = hrtimer_get_expires(hrtimer); 208 __entry->expires = hrtimer_get_expires(hrtimer);
192 __entry->softexpires = hrtimer_get_softexpires(hrtimer); 209 __entry->softexpires = hrtimer_get_softexpires(hrtimer);
210 __entry->mode = mode;
193 ), 211 ),
194 212
195 TP_printk("hrtimer=%p function=%pf expires=%llu softexpires=%llu", 213 TP_printk("hrtimer=%p function=%pf expires=%llu softexpires=%llu "
196 __entry->hrtimer, __entry->function, 214 "mode=%s", __entry->hrtimer, __entry->function,
197 (unsigned long long) __entry->expires, 215 (unsigned long long) __entry->expires,
198 (unsigned long long) __entry->softexpires) 216 (unsigned long long) __entry->softexpires,
217 decode_hrtimer_mode(__entry->mode))
199); 218);
200 219
201/** 220/**
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index aa9d2a2b1210..ae0c8a411fe7 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -60,6 +60,15 @@
60#include "tick-internal.h" 60#include "tick-internal.h"
61 61
62/* 62/*
63 * Masks for selecting the soft and hard context timers from
64 * cpu_base->active
65 */
66#define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT)
67#define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)
68#define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
69#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
70
71/*
63 * The timer bases: 72 * The timer bases:
64 * 73 *
65 * There are more clockids than hrtimer bases. Thus, we index 74 * There are more clockids than hrtimer bases. Thus, we index
@@ -70,7 +79,6 @@
70DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = 79DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
71{ 80{
72 .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock), 81 .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
73 .seq = SEQCNT_ZERO(hrtimer_bases.seq),
74 .clock_base = 82 .clock_base =
75 { 83 {
76 { 84 {
@@ -93,6 +101,26 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
93 .clockid = CLOCK_TAI, 101 .clockid = CLOCK_TAI,
94 .get_time = &ktime_get_clocktai, 102 .get_time = &ktime_get_clocktai,
95 }, 103 },
104 {
105 .index = HRTIMER_BASE_MONOTONIC_SOFT,
106 .clockid = CLOCK_MONOTONIC,
107 .get_time = &ktime_get,
108 },
109 {
110 .index = HRTIMER_BASE_REALTIME_SOFT,
111 .clockid = CLOCK_REALTIME,
112 .get_time = &ktime_get_real,
113 },
114 {
115 .index = HRTIMER_BASE_BOOTTIME_SOFT,
116 .clockid = CLOCK_BOOTTIME,
117 .get_time = &ktime_get_boottime,
118 },
119 {
120 .index = HRTIMER_BASE_TAI_SOFT,
121 .clockid = CLOCK_TAI,
122 .get_time = &ktime_get_clocktai,
123 },
96 } 124 }
97}; 125};
98 126
@@ -118,7 +146,6 @@ static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
118 * timer->base->cpu_base 146 * timer->base->cpu_base
119 */ 147 */
120static struct hrtimer_cpu_base migration_cpu_base = { 148static struct hrtimer_cpu_base migration_cpu_base = {
121 .seq = SEQCNT_ZERO(migration_cpu_base),
122 .clock_base = { { .cpu_base = &migration_cpu_base, }, }, 149 .clock_base = { { .cpu_base = &migration_cpu_base, }, },
123}; 150};
124 151
@@ -156,45 +183,33 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
156} 183}
157 184
158/* 185/*
159 * With HIGHRES=y we do not migrate the timer when it is expiring 186 * We do not migrate the timer when it is expiring before the next
160 * before the next event on the target cpu because we cannot reprogram 187 * event on the target cpu. When high resolution is enabled, we cannot
161 * the target cpu hardware and we would cause it to fire late. 188 * reprogram the target cpu hardware and we would cause it to fire
189 * late. To keep it simple, we handle the high resolution enabled and
190 * disabled case similar.
162 * 191 *
163 * Called with cpu_base->lock of target cpu held. 192 * Called with cpu_base->lock of target cpu held.
164 */ 193 */
165static int 194static int
166hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base) 195hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
167{ 196{
168#ifdef CONFIG_HIGH_RES_TIMERS
169 ktime_t expires; 197 ktime_t expires;
170 198
171 if (!new_base->cpu_base->hres_active)
172 return 0;
173
174 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); 199 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
175 return expires <= new_base->cpu_base->expires_next; 200 return expires < new_base->cpu_base->expires_next;
176#else
177 return 0;
178#endif
179} 201}
180 202
181#ifdef CONFIG_NO_HZ_COMMON
182static inline
183struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
184 int pinned)
185{
186 if (pinned || !base->migration_enabled)
187 return base;
188 return &per_cpu(hrtimer_bases, get_nohz_timer_target());
189}
190#else
191static inline 203static inline
192struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, 204struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
193 int pinned) 205 int pinned)
194{ 206{
207#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
208 if (static_branch_likely(&timers_migration_enabled) && !pinned)
209 return &per_cpu(hrtimer_bases, get_nohz_timer_target());
210#endif
195 return base; 211 return base;
196} 212}
197#endif
198 213
199/* 214/*
200 * We switch the timer base to a power-optimized selected CPU target, 215 * We switch the timer base to a power-optimized selected CPU target,
@@ -396,7 +411,8 @@ static inline void debug_hrtimer_init(struct hrtimer *timer)
396 debug_object_init(timer, &hrtimer_debug_descr); 411 debug_object_init(timer, &hrtimer_debug_descr);
397} 412}
398 413
399static inline void debug_hrtimer_activate(struct hrtimer *timer) 414static inline void debug_hrtimer_activate(struct hrtimer *timer,
415 enum hrtimer_mode mode)
400{ 416{
401 debug_object_activate(timer, &hrtimer_debug_descr); 417 debug_object_activate(timer, &hrtimer_debug_descr);
402} 418}
@@ -429,8 +445,10 @@ void destroy_hrtimer_on_stack(struct hrtimer *timer)
429EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack); 445EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
430 446
431#else 447#else
448
432static inline void debug_hrtimer_init(struct hrtimer *timer) { } 449static inline void debug_hrtimer_init(struct hrtimer *timer) { }
433static inline void debug_hrtimer_activate(struct hrtimer *timer) { } 450static inline void debug_hrtimer_activate(struct hrtimer *timer,
451 enum hrtimer_mode mode) { }
434static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } 452static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
435#endif 453#endif
436 454
@@ -442,10 +460,11 @@ debug_init(struct hrtimer *timer, clockid_t clockid,
442 trace_hrtimer_init(timer, clockid, mode); 460 trace_hrtimer_init(timer, clockid, mode);
443} 461}
444 462
445static inline void debug_activate(struct hrtimer *timer) 463static inline void debug_activate(struct hrtimer *timer,
464 enum hrtimer_mode mode)
446{ 465{
447 debug_hrtimer_activate(timer); 466 debug_hrtimer_activate(timer, mode);
448 trace_hrtimer_start(timer); 467 trace_hrtimer_start(timer, mode);
449} 468}
450 469
451static inline void debug_deactivate(struct hrtimer *timer) 470static inline void debug_deactivate(struct hrtimer *timer)
@@ -454,35 +473,43 @@ static inline void debug_deactivate(struct hrtimer *timer)
454 trace_hrtimer_cancel(timer); 473 trace_hrtimer_cancel(timer);
455} 474}
456 475
457#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS) 476static struct hrtimer_clock_base *
458static inline void hrtimer_update_next_timer(struct hrtimer_cpu_base *cpu_base, 477__next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
459 struct hrtimer *timer)
460{ 478{
461#ifdef CONFIG_HIGH_RES_TIMERS 479 unsigned int idx;
462 cpu_base->next_timer = timer; 480
463#endif 481 if (!*active)
482 return NULL;
483
484 idx = __ffs(*active);
485 *active &= ~(1U << idx);
486
487 return &cpu_base->clock_base[idx];
464} 488}
465 489
466static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base) 490#define for_each_active_base(base, cpu_base, active) \
491 while ((base = __next_base((cpu_base), &(active))))
492
493static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
494 unsigned int active,
495 ktime_t expires_next)
467{ 496{
468 struct hrtimer_clock_base *base = cpu_base->clock_base; 497 struct hrtimer_clock_base *base;
469 unsigned int active = cpu_base->active_bases; 498 ktime_t expires;
470 ktime_t expires, expires_next = KTIME_MAX;
471 499
472 hrtimer_update_next_timer(cpu_base, NULL); 500 for_each_active_base(base, cpu_base, active) {
473 for (; active; base++, active >>= 1) {
474 struct timerqueue_node *next; 501 struct timerqueue_node *next;
475 struct hrtimer *timer; 502 struct hrtimer *timer;
476 503
477 if (!(active & 0x01))
478 continue;
479
480 next = timerqueue_getnext(&base->active); 504 next = timerqueue_getnext(&base->active);
481 timer = container_of(next, struct hrtimer, node); 505 timer = container_of(next, struct hrtimer, node);
482 expires = ktime_sub(hrtimer_get_expires(timer), base->offset); 506 expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
483 if (expires < expires_next) { 507 if (expires < expires_next) {
484 expires_next = expires; 508 expires_next = expires;
485 hrtimer_update_next_timer(cpu_base, timer); 509 if (timer->is_soft)
510 cpu_base->softirq_next_timer = timer;
511 else
512 cpu_base->next_timer = timer;
486 } 513 }
487 } 514 }
488 /* 515 /*
@@ -494,7 +521,47 @@ static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base)
494 expires_next = 0; 521 expires_next = 0;
495 return expires_next; 522 return expires_next;
496} 523}
497#endif 524
525/*
526 * Recomputes cpu_base::*next_timer and returns the earliest expires_next but
527 * does not set cpu_base::*expires_next, that is done by hrtimer_reprogram.
528 *
529 * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
530 * those timers will get run whenever the softirq gets handled, at the end of
531 * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
532 *
533 * Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
534 * The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
535 * softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
536 *
537 * @active_mask must be one of:
538 * - HRTIMER_ACTIVE_ALL,
539 * - HRTIMER_ACTIVE_SOFT, or
540 * - HRTIMER_ACTIVE_HARD.
541 */
542static ktime_t
543__hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
544{
545 unsigned int active;
546 struct hrtimer *next_timer = NULL;
547 ktime_t expires_next = KTIME_MAX;
548
549 if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
550 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
551 cpu_base->softirq_next_timer = NULL;
552 expires_next = __hrtimer_next_event_base(cpu_base, active, KTIME_MAX);
553
554 next_timer = cpu_base->softirq_next_timer;
555 }
556
557 if (active_mask & HRTIMER_ACTIVE_HARD) {
558 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
559 cpu_base->next_timer = next_timer;
560 expires_next = __hrtimer_next_event_base(cpu_base, active, expires_next);
561 }
562
563 return expires_next;
564}
498 565
499static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) 566static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
500{ 567{
@@ -502,36 +569,14 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
502 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; 569 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
503 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; 570 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
504 571
505 return ktime_get_update_offsets_now(&base->clock_was_set_seq, 572 ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
506 offs_real, offs_boot, offs_tai); 573 offs_real, offs_boot, offs_tai);
507}
508
509/* High resolution timer related functions */
510#ifdef CONFIG_HIGH_RES_TIMERS
511 574
512/* 575 base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
513 * High resolution timer enabled ? 576 base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
514 */ 577 base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
515static bool hrtimer_hres_enabled __read_mostly = true;
516unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
517EXPORT_SYMBOL_GPL(hrtimer_resolution);
518 578
519/* 579 return now;
520 * Enable / Disable high resolution mode
521 */
522static int __init setup_hrtimer_hres(char *str)
523{
524 return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
525}
526
527__setup("highres=", setup_hrtimer_hres);
528
529/*
530 * hrtimer_high_res_enabled - query, if the highres mode is enabled
531 */
532static inline int hrtimer_is_hres_enabled(void)
533{
534 return hrtimer_hres_enabled;
535} 580}
536 581
537/* 582/*
@@ -539,7 +584,8 @@ static inline int hrtimer_is_hres_enabled(void)
539 */ 584 */
540static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base) 585static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
541{ 586{
542 return cpu_base->hres_active; 587 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
588 cpu_base->hres_active : 0;
543} 589}
544 590
545static inline int hrtimer_hres_active(void) 591static inline int hrtimer_hres_active(void)
@@ -557,10 +603,23 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
557{ 603{
558 ktime_t expires_next; 604 ktime_t expires_next;
559 605
560 if (!cpu_base->hres_active) 606 /*
561 return; 607 * Find the current next expiration time.
608 */
609 expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
562 610
563 expires_next = __hrtimer_get_next_event(cpu_base); 611 if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {
612 /*
613 * When the softirq is activated, hrtimer has to be
614 * programmed with the first hard hrtimer because soft
615 * timer interrupt could occur too late.
616 */
617 if (cpu_base->softirq_activated)
618 expires_next = __hrtimer_get_next_event(cpu_base,
619 HRTIMER_ACTIVE_HARD);
620 else
621 cpu_base->softirq_expires_next = expires_next;
622 }
564 623
565 if (skip_equal && expires_next == cpu_base->expires_next) 624 if (skip_equal && expires_next == cpu_base->expires_next)
566 return; 625 return;
@@ -568,6 +627,9 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
568 cpu_base->expires_next = expires_next; 627 cpu_base->expires_next = expires_next;
569 628
570 /* 629 /*
630 * If hres is not active, hardware does not have to be
631 * reprogrammed yet.
632 *
571 * If a hang was detected in the last timer interrupt then we 633 * If a hang was detected in the last timer interrupt then we
572 * leave the hang delay active in the hardware. We want the 634 * leave the hang delay active in the hardware. We want the
573 * system to make progress. That also prevents the following 635 * system to make progress. That also prevents the following
@@ -581,83 +643,38 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
581 * set. So we'd effectivly block all timers until the T2 event 643 * set. So we'd effectivly block all timers until the T2 event
582 * fires. 644 * fires.
583 */ 645 */
584 if (cpu_base->hang_detected) 646 if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
585 return; 647 return;
586 648
587 tick_program_event(cpu_base->expires_next, 1); 649 tick_program_event(cpu_base->expires_next, 1);
588} 650}
589 651
652/* High resolution timer related functions */
653#ifdef CONFIG_HIGH_RES_TIMERS
654
590/* 655/*
591 * When a timer is enqueued and expires earlier than the already enqueued 656 * High resolution timer enabled ?
592 * timers, we have to check, whether it expires earlier than the timer for
593 * which the clock event device was armed.
594 *
595 * Called with interrupts disabled and base->cpu_base.lock held
596 */ 657 */
597static void hrtimer_reprogram(struct hrtimer *timer, 658static bool hrtimer_hres_enabled __read_mostly = true;
598 struct hrtimer_clock_base *base) 659unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
599{ 660EXPORT_SYMBOL_GPL(hrtimer_resolution);
600 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
601 ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
602
603 WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
604
605 /*
606 * If the timer is not on the current cpu, we cannot reprogram
607 * the other cpus clock event device.
608 */
609 if (base->cpu_base != cpu_base)
610 return;
611
612 /*
613 * If the hrtimer interrupt is running, then it will
614 * reevaluate the clock bases and reprogram the clock event
615 * device. The callbacks are always executed in hard interrupt
616 * context so we don't need an extra check for a running
617 * callback.
618 */
619 if (cpu_base->in_hrtirq)
620 return;
621
622 /*
623 * CLOCK_REALTIME timer might be requested with an absolute
624 * expiry time which is less than base->offset. Set it to 0.
625 */
626 if (expires < 0)
627 expires = 0;
628
629 if (expires >= cpu_base->expires_next)
630 return;
631
632 /* Update the pointer to the next expiring timer */
633 cpu_base->next_timer = timer;
634
635 /*
636 * If a hang was detected in the last timer interrupt then we
637 * do not schedule a timer which is earlier than the expiry
638 * which we enforced in the hang detection. We want the system
639 * to make progress.
640 */
641 if (cpu_base->hang_detected)
642 return;
643 661
644 /* 662/*
645 * Program the timer hardware. We enforce the expiry for 663 * Enable / Disable high resolution mode
646 * events which are already in the past. 664 */
647 */ 665static int __init setup_hrtimer_hres(char *str)
648 cpu_base->expires_next = expires; 666{
649 tick_program_event(expires, 1); 667 return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
650} 668}
651 669
670__setup("highres=", setup_hrtimer_hres);
671
652/* 672/*
653 * Initialize the high resolution related parts of cpu_base 673 * hrtimer_high_res_enabled - query, if the highres mode is enabled
654 */ 674 */
655static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) 675static inline int hrtimer_is_hres_enabled(void)
656{ 676{
657 base->expires_next = KTIME_MAX; 677 return hrtimer_hres_enabled;
658 base->hang_detected = 0;
659 base->hres_active = 0;
660 base->next_timer = NULL;
661} 678}
662 679
663/* 680/*
@@ -669,7 +686,7 @@ static void retrigger_next_event(void *arg)
669{ 686{
670 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases); 687 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
671 688
672 if (!base->hres_active) 689 if (!__hrtimer_hres_active(base))
673 return; 690 return;
674 691
675 raw_spin_lock(&base->lock); 692 raw_spin_lock(&base->lock);
@@ -716,23 +733,102 @@ void clock_was_set_delayed(void)
716 733
717#else 734#else
718 735
719static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *b) { return 0; }
720static inline int hrtimer_hres_active(void) { return 0; }
721static inline int hrtimer_is_hres_enabled(void) { return 0; } 736static inline int hrtimer_is_hres_enabled(void) { return 0; }
722static inline void hrtimer_switch_to_hres(void) { } 737static inline void hrtimer_switch_to_hres(void) { }
723static inline void
724hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }
725static inline int hrtimer_reprogram(struct hrtimer *timer,
726 struct hrtimer_clock_base *base)
727{
728 return 0;
729}
730static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
731static inline void retrigger_next_event(void *arg) { } 738static inline void retrigger_next_event(void *arg) { }
732 739
733#endif /* CONFIG_HIGH_RES_TIMERS */ 740#endif /* CONFIG_HIGH_RES_TIMERS */
734 741
735/* 742/*
743 * When a timer is enqueued and expires earlier than the already enqueued
744 * timers, we have to check, whether it expires earlier than the timer for
745 * which the clock event device was armed.
746 *
747 * Called with interrupts disabled and base->cpu_base.lock held
748 */
749static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
750{
751 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
752 struct hrtimer_clock_base *base = timer->base;
753 ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
754
755 WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
756
757 /*
758 * CLOCK_REALTIME timer might be requested with an absolute
759 * expiry time which is less than base->offset. Set it to 0.
760 */
761 if (expires < 0)
762 expires = 0;
763
764 if (timer->is_soft) {
765 /*
766 * soft hrtimer could be started on a remote CPU. In this
767 * case softirq_expires_next needs to be updated on the
768 * remote CPU. The soft hrtimer will not expire before the
769 * first hard hrtimer on the remote CPU -
770 * hrtimer_check_target() prevents this case.
771 */
772 struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
773
774 if (timer_cpu_base->softirq_activated)
775 return;
776
777 if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
778 return;
779
780 timer_cpu_base->softirq_next_timer = timer;
781 timer_cpu_base->softirq_expires_next = expires;
782
783 if (!ktime_before(expires, timer_cpu_base->expires_next) ||
784 !reprogram)
785 return;
786 }
787
788 /*
789 * If the timer is not on the current cpu, we cannot reprogram
790 * the other cpus clock event device.
791 */
792 if (base->cpu_base != cpu_base)
793 return;
794
795 /*
796 * If the hrtimer interrupt is running, then it will
797 * reevaluate the clock bases and reprogram the clock event
798 * device. The callbacks are always executed in hard interrupt
799 * context so we don't need an extra check for a running
800 * callback.
801 */
802 if (cpu_base->in_hrtirq)
803 return;
804
805 if (expires >= cpu_base->expires_next)
806 return;
807
808 /* Update the pointer to the next expiring timer */
809 cpu_base->next_timer = timer;
810 cpu_base->expires_next = expires;
811
812 /*
813 * If hres is not active, hardware does not have to be
814 * programmed yet.
815 *
816 * If a hang was detected in the last timer interrupt then we
817 * do not schedule a timer which is earlier than the expiry
818 * which we enforced in the hang detection. We want the system
819 * to make progress.
820 */
821 if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
822 return;
823
824 /*
825 * Program the timer hardware. We enforce the expiry for
826 * events which are already in the past.
827 */
828 tick_program_event(expires, 1);
829}
830
831/*
736 * Clock realtime was set 832 * Clock realtime was set
737 * 833 *
738 * Change the offset of the realtime clock vs. the monotonic 834 * Change the offset of the realtime clock vs. the monotonic
@@ -837,9 +933,10 @@ EXPORT_SYMBOL_GPL(hrtimer_forward);
837 * Returns 1 when the new timer is the leftmost timer in the tree. 933 * Returns 1 when the new timer is the leftmost timer in the tree.
838 */ 934 */
839static int enqueue_hrtimer(struct hrtimer *timer, 935static int enqueue_hrtimer(struct hrtimer *timer,
840 struct hrtimer_clock_base *base) 936 struct hrtimer_clock_base *base,
937 enum hrtimer_mode mode)
841{ 938{
842 debug_activate(timer); 939 debug_activate(timer, mode);
843 940
844 base->cpu_base->active_bases |= 1 << base->index; 941 base->cpu_base->active_bases |= 1 << base->index;
845 942
@@ -872,7 +969,6 @@ static void __remove_hrtimer(struct hrtimer *timer,
872 if (!timerqueue_del(&base->active, &timer->node)) 969 if (!timerqueue_del(&base->active, &timer->node))
873 cpu_base->active_bases &= ~(1 << base->index); 970 cpu_base->active_bases &= ~(1 << base->index);
874 971
875#ifdef CONFIG_HIGH_RES_TIMERS
876 /* 972 /*
877 * Note: If reprogram is false we do not update 973 * Note: If reprogram is false we do not update
878 * cpu_base->next_timer. This happens when we remove the first 974 * cpu_base->next_timer. This happens when we remove the first
@@ -883,7 +979,6 @@ static void __remove_hrtimer(struct hrtimer *timer,
883 */ 979 */
884 if (reprogram && timer == cpu_base->next_timer) 980 if (reprogram && timer == cpu_base->next_timer)
885 hrtimer_force_reprogram(cpu_base, 1); 981 hrtimer_force_reprogram(cpu_base, 1);
886#endif
887} 982}
888 983
889/* 984/*
@@ -932,22 +1027,36 @@ static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
932 return tim; 1027 return tim;
933} 1028}
934 1029
935/** 1030static void
936 * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU 1031hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
937 * @timer: the timer to be added
938 * @tim: expiry time
939 * @delta_ns: "slack" range for the timer
940 * @mode: expiry mode: absolute (HRTIMER_MODE_ABS) or
941 * relative (HRTIMER_MODE_REL)
942 */
943void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
944 u64 delta_ns, const enum hrtimer_mode mode)
945{ 1032{
946 struct hrtimer_clock_base *base, *new_base; 1033 ktime_t expires;
947 unsigned long flags;
948 int leftmost;
949 1034
950 base = lock_hrtimer_base(timer, &flags); 1035 /*
1036 * Find the next SOFT expiration.
1037 */
1038 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
1039
1040 /*
1041 * reprogramming needs to be triggered, even if the next soft
1042 * hrtimer expires at the same time than the next hard
1043 * hrtimer. cpu_base->softirq_expires_next needs to be updated!
1044 */
1045 if (expires == KTIME_MAX)
1046 return;
1047
1048 /*
1049 * cpu_base->*next_timer is recomputed by __hrtimer_get_next_event()
1050 * cpu_base->*expires_next is only set by hrtimer_reprogram()
1051 */
1052 hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
1053}
1054
1055static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1056 u64 delta_ns, const enum hrtimer_mode mode,
1057 struct hrtimer_clock_base *base)
1058{
1059 struct hrtimer_clock_base *new_base;
951 1060
952 /* Remove an active timer from the queue: */ 1061 /* Remove an active timer from the queue: */
953 remove_hrtimer(timer, base, true); 1062 remove_hrtimer(timer, base, true);
@@ -962,21 +1071,35 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
962 /* Switch the timer base, if necessary: */ 1071 /* Switch the timer base, if necessary: */
963 new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); 1072 new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
964 1073
965 leftmost = enqueue_hrtimer(timer, new_base); 1074 return enqueue_hrtimer(timer, new_base, mode);
966 if (!leftmost) 1075}
967 goto unlock; 1076
1077/**
1078 * hrtimer_start_range_ns - (re)start an hrtimer
1079 * @timer: the timer to be added
1080 * @tim: expiry time
1081 * @delta_ns: "slack" range for the timer
1082 * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
1083 * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
1084 * softirq based mode is considered for debug purpose only!
1085 */
1086void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1087 u64 delta_ns, const enum hrtimer_mode mode)
1088{
1089 struct hrtimer_clock_base *base;
1090 unsigned long flags;
1091
1092 /*
1093 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
1094 * match.
1095 */
1096 WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1097
1098 base = lock_hrtimer_base(timer, &flags);
1099
1100 if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
1101 hrtimer_reprogram(timer, true);
968 1102
969 if (!hrtimer_is_hres_active(timer)) {
970 /*
971 * Kick to reschedule the next tick to handle the new timer
972 * on dynticks target.
973 */
974 if (new_base->cpu_base->nohz_active)
975 wake_up_nohz_cpu(new_base->cpu_base->cpu);
976 } else {
977 hrtimer_reprogram(timer, new_base);
978 }
979unlock:
980 unlock_hrtimer_base(timer, &flags); 1103 unlock_hrtimer_base(timer, &flags);
981} 1104}
982EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); 1105EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
@@ -1074,7 +1197,7 @@ u64 hrtimer_get_next_event(void)
1074 raw_spin_lock_irqsave(&cpu_base->lock, flags); 1197 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1075 1198
1076 if (!__hrtimer_hres_active(cpu_base)) 1199 if (!__hrtimer_hres_active(cpu_base))
1077 expires = __hrtimer_get_next_event(cpu_base); 1200 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1078 1201
1079 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); 1202 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1080 1203
@@ -1097,17 +1220,24 @@ static inline int hrtimer_clockid_to_base(clockid_t clock_id)
1097static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, 1220static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1098 enum hrtimer_mode mode) 1221 enum hrtimer_mode mode)
1099{ 1222{
1223 bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
1224 int base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
1100 struct hrtimer_cpu_base *cpu_base; 1225 struct hrtimer_cpu_base *cpu_base;
1101 int base;
1102 1226
1103 memset(timer, 0, sizeof(struct hrtimer)); 1227 memset(timer, 0, sizeof(struct hrtimer));
1104 1228
1105 cpu_base = raw_cpu_ptr(&hrtimer_bases); 1229 cpu_base = raw_cpu_ptr(&hrtimer_bases);
1106 1230
1107 if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS) 1231 /*
1232 * POSIX magic: Relative CLOCK_REALTIME timers are not affected by
1233 * clock modifications, so they needs to become CLOCK_MONOTONIC to
1234 * ensure POSIX compliance.
1235 */
1236 if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
1108 clock_id = CLOCK_MONOTONIC; 1237 clock_id = CLOCK_MONOTONIC;
1109 1238
1110 base = hrtimer_clockid_to_base(clock_id); 1239 base += hrtimer_clockid_to_base(clock_id);
1240 timer->is_soft = softtimer;
1111 timer->base = &cpu_base->clock_base[base]; 1241 timer->base = &cpu_base->clock_base[base];
1112 timerqueue_init(&timer->node); 1242 timerqueue_init(&timer->node);
1113} 1243}
@@ -1116,7 +1246,13 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1116 * hrtimer_init - initialize a timer to the given clock 1246 * hrtimer_init - initialize a timer to the given clock
1117 * @timer: the timer to be initialized 1247 * @timer: the timer to be initialized
1118 * @clock_id: the clock to be used 1248 * @clock_id: the clock to be used
1119 * @mode: timer mode abs/rel 1249 * @mode: The modes which are relevant for intitialization:
1250 * HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
1251 * HRTIMER_MODE_REL_SOFT
1252 *
1253 * The PINNED variants of the above can be handed in,
1254 * but the PINNED bit is ignored as pinning happens
1255 * when the hrtimer is started
1120 */ 1256 */
1121void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, 1257void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1122 enum hrtimer_mode mode) 1258 enum hrtimer_mode mode)
@@ -1135,19 +1271,19 @@ EXPORT_SYMBOL_GPL(hrtimer_init);
1135 */ 1271 */
1136bool hrtimer_active(const struct hrtimer *timer) 1272bool hrtimer_active(const struct hrtimer *timer)
1137{ 1273{
1138 struct hrtimer_cpu_base *cpu_base; 1274 struct hrtimer_clock_base *base;
1139 unsigned int seq; 1275 unsigned int seq;
1140 1276
1141 do { 1277 do {
1142 cpu_base = READ_ONCE(timer->base->cpu_base); 1278 base = READ_ONCE(timer->base);
1143 seq = raw_read_seqcount_begin(&cpu_base->seq); 1279 seq = raw_read_seqcount_begin(&base->seq);
1144 1280
1145 if (timer->state != HRTIMER_STATE_INACTIVE || 1281 if (timer->state != HRTIMER_STATE_INACTIVE ||
1146 cpu_base->running == timer) 1282 base->running == timer)
1147 return true; 1283 return true;
1148 1284
1149 } while (read_seqcount_retry(&cpu_base->seq, seq) || 1285 } while (read_seqcount_retry(&base->seq, seq) ||
1150 cpu_base != READ_ONCE(timer->base->cpu_base)); 1286 base != READ_ONCE(timer->base));
1151 1287
1152 return false; 1288 return false;
1153} 1289}
@@ -1173,7 +1309,8 @@ EXPORT_SYMBOL_GPL(hrtimer_active);
1173 1309
1174static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, 1310static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1175 struct hrtimer_clock_base *base, 1311 struct hrtimer_clock_base *base,
1176 struct hrtimer *timer, ktime_t *now) 1312 struct hrtimer *timer, ktime_t *now,
1313 unsigned long flags)
1177{ 1314{
1178 enum hrtimer_restart (*fn)(struct hrtimer *); 1315 enum hrtimer_restart (*fn)(struct hrtimer *);
1179 int restart; 1316 int restart;
@@ -1181,16 +1318,16 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1181 lockdep_assert_held(&cpu_base->lock); 1318 lockdep_assert_held(&cpu_base->lock);
1182 1319
1183 debug_deactivate(timer); 1320 debug_deactivate(timer);
1184 cpu_base->running = timer; 1321 base->running = timer;
1185 1322
1186 /* 1323 /*
1187 * Separate the ->running assignment from the ->state assignment. 1324 * Separate the ->running assignment from the ->state assignment.
1188 * 1325 *
1189 * As with a regular write barrier, this ensures the read side in 1326 * As with a regular write barrier, this ensures the read side in
1190 * hrtimer_active() cannot observe cpu_base->running == NULL && 1327 * hrtimer_active() cannot observe base->running == NULL &&
1191 * timer->state == INACTIVE. 1328 * timer->state == INACTIVE.
1192 */ 1329 */
1193 raw_write_seqcount_barrier(&cpu_base->seq); 1330 raw_write_seqcount_barrier(&base->seq);
1194 1331
1195 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0); 1332 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
1196 fn = timer->function; 1333 fn = timer->function;
@@ -1204,15 +1341,15 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1204 timer->is_rel = false; 1341 timer->is_rel = false;
1205 1342
1206 /* 1343 /*
1207 * Because we run timers from hardirq context, there is no chance 1344 * The timer is marked as running in the CPU base, so it is
1208 * they get migrated to another cpu, therefore its safe to unlock 1345 * protected against migration to a different CPU even if the lock
1209 * the timer base. 1346 * is dropped.
1210 */ 1347 */
1211 raw_spin_unlock(&cpu_base->lock); 1348 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1212 trace_hrtimer_expire_entry(timer, now); 1349 trace_hrtimer_expire_entry(timer, now);
1213 restart = fn(timer); 1350 restart = fn(timer);
1214 trace_hrtimer_expire_exit(timer); 1351 trace_hrtimer_expire_exit(timer);
1215 raw_spin_lock(&cpu_base->lock); 1352 raw_spin_lock_irq(&cpu_base->lock);
1216 1353
1217 /* 1354 /*
1218 * Note: We clear the running state after enqueue_hrtimer and 1355 * Note: We clear the running state after enqueue_hrtimer and
@@ -1225,33 +1362,31 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1225 */ 1362 */
1226 if (restart != HRTIMER_NORESTART && 1363 if (restart != HRTIMER_NORESTART &&
1227 !(timer->state & HRTIMER_STATE_ENQUEUED)) 1364 !(timer->state & HRTIMER_STATE_ENQUEUED))
1228 enqueue_hrtimer(timer, base); 1365 enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
1229 1366
1230 /* 1367 /*
1231 * Separate the ->running assignment from the ->state assignment. 1368 * Separate the ->running assignment from the ->state assignment.
1232 * 1369 *
1233 * As with a regular write barrier, this ensures the read side in 1370 * As with a regular write barrier, this ensures the read side in
1234 * hrtimer_active() cannot observe cpu_base->running == NULL && 1371 * hrtimer_active() cannot observe base->running.timer == NULL &&
1235 * timer->state == INACTIVE. 1372 * timer->state == INACTIVE.
1236 */ 1373 */
1237 raw_write_seqcount_barrier(&cpu_base->seq); 1374 raw_write_seqcount_barrier(&base->seq);
1238 1375
1239 WARN_ON_ONCE(cpu_base->running != timer); 1376 WARN_ON_ONCE(base->running != timer);
1240 cpu_base->running = NULL; 1377 base->running = NULL;
1241} 1378}
1242 1379
1243static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now) 1380static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
1381 unsigned long flags, unsigned int active_mask)
1244{ 1382{
1245 struct hrtimer_clock_base *base = cpu_base->clock_base; 1383 struct hrtimer_clock_base *base;
1246 unsigned int active = cpu_base->active_bases; 1384 unsigned int active = cpu_base->active_bases & active_mask;
1247 1385
1248 for (; active; base++, active >>= 1) { 1386 for_each_active_base(base, cpu_base, active) {
1249 struct timerqueue_node *node; 1387 struct timerqueue_node *node;
1250 ktime_t basenow; 1388 ktime_t basenow;
1251 1389
1252 if (!(active & 0x01))
1253 continue;
1254
1255 basenow = ktime_add(now, base->offset); 1390 basenow = ktime_add(now, base->offset);
1256 1391
1257 while ((node = timerqueue_getnext(&base->active))) { 1392 while ((node = timerqueue_getnext(&base->active))) {
@@ -1274,11 +1409,28 @@ static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
1274 if (basenow < hrtimer_get_softexpires_tv64(timer)) 1409 if (basenow < hrtimer_get_softexpires_tv64(timer))
1275 break; 1410 break;
1276 1411
1277 __run_hrtimer(cpu_base, base, timer, &basenow); 1412 __run_hrtimer(cpu_base, base, timer, &basenow, flags);
1278 } 1413 }
1279 } 1414 }
1280} 1415}
1281 1416
1417static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
1418{
1419 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1420 unsigned long flags;
1421 ktime_t now;
1422
1423 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1424
1425 now = hrtimer_update_base(cpu_base);
1426 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
1427
1428 cpu_base->softirq_activated = 0;
1429 hrtimer_update_softirq_timer(cpu_base, true);
1430
1431 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1432}
1433
1282#ifdef CONFIG_HIGH_RES_TIMERS 1434#ifdef CONFIG_HIGH_RES_TIMERS
1283 1435
1284/* 1436/*
@@ -1289,13 +1441,14 @@ void hrtimer_interrupt(struct clock_event_device *dev)
1289{ 1441{
1290 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1442 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1291 ktime_t expires_next, now, entry_time, delta; 1443 ktime_t expires_next, now, entry_time, delta;
1444 unsigned long flags;
1292 int retries = 0; 1445 int retries = 0;
1293 1446
1294 BUG_ON(!cpu_base->hres_active); 1447 BUG_ON(!cpu_base->hres_active);
1295 cpu_base->nr_events++; 1448 cpu_base->nr_events++;
1296 dev->next_event = KTIME_MAX; 1449 dev->next_event = KTIME_MAX;
1297 1450
1298 raw_spin_lock(&cpu_base->lock); 1451 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1299 entry_time = now = hrtimer_update_base(cpu_base); 1452 entry_time = now = hrtimer_update_base(cpu_base);
1300retry: 1453retry:
1301 cpu_base->in_hrtirq = 1; 1454 cpu_base->in_hrtirq = 1;
@@ -1308,17 +1461,23 @@ retry:
1308 */ 1461 */
1309 cpu_base->expires_next = KTIME_MAX; 1462 cpu_base->expires_next = KTIME_MAX;
1310 1463
1311 __hrtimer_run_queues(cpu_base, now); 1464 if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1465 cpu_base->softirq_expires_next = KTIME_MAX;
1466 cpu_base->softirq_activated = 1;
1467 raise_softirq_irqoff(HRTIMER_SOFTIRQ);
1468 }
1469
1470 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1312 1471
1313 /* Reevaluate the clock bases for the next expiry */ 1472 /* Reevaluate the clock bases for the next expiry */
1314 expires_next = __hrtimer_get_next_event(cpu_base); 1473 expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1315 /* 1474 /*
1316 * Store the new expiry value so the migration code can verify 1475 * Store the new expiry value so the migration code can verify
1317 * against it. 1476 * against it.
1318 */ 1477 */
1319 cpu_base->expires_next = expires_next; 1478 cpu_base->expires_next = expires_next;
1320 cpu_base->in_hrtirq = 0; 1479 cpu_base->in_hrtirq = 0;
1321 raw_spin_unlock(&cpu_base->lock); 1480 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1322 1481
1323 /* Reprogramming necessary ? */ 1482 /* Reprogramming necessary ? */
1324 if (!tick_program_event(expires_next, 0)) { 1483 if (!tick_program_event(expires_next, 0)) {
@@ -1339,7 +1498,7 @@ retry:
1339 * Acquire base lock for updating the offsets and retrieving 1498 * Acquire base lock for updating the offsets and retrieving
1340 * the current time. 1499 * the current time.
1341 */ 1500 */
1342 raw_spin_lock(&cpu_base->lock); 1501 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1343 now = hrtimer_update_base(cpu_base); 1502 now = hrtimer_update_base(cpu_base);
1344 cpu_base->nr_retries++; 1503 cpu_base->nr_retries++;
1345 if (++retries < 3) 1504 if (++retries < 3)
@@ -1352,7 +1511,8 @@ retry:
1352 */ 1511 */
1353 cpu_base->nr_hangs++; 1512 cpu_base->nr_hangs++;
1354 cpu_base->hang_detected = 1; 1513 cpu_base->hang_detected = 1;
1355 raw_spin_unlock(&cpu_base->lock); 1514 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1515
1356 delta = ktime_sub(now, entry_time); 1516 delta = ktime_sub(now, entry_time);
1357 if ((unsigned int)delta > cpu_base->max_hang_time) 1517 if ((unsigned int)delta > cpu_base->max_hang_time)
1358 cpu_base->max_hang_time = (unsigned int) delta; 1518 cpu_base->max_hang_time = (unsigned int) delta;
@@ -1394,6 +1554,7 @@ static inline void __hrtimer_peek_ahead_timers(void) { }
1394void hrtimer_run_queues(void) 1554void hrtimer_run_queues(void)
1395{ 1555{
1396 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); 1556 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1557 unsigned long flags;
1397 ktime_t now; 1558 ktime_t now;
1398 1559
1399 if (__hrtimer_hres_active(cpu_base)) 1560 if (__hrtimer_hres_active(cpu_base))
@@ -1411,10 +1572,17 @@ void hrtimer_run_queues(void)
1411 return; 1572 return;
1412 } 1573 }
1413 1574
1414 raw_spin_lock(&cpu_base->lock); 1575 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1415 now = hrtimer_update_base(cpu_base); 1576 now = hrtimer_update_base(cpu_base);
1416 __hrtimer_run_queues(cpu_base, now); 1577
1417 raw_spin_unlock(&cpu_base->lock); 1578 if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1579 cpu_base->softirq_expires_next = KTIME_MAX;
1580 cpu_base->softirq_activated = 1;
1581 raise_softirq_irqoff(HRTIMER_SOFTIRQ);
1582 }
1583
1584 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1585 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1418} 1586}
1419 1587
1420/* 1588/*
@@ -1591,9 +1759,14 @@ int hrtimers_prepare_cpu(unsigned int cpu)
1591 timerqueue_init_head(&cpu_base->clock_base[i].active); 1759 timerqueue_init_head(&cpu_base->clock_base[i].active);
1592 } 1760 }
1593 1761
1594 cpu_base->active_bases = 0;
1595 cpu_base->cpu = cpu; 1762 cpu_base->cpu = cpu;
1596 hrtimer_init_hres(cpu_base); 1763 cpu_base->active_bases = 0;
1764 cpu_base->hres_active = 0;
1765 cpu_base->hang_detected = 0;
1766 cpu_base->next_timer = NULL;
1767 cpu_base->softirq_next_timer = NULL;
1768 cpu_base->expires_next = KTIME_MAX;
1769 cpu_base->softirq_expires_next = KTIME_MAX;
1597 return 0; 1770 return 0;
1598} 1771}
1599 1772
@@ -1625,7 +1798,7 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1625 * sort out already expired timers and reprogram the 1798 * sort out already expired timers and reprogram the
1626 * event device. 1799 * event device.
1627 */ 1800 */
1628 enqueue_hrtimer(timer, new_base); 1801 enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
1629 } 1802 }
1630} 1803}
1631 1804
@@ -1637,6 +1810,12 @@ int hrtimers_dead_cpu(unsigned int scpu)
1637 BUG_ON(cpu_online(scpu)); 1810 BUG_ON(cpu_online(scpu));
1638 tick_cancel_sched_timer(scpu); 1811 tick_cancel_sched_timer(scpu);
1639 1812
1813 /*
1814 * this BH disable ensures that raise_softirq_irqoff() does
1815 * not wakeup ksoftirqd (and acquire the pi-lock) while
1816 * holding the cpu_base lock
1817 */
1818 local_bh_disable();
1640 local_irq_disable(); 1819 local_irq_disable();
1641 old_base = &per_cpu(hrtimer_bases, scpu); 1820 old_base = &per_cpu(hrtimer_bases, scpu);
1642 new_base = this_cpu_ptr(&hrtimer_bases); 1821 new_base = this_cpu_ptr(&hrtimer_bases);
@@ -1652,12 +1831,19 @@ int hrtimers_dead_cpu(unsigned int scpu)
1652 &new_base->clock_base[i]); 1831 &new_base->clock_base[i]);
1653 } 1832 }
1654 1833
1834 /*
1835 * The migration might have changed the first expiring softirq
1836 * timer on this CPU. Update it.
1837 */
1838 hrtimer_update_softirq_timer(new_base, false);
1839
1655 raw_spin_unlock(&old_base->lock); 1840 raw_spin_unlock(&old_base->lock);
1656 raw_spin_unlock(&new_base->lock); 1841 raw_spin_unlock(&new_base->lock);
1657 1842
1658 /* Check, if we got expired work to do */ 1843 /* Check, if we got expired work to do */
1659 __hrtimer_peek_ahead_timers(); 1844 __hrtimer_peek_ahead_timers();
1660 local_irq_enable(); 1845 local_irq_enable();
1846 local_bh_enable();
1661 return 0; 1847 return 0;
1662} 1848}
1663 1849
@@ -1666,18 +1852,19 @@ int hrtimers_dead_cpu(unsigned int scpu)
1666void __init hrtimers_init(void) 1852void __init hrtimers_init(void)
1667{ 1853{
1668 hrtimers_prepare_cpu(smp_processor_id()); 1854 hrtimers_prepare_cpu(smp_processor_id());
1855 open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
1669} 1856}
1670 1857
1671/** 1858/**
1672 * schedule_hrtimeout_range_clock - sleep until timeout 1859 * schedule_hrtimeout_range_clock - sleep until timeout
1673 * @expires: timeout value (ktime_t) 1860 * @expires: timeout value (ktime_t)
1674 * @delta: slack in expires timeout (ktime_t) 1861 * @delta: slack in expires timeout (ktime_t)
1675 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL 1862 * @mode: timer mode
1676 * @clock: timer clock, CLOCK_MONOTONIC or CLOCK_REALTIME 1863 * @clock_id: timer clock to be used
1677 */ 1864 */
1678int __sched 1865int __sched
1679schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, 1866schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
1680 const enum hrtimer_mode mode, int clock) 1867 const enum hrtimer_mode mode, clockid_t clock_id)
1681{ 1868{
1682 struct hrtimer_sleeper t; 1869 struct hrtimer_sleeper t;
1683 1870
@@ -1698,7 +1885,7 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
1698 return -EINTR; 1885 return -EINTR;
1699 } 1886 }
1700 1887
1701 hrtimer_init_on_stack(&t.timer, clock, mode); 1888 hrtimer_init_on_stack(&t.timer, clock_id, mode);
1702 hrtimer_set_expires_range_ns(&t.timer, *expires, delta); 1889 hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
1703 1890
1704 hrtimer_init_sleeper(&t, current); 1891 hrtimer_init_sleeper(&t, current);
@@ -1720,7 +1907,7 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
1720 * schedule_hrtimeout_range - sleep until timeout 1907 * schedule_hrtimeout_range - sleep until timeout
1721 * @expires: timeout value (ktime_t) 1908 * @expires: timeout value (ktime_t)
1722 * @delta: slack in expires timeout (ktime_t) 1909 * @delta: slack in expires timeout (ktime_t)
1723 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL 1910 * @mode: timer mode
1724 * 1911 *
1725 * Make the current task sleep until the given expiry time has 1912 * Make the current task sleep until the given expiry time has
1726 * elapsed. The routine will return immediately unless 1913 * elapsed. The routine will return immediately unless
@@ -1759,7 +1946,7 @@ EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
1759/** 1946/**
1760 * schedule_hrtimeout - sleep until timeout 1947 * schedule_hrtimeout - sleep until timeout
1761 * @expires: timeout value (ktime_t) 1948 * @expires: timeout value (ktime_t)
1762 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL 1949 * @mode: timer mode
1763 * 1950 *
1764 * Make the current task sleep until the given expiry time has 1951 * Make the current task sleep until the given expiry time has
1765 * elapsed. The routine will return immediately unless 1952 * elapsed. The routine will return immediately unless
diff --git a/kernel/time/posix-clock.c b/kernel/time/posix-clock.c
index 17cdc554c9fe..cc91d90abd84 100644
--- a/kernel/time/posix-clock.c
+++ b/kernel/time/posix-clock.c
@@ -216,7 +216,7 @@ struct posix_clock_desc {
216 216
217static int get_clock_desc(const clockid_t id, struct posix_clock_desc *cd) 217static int get_clock_desc(const clockid_t id, struct posix_clock_desc *cd)
218{ 218{
219 struct file *fp = fget(CLOCKID_TO_FD(id)); 219 struct file *fp = fget(clockid_to_fd(id));
220 int err = -EINVAL; 220 int err = -EINVAL;
221 221
222 if (!fp) 222 if (!fp)
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 1f27887aa194..ec9f5da6f163 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -1189,9 +1189,8 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx,
1189 u64 now; 1189 u64 now;
1190 1190
1191 WARN_ON_ONCE(clock_idx == CPUCLOCK_SCHED); 1191 WARN_ON_ONCE(clock_idx == CPUCLOCK_SCHED);
1192 cpu_timer_sample_group(clock_idx, tsk, &now);
1193 1192
1194 if (oldval) { 1193 if (oldval && cpu_timer_sample_group(clock_idx, tsk, &now) != -EINVAL) {
1195 /* 1194 /*
1196 * We are setting itimer. The *oldval is absolute and we update 1195 * We are setting itimer. The *oldval is absolute and we update
1197 * it to be relative, *newval argument is relative and we update 1196 * it to be relative, *newval argument is relative and we update
@@ -1363,8 +1362,8 @@ static long posix_cpu_nsleep_restart(struct restart_block *restart_block)
1363 return do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t); 1362 return do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t);
1364} 1363}
1365 1364
1366#define PROCESS_CLOCK MAKE_PROCESS_CPUCLOCK(0, CPUCLOCK_SCHED) 1365#define PROCESS_CLOCK make_process_cpuclock(0, CPUCLOCK_SCHED)
1367#define THREAD_CLOCK MAKE_THREAD_CPUCLOCK(0, CPUCLOCK_SCHED) 1366#define THREAD_CLOCK make_thread_cpuclock(0, CPUCLOCK_SCHED)
1368 1367
1369static int process_cpu_clock_getres(const clockid_t which_clock, 1368static int process_cpu_clock_getres(const clockid_t which_clock,
1370 struct timespec64 *tp) 1369 struct timespec64 *tp)
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index f8e1845aa464..e277284c2831 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -150,16 +150,15 @@ static inline void tick_nohz_init(void) { }
150 150
151#ifdef CONFIG_NO_HZ_COMMON 151#ifdef CONFIG_NO_HZ_COMMON
152extern unsigned long tick_nohz_active; 152extern unsigned long tick_nohz_active;
153#else 153extern void timers_update_nohz(void);
154# ifdef CONFIG_SMP
155extern struct static_key_false timers_migration_enabled;
156# endif
157#else /* CONFIG_NO_HZ_COMMON */
158static inline void timers_update_nohz(void) { }
154#define tick_nohz_active (0) 159#define tick_nohz_active (0)
155#endif 160#endif
156 161
157#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
158extern void timers_update_migration(bool update_nohz);
159#else
160static inline void timers_update_migration(bool update_nohz) { }
161#endif
162
163DECLARE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases); 162DECLARE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases);
164 163
165extern u64 get_next_timer_interrupt(unsigned long basej, u64 basem); 164extern u64 get_next_timer_interrupt(unsigned long basej, u64 basem);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index f7cc7abfcf25..29a5733eff83 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -1107,7 +1107,7 @@ static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
1107 ts->nohz_mode = mode; 1107 ts->nohz_mode = mode;
1108 /* One update is enough */ 1108 /* One update is enough */
1109 if (!test_and_set_bit(0, &tick_nohz_active)) 1109 if (!test_and_set_bit(0, &tick_nohz_active))
1110 timers_update_migration(true); 1110 timers_update_nohz();
1111} 1111}
1112 1112
1113/** 1113/**
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 0bcf00e3ce48..48150ab42de9 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -200,8 +200,6 @@ struct timer_base {
200 unsigned long clk; 200 unsigned long clk;
201 unsigned long next_expiry; 201 unsigned long next_expiry;
202 unsigned int cpu; 202 unsigned int cpu;
203 bool migration_enabled;
204 bool nohz_active;
205 bool is_idle; 203 bool is_idle;
206 bool must_forward_clk; 204 bool must_forward_clk;
207 DECLARE_BITMAP(pending_map, WHEEL_SIZE); 205 DECLARE_BITMAP(pending_map, WHEEL_SIZE);
@@ -210,45 +208,64 @@ struct timer_base {
210 208
211static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]); 209static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);
212 210
213#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) 211#ifdef CONFIG_NO_HZ_COMMON
212
213static DEFINE_STATIC_KEY_FALSE(timers_nohz_active);
214static DEFINE_MUTEX(timer_keys_mutex);
215
216static void timer_update_keys(struct work_struct *work);
217static DECLARE_WORK(timer_update_work, timer_update_keys);
218
219#ifdef CONFIG_SMP
214unsigned int sysctl_timer_migration = 1; 220unsigned int sysctl_timer_migration = 1;
215 221
216void timers_update_migration(bool update_nohz) 222DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);
223
224static void timers_update_migration(void)
217{ 225{
218 bool on = sysctl_timer_migration && tick_nohz_active; 226 if (sysctl_timer_migration && tick_nohz_active)
219 unsigned int cpu; 227 static_branch_enable(&timers_migration_enabled);
228 else
229 static_branch_disable(&timers_migration_enabled);
230}
231#else
232static inline void timers_update_migration(void) { }
233#endif /* !CONFIG_SMP */
220 234
221 /* Avoid the loop, if nothing to update */ 235static void timer_update_keys(struct work_struct *work)
222 if (this_cpu_read(timer_bases[BASE_STD].migration_enabled) == on) 236{
223 return; 237 mutex_lock(&timer_keys_mutex);
238 timers_update_migration();
239 static_branch_enable(&timers_nohz_active);
240 mutex_unlock(&timer_keys_mutex);
241}
224 242
225 for_each_possible_cpu(cpu) { 243void timers_update_nohz(void)
226 per_cpu(timer_bases[BASE_STD].migration_enabled, cpu) = on; 244{
227 per_cpu(timer_bases[BASE_DEF].migration_enabled, cpu) = on; 245 schedule_work(&timer_update_work);
228 per_cpu(hrtimer_bases.migration_enabled, cpu) = on;
229 if (!update_nohz)
230 continue;
231 per_cpu(timer_bases[BASE_STD].nohz_active, cpu) = true;
232 per_cpu(timer_bases[BASE_DEF].nohz_active, cpu) = true;
233 per_cpu(hrtimer_bases.nohz_active, cpu) = true;
234 }
235} 246}
236 247
237int timer_migration_handler(struct ctl_table *table, int write, 248int timer_migration_handler(struct ctl_table *table, int write,
238 void __user *buffer, size_t *lenp, 249 void __user *buffer, size_t *lenp,
239 loff_t *ppos) 250 loff_t *ppos)
240{ 251{
241 static DEFINE_MUTEX(mutex);
242 int ret; 252 int ret;
243 253
244 mutex_lock(&mutex); 254 mutex_lock(&timer_keys_mutex);
245 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 255 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
246 if (!ret && write) 256 if (!ret && write)
247 timers_update_migration(false); 257 timers_update_migration();
248 mutex_unlock(&mutex); 258 mutex_unlock(&timer_keys_mutex);
249 return ret; 259 return ret;
250} 260}
251#endif 261
262static inline bool is_timers_nohz_active(void)
263{
264 return static_branch_unlikely(&timers_nohz_active);
265}
266#else
267static inline bool is_timers_nohz_active(void) { return false; }
268#endif /* NO_HZ_COMMON */
252 269
253static unsigned long round_jiffies_common(unsigned long j, int cpu, 270static unsigned long round_jiffies_common(unsigned long j, int cpu,
254 bool force_up) 271 bool force_up)
@@ -534,7 +551,7 @@ __internal_add_timer(struct timer_base *base, struct timer_list *timer)
534static void 551static void
535trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer) 552trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)
536{ 553{
537 if (!IS_ENABLED(CONFIG_NO_HZ_COMMON) || !base->nohz_active) 554 if (!is_timers_nohz_active())
538 return; 555 return;
539 556
540 /* 557 /*
@@ -849,21 +866,20 @@ static inline struct timer_base *get_timer_base(u32 tflags)
849 return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK); 866 return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);
850} 867}
851 868
852#ifdef CONFIG_NO_HZ_COMMON
853static inline struct timer_base * 869static inline struct timer_base *
854get_target_base(struct timer_base *base, unsigned tflags) 870get_target_base(struct timer_base *base, unsigned tflags)
855{ 871{
856#ifdef CONFIG_SMP 872#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
857 if ((tflags & TIMER_PINNED) || !base->migration_enabled) 873 if (static_branch_likely(&timers_migration_enabled) &&
858 return get_timer_this_cpu_base(tflags); 874 !(tflags & TIMER_PINNED))
859 return get_timer_cpu_base(tflags, get_nohz_timer_target()); 875 return get_timer_cpu_base(tflags, get_nohz_timer_target());
860#else
861 return get_timer_this_cpu_base(tflags);
862#endif 876#endif
877 return get_timer_this_cpu_base(tflags);
863} 878}
864 879
865static inline void forward_timer_base(struct timer_base *base) 880static inline void forward_timer_base(struct timer_base *base)
866{ 881{
882#ifdef CONFIG_NO_HZ_COMMON
867 unsigned long jnow; 883 unsigned long jnow;
868 884
869 /* 885 /*
@@ -887,16 +903,8 @@ static inline void forward_timer_base(struct timer_base *base)
887 base->clk = jnow; 903 base->clk = jnow;
888 else 904 else
889 base->clk = base->next_expiry; 905 base->clk = base->next_expiry;
890}
891#else
892static inline struct timer_base *
893get_target_base(struct timer_base *base, unsigned tflags)
894{
895 return get_timer_this_cpu_base(tflags);
896}
897
898static inline void forward_timer_base(struct timer_base *base) { }
899#endif 906#endif
907}
900 908
901 909
902/* 910/*
diff --git a/sound/drivers/dummy.c b/sound/drivers/dummy.c
index 69db45bc0197..8fb9a54fe8ba 100644
--- a/sound/drivers/dummy.c
+++ b/sound/drivers/dummy.c
@@ -375,17 +375,9 @@ struct dummy_hrtimer_pcm {
375 ktime_t period_time; 375 ktime_t period_time;
376 atomic_t running; 376 atomic_t running;
377 struct hrtimer timer; 377 struct hrtimer timer;
378 struct tasklet_struct tasklet;
379 struct snd_pcm_substream *substream; 378 struct snd_pcm_substream *substream;
380}; 379};
381 380
382static void dummy_hrtimer_pcm_elapsed(unsigned long priv)
383{
384 struct dummy_hrtimer_pcm *dpcm = (struct dummy_hrtimer_pcm *)priv;
385 if (atomic_read(&dpcm->running))
386 snd_pcm_period_elapsed(dpcm->substream);
387}
388
389static enum hrtimer_restart dummy_hrtimer_callback(struct hrtimer *timer) 381static enum hrtimer_restart dummy_hrtimer_callback(struct hrtimer *timer)
390{ 382{
391 struct dummy_hrtimer_pcm *dpcm; 383 struct dummy_hrtimer_pcm *dpcm;
@@ -393,7 +385,14 @@ static enum hrtimer_restart dummy_hrtimer_callback(struct hrtimer *timer)
393 dpcm = container_of(timer, struct dummy_hrtimer_pcm, timer); 385 dpcm = container_of(timer, struct dummy_hrtimer_pcm, timer);
394 if (!atomic_read(&dpcm->running)) 386 if (!atomic_read(&dpcm->running))
395 return HRTIMER_NORESTART; 387 return HRTIMER_NORESTART;
396 tasklet_schedule(&dpcm->tasklet); 388 /*
389 * In cases of XRUN and draining, this calls .trigger to stop PCM
390 * substream.
391 */
392 snd_pcm_period_elapsed(dpcm->substream);
393 if (!atomic_read(&dpcm->running))
394 return HRTIMER_NORESTART;
395
397 hrtimer_forward_now(timer, dpcm->period_time); 396 hrtimer_forward_now(timer, dpcm->period_time);
398 return HRTIMER_RESTART; 397 return HRTIMER_RESTART;
399} 398}
@@ -403,7 +402,7 @@ static int dummy_hrtimer_start(struct snd_pcm_substream *substream)
403 struct dummy_hrtimer_pcm *dpcm = substream->runtime->private_data; 402 struct dummy_hrtimer_pcm *dpcm = substream->runtime->private_data;
404 403
405 dpcm->base_time = hrtimer_cb_get_time(&dpcm->timer); 404 dpcm->base_time = hrtimer_cb_get_time(&dpcm->timer);
406 hrtimer_start(&dpcm->timer, dpcm->period_time, HRTIMER_MODE_REL); 405 hrtimer_start(&dpcm->timer, dpcm->period_time, HRTIMER_MODE_REL_SOFT);
407 atomic_set(&dpcm->running, 1); 406 atomic_set(&dpcm->running, 1);
408 return 0; 407 return 0;
409} 408}
@@ -413,14 +412,14 @@ static int dummy_hrtimer_stop(struct snd_pcm_substream *substream)
413 struct dummy_hrtimer_pcm *dpcm = substream->runtime->private_data; 412 struct dummy_hrtimer_pcm *dpcm = substream->runtime->private_data;
414 413
415 atomic_set(&dpcm->running, 0); 414 atomic_set(&dpcm->running, 0);
416 hrtimer_cancel(&dpcm->timer); 415 if (!hrtimer_callback_running(&dpcm->timer))
416 hrtimer_cancel(&dpcm->timer);
417 return 0; 417 return 0;
418} 418}
419 419
420static inline void dummy_hrtimer_sync(struct dummy_hrtimer_pcm *dpcm) 420static inline void dummy_hrtimer_sync(struct dummy_hrtimer_pcm *dpcm)
421{ 421{
422 hrtimer_cancel(&dpcm->timer); 422 hrtimer_cancel(&dpcm->timer);
423 tasklet_kill(&dpcm->tasklet);
424} 423}
425 424
426static snd_pcm_uframes_t 425static snd_pcm_uframes_t
@@ -465,12 +464,10 @@ static int dummy_hrtimer_create(struct snd_pcm_substream *substream)
465 if (!dpcm) 464 if (!dpcm)
466 return -ENOMEM; 465 return -ENOMEM;
467 substream->runtime->private_data = dpcm; 466 substream->runtime->private_data = dpcm;
468 hrtimer_init(&dpcm->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 467 hrtimer_init(&dpcm->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
469 dpcm->timer.function = dummy_hrtimer_callback; 468 dpcm->timer.function = dummy_hrtimer_callback;
470 dpcm->substream = substream; 469 dpcm->substream = substream;
471 atomic_set(&dpcm->running, 0); 470 atomic_set(&dpcm->running, 0);
472 tasklet_init(&dpcm->tasklet, dummy_hrtimer_pcm_elapsed,
473 (unsigned long)dpcm);
474 return 0; 471 return 0;
475} 472}
476 473
diff --git a/tools/testing/selftests/ptp/testptp.c b/tools/testing/selftests/ptp/testptp.c
index 5d2eae16f7ee..a5d8f0ab0da0 100644
--- a/tools/testing/selftests/ptp/testptp.c
+++ b/tools/testing/selftests/ptp/testptp.c
@@ -60,9 +60,7 @@ static int clock_adjtime(clockid_t id, struct timex *tx)
60static clockid_t get_clockid(int fd) 60static clockid_t get_clockid(int fd)
61{ 61{
62#define CLOCKFD 3 62#define CLOCKFD 3
63#define FD_TO_CLOCKID(fd) ((~(clockid_t) (fd) << 3) | CLOCKFD) 63 return (((unsigned int) ~fd) << 3) | CLOCKFD;
64
65 return FD_TO_CLOCKID(fd);
66} 64}
67 65
68static void handle_alarm(int s) 66static void handle_alarm(int s)