@@ -155,7 +155,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
* Returns the index of the idle state.
*/
int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
- int latency_req, int next_event)
+ struct cpuidle_times *times)
{
if (off || !initialized)
return -ENODEV;
@@ -166,7 +166,7 @@ int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
if (unlikely(use_deepest_state))
return cpuidle_find_deepest_state(drv, dev);
- return cpuidle_curr_governor->select(drv, dev, latency_req, next_event);
+ return cpuidle_curr_governor->select(drv, dev, times);
}
/**
@@ -65,10 +65,11 @@ static inline void ladder_do_selection(struct ladder_device *ldev,
*/
static int ladder_select_state(struct cpuidle_driver *drv,
struct cpuidle_device *dev,
- int latency_req, int next_event)
+ struct cpuidle_times *times)
{
struct ladder_device *ldev = &__get_cpu_var(ladder_devices);
struct ladder_device_state *last_state;
+ int latency_req = times->latency_req;
int last_residency, last_idx = ldev->last_state_idx;
last_state = &ldev->states[last_idx];
@@ -284,9 +284,10 @@ again:
* @dev: the CPU
*/
static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
- int latency_req, int next_event)
+ struct cpuidle_times *times)
{
struct menu_device *data = &__get_cpu_var(menu_devices);
+ int latency_req = times->latency_req;
int i;
unsigned int interactivity_req;
unsigned long nr_iowaiters, cpu_load;
@@ -299,7 +300,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
data->last_state_idx = CPUIDLE_DRIVER_STATE_START - 1;
/* determine the expected residency time, round up */
- data->next_timer_us = next_event;
+ data->next_timer_us = times->next_timer_event;
get_iowait_load(&nr_iowaiters, &cpu_load);
data->bucket = which_bucket(data->next_timer_us, nr_iowaiters);
@@ -83,6 +83,12 @@ struct cpuidle_device {
#endif
};
+struct cpuidle_times {
+ unsigned int latency_req;
+ unsigned int next_timer_event;
+ unsigned int next_io_event;
+};
+
DECLARE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
DECLARE_PER_CPU(struct cpuidle_device, cpuidle_dev);
@@ -123,7 +129,7 @@ extern void disable_cpuidle(void);
extern int cpuidle_select(struct cpuidle_driver *drv,
struct cpuidle_device *dev,
- int latency_req, int next_event);
+ struct cpuidle_times *times);
extern int cpuidle_enter(struct cpuidle_driver *drv,
struct cpuidle_device *dev, int index);
extern void cpuidle_reflect(struct cpuidle_device *dev, int index);
@@ -152,7 +158,7 @@ extern struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev)
static inline void disable_cpuidle(void) { }
static inline int cpuidle_select(struct cpuidle_driver *drv,
struct cpuidle_device *dev,
- int latency_req, int next_event)
+ struct cpuidle_times *times)
{return -ENODEV; }
static inline int cpuidle_enter(struct cpuidle_driver *drv,
struct cpuidle_device *dev, int index)
@@ -208,7 +214,7 @@ struct cpuidle_governor {
int (*select) (struct cpuidle_driver *drv,
struct cpuidle_device *dev,
- int latency_req, int next_event);
+ struct cpuidle_times *times);
void (*reflect) (struct cpuidle_device *dev, int index);
struct module *owner;
@@ -4,7 +4,7 @@
#include <linux/sched.h>
#include <linux/cpu.h>
#include <linux/cpuidle.h>
-#include <linux/tick.h>
+#include <linux/ktime.h>
#include <linux/pm_qos.h>
#include <linux/mm.h>
#include <linux/stackprotector.h>
@@ -14,6 +14,7 @@
#include <trace/events/power.h>
#include "sched.h"
+#include "io_latency.h"
static int __read_mostly cpu_idle_force_poll;
@@ -79,9 +80,9 @@ static void cpuidle_idle_call(void)
{
struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
- struct timespec t;
- int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
- unsigned int broadcast;
+ struct cpuidle_times times;
+ int next_state, entered_state;
+ bool broadcast;
/*
* Check if the idle task must be rescheduled. If it is the
@@ -105,25 +106,29 @@ static void cpuidle_idle_call(void)
*/
rcu_idle_enter();
+ times.latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
/*
* The latency requirement does not allow any latency, jump to
- * the default idle function
+ * the default idle function without entering the cpuidle code
*/
- if (latency_req == 0)
+ if (times.latency_req == 0)
goto use_default;
- t = ktime_to_timespec(tick_nohz_get_sleep_length());
+ /*
+ * Retrieve the next timer event
+ */
+ times.next_timer_event = ktime_to_us(tick_nohz_get_sleep_length());
- /*
- * The next timer event for this in us
+ /*
+ * Retrieve the next IO guessed event
*/
- next_event = t.tv_sec * USEC_PER_SEC + t.tv_nsec / NSEC_PER_USEC;
+ times.next_io_event = io_latency_get_sleep_length(this_rq());
/*
* Ask the cpuidle framework to choose a convenient idle state.
* Fall back to the default arch idle method on errors.
*/
- next_state = cpuidle_select(drv, dev, latency_req, next_event);
+ next_state = cpuidle_select(drv, dev, ×);
if (next_state < 0) {
use_default:
/*
As we want to improve the sleep duration estimation, the IO latency expected duration is passed to the cpuidle framework. The governors will have to deal with if they are interested in this information. Signed-off-by: Daniel Lezcano <daniel.lezcano@linaro.org> --- drivers/cpuidle/cpuidle.c | 4 ++-- drivers/cpuidle/governors/ladder.c | 3 ++- drivers/cpuidle/governors/menu.c | 5 +++-- include/linux/cpuidle.h | 12 +++++++++--- kernel/sched/idle.c | 27 ++++++++++++++++----------- 5 files changed, 32 insertions(+), 19 deletions(-)