@@ -232,6 +232,19 @@ AC_ARG_ENABLE([tracing-timers],
ODP_CFLAGS="$ODP_CFLAGS -DTRACING_TIMERS=$TRACING_TIMERS"
##########################################################################
+# Enable/disable ODP_POLLED_TIMERS
+##########################################################################
+ODP_POLLED_TIMERS=0
+AC_ARG_ENABLE([polled-timers],
+ [ --enable-polled-timers polled timers],
+ [if test "x$enableval" = "xyes"; then
+ ODP_POLLED_TIMERS=1
+ else
+ ODP_POLLED_TIMERS=0
+ fi])
+ODP_CFLAGS="$ODP_CFLAGS -DODP_POLLED_TIMERS=$ODP_POLLED_TIMERS"
+
+##########################################################################
# Enable/disable ODP_DEBUG_PRINT
##########################################################################
ODP_DEBUG_PRINT=0
@@ -118,6 +118,9 @@ extern "C" {
*/
#define CONFIG_BURST_SIZE 16
+/* Value used to rate limit timer pool expiration processing. */
+#define ODP_CONFIG_TIMER_RUN_NSEC (250)
+
#ifdef __cplusplus
}
#endif
@@ -24,4 +24,6 @@ static inline uint64_t core_tick(void)
#endif
}
+uint64_t core_tick_diff_ns(uint64_t before, uint64_t after);
+
#endif
@@ -39,4 +39,6 @@ typedef struct odp_timeout_hdr_stride {
uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_timeout_hdr_t))];
} odp_timeout_hdr_stride;
+int timer_run(void);
+
#endif
@@ -22,6 +22,8 @@
#include <odp_align_internal.h>
#include <odp_schedule_internal.h>
#include <odp_schedule_ordered_internal.h>
+#include <odp_time_internal.h>
+#include <odp_timer_internal.h>
#include <odp/api/sync.h>
/* Number of priority levels */
@@ -788,19 +790,31 @@ static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[],
return 0;
}
+#ifdef ODP_POLLED_TIMERS
+static __thread uint64_t last_timer_run;
+#endif
static int schedule_loop(odp_queue_t *out_queue, uint64_t wait,
odp_event_t out_ev[],
unsigned int max_num)
{
- odp_time_t next, wtime;
- int first = 1;
- int ret;
-
- while (1) {
- ret = do_schedule(out_queue, out_ev, max_num);
+ uint64_t start, now;
+ int nr_events;
+
+ start = core_tick();
+
+ for (;;) {
+#ifdef ODP_POLLED_TIMERS
+ now = core_tick();
+ if (ODP_CONFIG_TIMER_RUN_NSEC <=
+ core_tick_diff_ns(last_timer_run, now)) {
+ last_timer_run = now;
+ (void)timer_run();
+ }
+#endif
+ nr_events = do_schedule(out_queue, out_ev, max_num);
- if (ret)
+ if (nr_events)
break;
if (wait == ODP_SCHED_WAIT)
@@ -809,18 +823,12 @@ static int schedule_loop(odp_queue_t *out_queue, uint64_t wait,
if (wait == ODP_SCHED_NO_WAIT)
break;
- if (first) {
- wtime = odp_time_local_from_ns(wait);
- next = odp_time_sum(odp_time_local(), wtime);
- first = 0;
- continue;
- }
-
- if (odp_time_cmp(next, odp_time_local()) < 0)
+ now = core_tick();
+ if (core_tick_diff_ns(start, now) >= wait)
break;
}
- return ret;
+ return nr_events;
}
static odp_event_t schedule(odp_queue_t *out_queue, uint64_t wait)
@@ -6,13 +6,68 @@
#include <odp_posix_extensions.h>
-#include <time.h>
+#include <odp/api/cpu.h>
#include <odp/api/time.h>
#include <odp/api/hints.h>
#include <odp_debug_internal.h>
+#include <odp_time_internal.h>
+
+#include <math.h>
+#include <time.h>
+#include <unistd.h>
+
+/* Number of nanoseconds per second. */
+#define ODP_NSEC_PER_SEC (ODP_TIME_SEC_IN_NS)
+
+static uint64_t core_tick_hz;
static odp_time_t start_time;
+uint64_t core_tick_diff_ns(uint64_t before, uint64_t after)
+{
+ uint64_t diff = after - before;
+
+ //TODO: khz instead of hz, skip llround
+ return llround(((double)diff / (double)core_tick_hz) *
+ ODP_NSEC_PER_SEC);
+}
+
+static bool core_tick_init(void)
+{
+#if defined(__aarch64__)
+ uint64_t hz;
+ __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(hz));
+ core_tick_hz = hz;
+ return true;
+#elif defined(__x86_64__)
+ uint32_t cpu_info[4];
+
+ /* Check for Invariant TSC. */
+ __asm__ volatile("cpuid" :
+ "=a"(cpu_info[0]),
+ "=b"(cpu_info[1]),
+ "=c"(cpu_info[2]),
+ "=d"(cpu_info[3]) :
+ "a"(0x80000000));
+ if (cpu_info[0] >= 0x80000007) {
+ __asm__ volatile("cpuid" :
+ "=a"(cpu_info[0]),
+ "=b"(cpu_info[1]),
+ "=c"(cpu_info[2]),
+ "=d"(cpu_info[3]) :
+ "a"(0x80000007));
+ if (!(cpu_info[3] & (1 << 8)))
+ return false;
+ }
+ //TODO: VM cpuid
+
+ core_tick_hz = odp_cpu_hz();
+ //TODO: libcpufreq
+ return true;
+#endif
+ return false;
+}
+
static inline
uint64_t time_to_ns(odp_time_t time)
{
@@ -204,6 +259,9 @@ int odp_time_init_global(void)
start_time.tv_nsec = time.tv_nsec;
}
+ if (!core_tick_init())
+ ODP_ABORT("core_tick_init failed\n");
+
return ret;
}
@@ -166,6 +166,9 @@ static inline void set_next_free(odp_timer *tim, uint32_t nf)
typedef struct odp_timer_pool_s {
/* Put frequently accessed fields in the first cache line */
+#ifdef ODP_POLLED_TIMERS
+ uint64_t cur_ts;
+#endif
odp_atomic_u64_t cur_tick;/* Current tick value */
uint64_t min_rel_tck;
uint64_t max_rel_tck;
@@ -179,11 +182,13 @@ typedef struct odp_timer_pool_s {
odp_timer_pool_param_t param;
char name[ODP_TIMER_POOL_NAME_LEN];
odp_shm_t shm;
+#ifndef ODP_POLLED_TIMERS
timer_t timerid;
int notify_overrun;
pthread_t timer_thread; /* pthread_t of timer thread */
pid_t timer_thread_id; /* gettid() for timer thread */
int timer_thread_exit; /* request to exit for timer thread */
+#endif
} odp_timer_pool;
#define MAX_TIMER_POOLS 255 /* Leave one for ODP_TIMER_INVALID */
@@ -219,9 +224,11 @@ static inline odp_timer_t tp_idx_to_handle(struct odp_timer_pool_s *tp,
return _odp_cast_scalar(odp_timer_t, (tp->tp_idx << INDEX_BITS) | idx);
}
+#ifndef ODP_POLLED_TIMERS
/* Forward declarations */
static void itimer_init(odp_timer_pool *tp);
static void itimer_fini(odp_timer_pool *tp);
+#endif
static odp_timer_pool_t odp_timer_pool_new(const char *_name,
const odp_timer_pool_param_t *param)
@@ -245,6 +252,9 @@ static odp_timer_pool_t odp_timer_pool_new(const char *_name,
ODP_ABORT("%s: timer pool shm-alloc(%zuKB) failed\n",
_name, (sz0 + sz1 + sz2) / 1024);
odp_timer_pool *tp = (odp_timer_pool *)odp_shm_addr(shm);
+#ifdef ODP_POLLED_TIMERS
+ __atomic_store_n(&tp->cur_ts, core_tick(), __ATOMIC_RELAXED);
+#endif
odp_atomic_init_u64(&tp->cur_tick, 0);
snprintf(tp->name, sizeof(tp->name), "%s", _name);
tp->shm = shm;
@@ -254,7 +264,9 @@ static odp_timer_pool_t odp_timer_pool_new(const char *_name,
tp->num_alloc = 0;
odp_atomic_init_u32(&tp->high_wm, 0);
tp->first_free = 0;
+#ifndef ODP_POLLED_TIMERS
tp->notify_overrun = 1;
+#endif
tp->tick_buf = (void *)((char *)odp_shm_addr(shm) + sz0);
tp->timers = (void *)((char *)odp_shm_addr(shm) + sz0 + sz1);
/* Initialize all odp_timer entries */
@@ -273,11 +285,14 @@ static odp_timer_pool_t odp_timer_pool_new(const char *_name,
tp->tp_idx = tp_idx;
odp_spinlock_init(&tp->lock);
timer_pool[tp_idx] = tp;
+#ifndef ODP_POLLED_TIMERS
if (tp->param.clk_src == ODP_CLOCK_CPU)
itimer_init(tp);
+#endif
return tp;
}
+#ifndef ODP_POLLED_TIMERS
static void block_sigalarm(void)
{
sigset_t sigset;
@@ -297,17 +312,20 @@ static void stop_timer_thread(odp_timer_pool *tp)
if (ret != 0)
ODP_ABORT("unable to join thread, err %d\n", ret);
}
+#endif
static void odp_timer_pool_del(odp_timer_pool *tp)
{
odp_spinlock_lock(&tp->lock);
timer_pool[tp->tp_idx] = NULL;
+#ifndef ODP_POLLED_TIMERS
/* Stop timer triggering */
if (tp->param.clk_src == ODP_CLOCK_CPU)
itimer_fini(tp);
stop_timer_thread(tp);
+#endif
if (tp->num_alloc != 0) {
/* It's a programming error to attempt to destroy a */
@@ -673,6 +691,44 @@ static unsigned odp_timer_pool_expire(odp_timer_pool_t tpid, uint64_t tick)
return nexp;
}
+#ifdef ODP_POLLED_TIMERS
+
+int timer_run(void)
+{
+ odp_timer_pool *tp;
+ size_t i;
+ int nexp = 0;
+
+ for (i = 0; i < MAX_TIMER_POOLS; i++) {
+ tp = timer_pool[i];
+
+ if (tp == NULL) break;
+
+ /*
+ * Check the last time this timer pool was expired. If one or
+ * more periods have passed, attempt to expire it.
+ */
+ uint64_t tp_ts =
+ __atomic_load_n(&tp->cur_ts, __ATOMIC_RELAXED);
+ uint64_t now = core_tick();
+
+ if (core_tick_diff_ns(tp_ts, now) >= odp_timer_pool_resolution(tp)) {
+ if (__atomic_compare_exchange_n(
+ &tp->cur_ts, &tp_ts, now, false,
+ __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
+ uint64_t tp_tick;
+ tp_tick = odp_atomic_fetch_inc_u64(
+ &tp->cur_tick);
+ nexp += odp_timer_pool_expire(tp, tp_tick + 1);
+ }
+ }
+ }
+
+ return nexp;
+}
+
+#else
+
/******************************************************************************
* POSIX timer support
* Functions that use Linux/POSIX per-process timers and related facilities
@@ -773,8 +829,8 @@ static void itimer_init(odp_timer_pool *tp)
nsec = res - sec * ODP_TIME_SEC_IN_NS;
memset(&ispec, 0, sizeof(ispec));
- ispec.it_interval.tv_sec = (time_t)sec;
- ispec.it_interval.tv_nsec = (long)nsec;
+ ispec.it_internal.tv_sec = (time_t)sec;
+ ispec.it_internal.tv_nsec = (long)nsec;
ispec.it_value.tv_sec = (time_t)sec;
ispec.it_value.tv_nsec = (long)nsec;
@@ -790,6 +846,8 @@ static void itimer_fini(odp_timer_pool *tp)
strerror(errno));
}
+#endif
+
/******************************************************************************
* Public API functions
* Some parameter checks and error messages
@@ -1013,8 +1071,9 @@ int odp_timer_init_global(void)
#endif
odp_atomic_init_u32(&num_timer_pools, 0);
+#ifndef ODP_POLLED_TIMERS
block_sigalarm();
-
+#endif
return 0;
}
--enable-polled-timers runs timer expiration processing directly inside odp_schedule() and avoids the use of timer pool background threads and itimers. odp_timers stress test shows improvements for test cases with higher resolution and lower latency timers. The most noticable improvement is in scenarios where the number of timer pools is greater than the number of control plane cores. For further explanation please see: https://docs.google.com/a/linaro.org/document/d/1AI0TFlIb3QJFAd3mJz74kPzMrLmiQgwmuOx31LlffZA/edit?usp=sharing Signed-off-by: Brian Brooks <brian.brooks@linaro.org> --- configure.ac | 13 +++++ .../linux-generic/include/odp_config_internal.h | 3 + platform/linux-generic/include/odp_time_internal.h | 2 + .../linux-generic/include/odp_timer_internal.h | 2 + platform/linux-generic/odp_schedule.c | 40 +++++++------ platform/linux-generic/odp_time.c | 60 +++++++++++++++++++- platform/linux-generic/odp_timer.c | 65 +++++++++++++++++++++- 7 files changed, 165 insertions(+), 20 deletions(-) -- 2.7.4