#include <linux/kernel_stat.h>

#include <linux/export.h>

#include <linux/interrupt.h>

#include <linux/percpu.h>

#include <linux/init.h>

#include <linux/mm.h>

#include <linux/swap.h>

#include <linux/pid_namespace.h>

#include <linux/notifier.h>

#include <linux/thread_info.h>

#include <linux/time.h>

#include <linux/jiffies.h>

#include <linux/posix-timers.h>

#include <linux/cpu.h>

#include <linux/syscalls.h>

#include <linux/delay.h>

#include <linux/tick.h>

#include <linux/kallsyms.h>

#include <linux/irq_work.h>

#include <linux/sched.h>

#include <linux/sched/sysctl.h>

#include <linux/slab.h>

#include <linux/compat.h>

#include <asm/uaccess.h>

#include <asm/unistd.h>

#include <asm/div64.h>

#include <asm/timex.h>

#include <asm/io.h>

#include "tick-internal.h"

#define CREATE_TRACE_POINTS

#include <trace/events/timer.h>

__visible u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;

EXPORT_SYMBOL(jiffies_64);

#define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)

#define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)

#define TVN_SIZE (1 << TVN_BITS)

#define TVR_SIZE (1 << TVR_BITS)

#define TVN_MASK (TVN_SIZE - 1)

#define TVR_MASK (TVR_SIZE - 1)

#define MAX_TVAL ((unsigned long)((1ULL << (TVR_BITS + 4*TVN_BITS)) - 1))

struct tvec {

struct hlist_head vec[TVN_SIZE];

};

struct tvec_root {

struct hlist_head vec[TVR_SIZE];

};

struct tvec_base {

} ____cacheline_aligned;

static DEFINE_PER_CPU(struct tvec_base, tvec_bases);

#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)

unsigned int sysctl_timer_migration = 1;

void timers_update_migration(bool update_nohz)

{

bool on = sysctl_timer_migration && tick_nohz_active;

unsigned int cpu;

/* Avoid the loop, if nothing to update */

if (this_cpu_read(tvec_bases.migration_enabled) == on)

return;

for_each_possible_cpu(cpu) {

per_cpu(tvec_bases.migration_enabled, cpu) = on;

per_cpu(hrtimer_bases.migration_enabled, cpu) = on;

if (!update_nohz)

continue;

per_cpu(tvec_bases.nohz_active, cpu) = true;

per_cpu(hrtimer_bases.nohz_active, cpu) = true;

}

}

int timer_migration_handler(struct ctl_table *table, int write,

void __user *buffer, size_t *lenp,

loff_t *ppos)

{

static DEFINE_MUTEX(mutex);

int ret;

mutex_lock(&mutex);

ret = proc_dointvec(table, write, buffer, lenp, ppos);

if (!ret && write)

timers_update_migration(false);

mutex_unlock(&mutex);

return ret;

}

static inline struct tvec_base *get_target_base(struct tvec_base *base,

int pinned)

{

if (pinned || !base->migration_enabled)

return this_cpu_ptr(&tvec_bases);

return per_cpu_ptr(&tvec_bases, get_nohz_timer_target());

}

static inline struct tvec_base *get_target_base(struct tvec_base *base,

int pinned)

{

return this_cpu_ptr(&tvec_bases);

}

static unsigned long round_jiffies_common(unsigned long j, int cpu,

bool force_up)

{

int rem;

unsigned long original = j;

/*

j += cpu * 3;

rem = j % HZ;

/*

if (rem < HZ/4 && !force_up) /* round down */

j = j - rem;

else /* round up */

j = j - rem + HZ;

/* now that we have rounded, subtract the extra skew again */

j -= cpu * 3;

/*

return time_is_after_jiffies(j) ? j : original;

}

unsigned long __round_jiffies(unsigned long j, int cpu)

{

return round_jiffies_common(j, cpu, false);

}

EXPORT_SYMBOL_GPL(__round_jiffies);

unsigned long __round_jiffies_relative(unsigned long j, int cpu)

{

unsigned long j0 = jiffies;

/* Use j0 because jiffies might change while we run */

return round_jiffies_common(j + j0, cpu, false) - j0;

}

EXPORT_SYMBOL_GPL(__round_jiffies_relative);

unsigned long round_jiffies(unsigned long j)

{

return round_jiffies_common(j, raw_smp_processor_id(), false);

}

EXPORT_SYMBOL_GPL(round_jiffies);

unsigned long round_jiffies_relative(unsigned long j)

{

return __round_jiffies_relative(j, raw_smp_processor_id());

}

EXPORT_SYMBOL_GPL(round_jiffies_relative);

unsigned long __round_jiffies_up(unsigned long j, int cpu)

{

return round_jiffies_common(j, cpu, true);

}

EXPORT_SYMBOL_GPL(__round_jiffies_up);

unsigned long __round_jiffies_up_relative(unsigned long j, int cpu)

{

unsigned long j0 = jiffies;

/* Use j0 because jiffies might change while we run */

return round_jiffies_common(j + j0, cpu, true) - j0;

}

EXPORT_SYMBOL_GPL(__round_jiffies_up_relative);

unsigned long round_jiffies_up(unsigned long j)

{

return round_jiffies_common(j, raw_smp_processor_id(), true);

}

EXPORT_SYMBOL_GPL(round_jiffies_up);

unsigned long round_jiffies_up_relative(unsigned long j)

{

return __round_jiffies_up_relative(j, raw_smp_processor_id());

}

EXPORT_SYMBOL_GPL(round_jiffies_up_relative);

void set_timer_slack(struct timer_list *timer, int slack_hz)

{

timer->slack = slack_hz;

}

EXPORT_SYMBOL_GPL(set_timer_slack);

static void

__internal_add_timer(struct tvec_base *base, struct timer_list *timer)

{

unsigned long expires = timer->expires;

unsigned long idx = expires - base->timer_jiffies;

struct hlist_head *vec;

if (idx < TVR_SIZE) {

int i = expires & TVR_MASK;

vec = base->tv1.vec + i;

} else if (idx < 1 << (TVR_BITS + TVN_BITS)) {

int i = (expires >> TVR_BITS) & TVN_MASK;

vec = base->tv2.vec + i;

} else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) {

int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;

vec = base->tv3.vec + i;

} else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) {

int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK;

vec = base->tv4.vec + i;

} else if ((signed long) idx < 0) {

/*

vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);

} else {

int i;

/* If the timeout is larger than MAX_TVAL (on 64-bit

if (idx > MAX_TVAL) {

idx = MAX_TVAL;

expires = idx + base->timer_jiffies;

}

i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;

vec = base->tv5.vec + i;

}

hlist_add_head(&timer->entry, vec);

}

static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)

{

/* Advance base->jiffies, if the base is empty */

if (!base->all_timers++)

base->timer_jiffies = jiffies;

__internal_add_timer(base, timer);

/*

if (!(timer->flags & TIMER_DEFERRABLE)) {

if (!base->active_timers++ ||

time_before(timer->expires, base->next_timer))

base->next_timer = timer->expires;

}

/*

if (base->nohz_active) {

if (!(timer->flags & TIMER_DEFERRABLE) ||

tick_nohz_full_cpu(base->cpu))

wake_up_nohz_cpu(base->cpu);

}

}

#ifdef CONFIG_TIMER_STATS

void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr)

{

if (timer->start_site)

return;

timer->start_site = addr;

memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);

timer->start_pid = current->pid;

}

static void timer_stats_account_timer(struct timer_list *timer)

{

void *site;

/*

site = READ_ONCE(timer->start_site);

if (likely(!site))

return;

timer_stats_update_stats(timer, timer->start_pid, site,

timer->function, timer->start_comm,

timer->flags);

}

static void timer_stats_account_timer(struct timer_list *timer) {}

#ifdef CONFIG_DEBUG_OBJECTS_TIMERS

static struct debug_obj_descr timer_debug_descr;

static void *timer_debug_hint(void *addr)

{

return ((struct timer_list *) addr)->function;

}

static int timer_fixup_init(void *addr, enum debug_obj_state state)

{

struct timer_list *timer = addr;

switch (state) {

case ODEBUG_STATE_ACTIVE:

del_timer_sync(timer);

debug_object_init(timer, &timer_debug_descr);

return 1;

default:

return 0;

}

}

static void stub_timer(unsigned long data)

{

WARN_ON(1);

}

static int timer_fixup_activate(void *addr, enum debug_obj_state state)

{

struct timer_list *timer = addr;

switch (state) {

case ODEBUG_STATE_NOTAVAILABLE:

/*

if (timer->entry.pprev == NULL &&

timer->entry.next == TIMER_ENTRY_STATIC) {

debug_object_init(timer, &timer_debug_descr);

debug_object_activate(timer, &timer_debug_descr);

return 0;

} else {

setup_timer(timer, stub_timer, 0);

return 1;

}

return 0;

case ODEBUG_STATE_ACTIVE:

WARN_ON(1);

default:

return 0;

}

}

static int timer_fixup_free(void *addr, enum debug_obj_state state)

{

struct timer_list *timer = addr;

switch (state) {

case ODEBUG_STATE_ACTIVE:

del_timer_sync(timer);

debug_object_free(timer, &timer_debug_descr);

return 1;

default:

return 0;

}

}

static int timer_fixup_assert_init(void *addr, enum debug_obj_state state)

{

struct timer_list *timer = addr;

switch (state) {

case ODEBUG_STATE_NOTAVAILABLE:

if (timer->entry.next == TIMER_ENTRY_STATIC) {

/*

debug_object_init(timer, &timer_debug_descr);

return 0;

} else {

setup_timer(timer, stub_timer, 0);

return 1;

}

default:

return 0;

}

}

static struct debug_obj_descr timer_debug_descr = {

.name = "timer_list",

.debug_hint = timer_debug_hint,

.fixup_init = timer_fixup_init,

.fixup_activate = timer_fixup_activate,

.fixup_free = timer_fixup_free,

.fixup_assert_init = timer_fixup_assert_init,

};

static inline void debug_timer_init(struct timer_list *timer)

{

debug_object_init(timer, &timer_debug_descr);

}

static inline void debug_timer_activate(struct timer_list *timer)

{

debug_object_activate(timer, &timer_debug_descr);

}

static inline void debug_timer_deactivate(struct timer_list *timer)

{

debug_object_deactivate(timer, &timer_debug_descr);

}

static inline void debug_timer_free(struct timer_list *timer)

{

debug_object_free(timer, &timer_debug_descr);

}

static inline void debug_timer_assert_init(struct timer_list *timer)

{

debug_object_assert_init(timer, &timer_debug_descr);

}

static void do_init_timer(struct timer_list *timer, unsigned int flags,

const char *name, struct lock_class_key *key);

void init_timer_on_stack_key(struct timer_list *timer, unsigned int flags,

const char *name, struct lock_class_key *key)

{

debug_object_init_on_stack(timer, &timer_debug_descr);

do_init_timer(timer, flags, name, key);

}

EXPORT_SYMBOL_GPL(init_timer_on_stack_key);

void destroy_timer_on_stack(struct timer_list *timer)

{

debug_object_free(timer, &timer_debug_descr);

}

EXPORT_SYMBOL_GPL(destroy_timer_on_stack);

static inline void debug_timer_init(struct timer_list *timer) { }

static inline void debug_timer_activate(struct timer_list *timer) { }

static inline void debug_timer_deactivate(struct timer_list *timer) { }

static inline void debug_timer_assert_init(struct timer_list *timer) { }

static inline void debug_init(struct timer_list *timer)

{

debug_timer_init(timer);

trace_timer_init(timer);

}

static inline void

debug_activate(struct timer_list *timer, unsigned long expires)

{

debug_timer_activate(timer);

trace_timer_start(timer, expires, timer->flags);

}

static inline void debug_deactivate(struct timer_list *timer)

{

debug_timer_deactivate(timer);

trace_timer_cancel(timer);

}

static inline void debug_assert_init(struct timer_list *timer)

{

debug_timer_assert_init(timer);

}

static void do_init_timer(struct timer_list *timer, unsigned int flags,

const char *name, struct lock_class_key *key)

{

timer->entry.pprev = NULL;

timer->flags = flags | raw_smp_processor_id();

timer->slack = -1;

#ifdef CONFIG_TIMER_STATS

timer->start_site = NULL;

timer->start_pid = -1;

memset(timer->start_comm, 0, TASK_COMM_LEN);

lockdep_init_map(&timer->lockdep_map, name, key, 0);

}

void init_timer_key(struct timer_list *timer, unsigned int flags,

const char *name, struct lock_class_key *key)

{

debug_init(timer);

do_init_timer(timer, flags, name, key);

}

EXPORT_SYMBOL(init_timer_key);

static inline void detach_timer(struct timer_list *timer, bool clear_pending)

{

struct hlist_node *entry = &timer->entry;

debug_deactivate(timer);

__hlist_del(entry);

if (clear_pending)

entry->pprev = NULL;

entry->next = LIST_POISON2;

}

static inline void

detach_expired_timer(struct timer_list *timer, struct tvec_base *base)

{

detach_timer(timer, true);

if (!(timer->flags & TIMER_DEFERRABLE))

base->active_timers--;

base->all_timers--;

}

static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,

bool clear_pending)

{

if (!timer_pending(timer))

return 0;

detach_timer(timer, clear_pending);

if (!(timer->flags & TIMER_DEFERRABLE)) {

base->active_timers--;

if (timer->expires == base->next_timer)

base->next_timer = base->timer_jiffies;

}

/* If this was the last timer, advance base->jiffies */

if (!--base->all_timers)

base->timer_jiffies = jiffies;

return 1;

}

static struct tvec_base *lock_timer_base(struct timer_list *timer,

unsigned long *flags)

__acquires(timer->base->lock)

{

for (;;) {

u32 tf = timer->flags;

struct tvec_base *base;

if (!(tf & TIMER_MIGRATING)) {

base = per_cpu_ptr(&tvec_bases, tf & TIMER_CPUMASK);

spin_lock_irqsave(&base->lock, *flags);

if (timer->flags == tf)

return base;

spin_unlock_irqrestore(&base->lock, *flags);

}

cpu_relax();

}

}

static inline int

__mod_timer(struct timer_list *timer, unsigned long expires,

bool pending_only, int pinned)

{

struct tvec_base *base, *new_base;

unsigned long flags;

int ret = 0;

timer_stats_timer_set_start_info(timer);

BUG_ON(!timer->function);

base = lock_timer_base(timer, &flags);

ret = detach_if_pending(timer, base, false);

if (!ret && pending_only)

goto out_unlock;

debug_activate(timer, expires);

new_base = get_target_base(base, pinned);

if (base != new_base) {

/*

if (likely(base->running_timer != timer)) {

/* See the comment in lock_timer_base() */

timer->flags |= TIMER_MIGRATING;

spin_unlock(&base->lock);

base = new_base;

spin_lock(&base->lock);

WRITE_ONCE(timer->flags,

(timer->flags & ~TIMER_BASEMASK) | base->cpu);

}

}

timer->expires = expires;

internal_add_timer(base, timer);

out_unlock:

spin_unlock_irqrestore(&base->lock, flags);

return ret;

}

int mod_timer_pending(struct timer_list *timer, unsigned long expires)

{

return __mod_timer(timer, expires, true, TIMER_NOT_PINNED);

}

EXPORT_SYMBOL(mod_timer_pending);

static inline

unsigned long apply_slack(struct timer_list *timer, unsigned long expires)

{

unsigned long expires_limit, mask;

int bit;

if (timer->slack >= 0) {

expires_limit = expires + timer->slack;

} else {

long delta = expires - jiffies;

if (delta < 256)

return expires;

expires_limit = expires + delta / 256;

}

mask = expires ^ expires_limit;

if (mask == 0)

return expires;

bit = __fls(mask);

mask = (1UL << bit) - 1;

expires_limit = expires_limit & ~(mask);

return expires_limit;

}

int mod_timer(struct timer_list *timer, unsigned long expires)

{

expires = apply_slack(timer, expires);

/*

if (timer_pending(timer) && timer->expires == expires)

return 1;

return __mod_timer(timer, expires, false, TIMER_NOT_PINNED);

}

EXPORT_SYMBOL(mod_timer);

int mod_timer_pinned(struct timer_list *timer, unsigned long expires)

{

if (timer->expires == expires && timer_pending(timer))

return 1;

return __mod_timer(timer, expires, false, TIMER_PINNED);

}

EXPORT_SYMBOL(mod_timer_pinned);

void add_timer(struct timer_list *timer)

{

BUG_ON(timer_pending(timer));

mod_timer(timer, timer->expires);

}

EXPORT_SYMBOL(add_timer);

void add_timer_on(struct timer_list *timer, int cpu)

{

struct tvec_base *new_base = per_cpu_ptr(&tvec_bases, cpu);

struct tvec_base *base;

unsigned long flags;

timer_stats_timer_set_start_info(timer);

BUG_ON(timer_pending(timer) || !timer->function);

/*

base = lock_timer_base(timer, &flags);

if (base != new_base) {

timer->flags |= TIMER_MIGRATING;

spin_unlock(&base->lock);

base = new_base;

spin_lock(&base->lock);

WRITE_ONCE(timer->flags,

(timer->flags & ~TIMER_BASEMASK) | cpu);