Showing error 1144

User: Jiri Slaby
Error type: Double Lock
Error type description: Some lock is locked twice unintentionally in a sequence
File location: kernel/sched.c
Line in file: 3903
Project: Linux Kernel
Project version: 2.6.28
Tools: Stanse (1.2)
Entered: 2012-04-29 14:49:11 UTC


Source:

   1/*
   2 *  kernel/sched.c
   3 *
   4 *  Kernel scheduler and related syscalls
   5 *
   6 *  Copyright (C) 1991-2002  Linus Torvalds
   7 *
   8 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
   9 *                make semaphores SMP safe
  10 *  1998-11-19        Implemented schedule_timeout() and related stuff
  11 *                by Andrea Arcangeli
  12 *  2002-01-04        New ultra-scalable O(1) scheduler by Ingo Molnar:
  13 *                hybrid priority-list and round-robin design with
  14 *                an array-switch method of distributing timeslices
  15 *                and per-CPU runqueues.  Cleanups and useful suggestions
  16 *                by Davide Libenzi, preemptible kernel bits by Robert Love.
  17 *  2003-09-03        Interactivity tuning by Con Kolivas.
  18 *  2004-04-02        Scheduler domains code by Nick Piggin
  19 *  2007-04-15  Work begun on replacing all interactivity tuning with a
  20 *              fair scheduling design by Con Kolivas.
  21 *  2007-05-05  Load balancing (smp-nice) and other improvements
  22 *              by Peter Williams
  23 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
  24 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
  25 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
  26 *              Thomas Gleixner, Mike Kravetz
  27 */
  28
  29#include <linux/mm.h>
  30#include <linux/module.h>
  31#include <linux/nmi.h>
  32#include <linux/init.h>
  33#include <linux/uaccess.h>
  34#include <linux/highmem.h>
  35#include <linux/smp_lock.h>
  36#include <asm/mmu_context.h>
  37#include <linux/interrupt.h>
  38#include <linux/capability.h>
  39#include <linux/completion.h>
  40#include <linux/kernel_stat.h>
  41#include <linux/debug_locks.h>
  42#include <linux/security.h>
  43#include <linux/notifier.h>
  44#include <linux/profile.h>
  45#include <linux/freezer.h>
  46#include <linux/vmalloc.h>
  47#include <linux/blkdev.h>
  48#include <linux/delay.h>
  49#include <linux/pid_namespace.h>
  50#include <linux/smp.h>
  51#include <linux/threads.h>
  52#include <linux/timer.h>
  53#include <linux/rcupdate.h>
  54#include <linux/cpu.h>
  55#include <linux/cpuset.h>
  56#include <linux/percpu.h>
  57#include <linux/kthread.h>
  58#include <linux/proc_fs.h>
  59#include <linux/seq_file.h>
  60#include <linux/sysctl.h>
  61#include <linux/syscalls.h>
  62#include <linux/times.h>
  63#include <linux/tsacct_kern.h>
  64#include <linux/kprobes.h>
  65#include <linux/delayacct.h>
  66#include <linux/reciprocal_div.h>
  67#include <linux/unistd.h>
  68#include <linux/pagemap.h>
  69#include <linux/hrtimer.h>
  70#include <linux/tick.h>
  71#include <linux/bootmem.h>
  72#include <linux/debugfs.h>
  73#include <linux/ctype.h>
  74#include <linux/ftrace.h>
  75#include <trace/sched.h>
  76
  77#include <asm/tlb.h>
  78#include <asm/irq_regs.h>
  79
  80#include "sched_cpupri.h"
  81
  82/*
  83 * Convert user-nice values [ -20 ... 0 ... 19 ]
  84 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
  85 * and back.
  86 */
  87#define NICE_TO_PRIO(nice)        (MAX_RT_PRIO + (nice) + 20)
  88#define PRIO_TO_NICE(prio)        ((prio) - MAX_RT_PRIO - 20)
  89#define TASK_NICE(p)                PRIO_TO_NICE((p)->static_prio)
  90
  91/*
  92 * 'User priority' is the nice value converted to something we
  93 * can work with better when scaling various scheduler parameters,
  94 * it's a [ 0 ... 39 ] range.
  95 */
  96#define USER_PRIO(p)                ((p)-MAX_RT_PRIO)
  97#define TASK_USER_PRIO(p)        USER_PRIO((p)->static_prio)
  98#define MAX_USER_PRIO                (USER_PRIO(MAX_PRIO))
  99
 100/*
 101 * Helpers for converting nanosecond timing to jiffy resolution
 102 */
 103#define NS_TO_JIFFIES(TIME)        ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
 104
 105#define NICE_0_LOAD                SCHED_LOAD_SCALE
 106#define NICE_0_SHIFT                SCHED_LOAD_SHIFT
 107
 108/*
 109 * These are the 'tuning knobs' of the scheduler:
 110 *
 111 * default timeslice is 100 msecs (used only for SCHED_RR tasks).
 112 * Timeslices get refilled after they expire.
 113 */
 114#define DEF_TIMESLICE                (100 * HZ / 1000)
 115
 116/*
 117 * single value that denotes runtime == period, ie unlimited time.
 118 */
 119#define RUNTIME_INF        ((u64)~0ULL)
 120
 121#ifdef CONFIG_SMP
 122/*
 123 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
 124 * Since cpu_power is a 'constant', we can use a reciprocal divide.
 125 */
 126static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
 127{
 128        return reciprocal_divide(load, sg->reciprocal_cpu_power);
 129}
 130
 131/*
 132 * Each time a sched group cpu_power is changed,
 133 * we must compute its reciprocal value
 134 */
 135static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
 136{
 137        sg->__cpu_power += val;
 138        sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
 139}
 140#endif
 141
 142static inline int rt_policy(int policy)
 143{
 144        if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
 145                return 1;
 146        return 0;
 147}
 148
 149static inline int task_has_rt_policy(struct task_struct *p)
 150{
 151        return rt_policy(p->policy);
 152}
 153
 154/*
 155 * This is the priority-queue data structure of the RT scheduling class:
 156 */
 157struct rt_prio_array {
 158        DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
 159        struct list_head queue[MAX_RT_PRIO];
 160};
 161
 162struct rt_bandwidth {
 163        /* nests inside the rq lock: */
 164        spinlock_t                rt_runtime_lock;
 165        ktime_t                        rt_period;
 166        u64                        rt_runtime;
 167        struct hrtimer                rt_period_timer;
 168};
 169
 170static struct rt_bandwidth def_rt_bandwidth;
 171
 172static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
 173
 174static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
 175{
 176        struct rt_bandwidth *rt_b =
 177                container_of(timer, struct rt_bandwidth, rt_period_timer);
 178        ktime_t now;
 179        int overrun;
 180        int idle = 0;
 181
 182        for (;;) {
 183                now = hrtimer_cb_get_time(timer);
 184                overrun = hrtimer_forward(timer, now, rt_b->rt_period);
 185
 186                if (!overrun)
 187                        break;
 188
 189                idle = do_sched_rt_period_timer(rt_b, overrun);
 190        }
 191
 192        return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
 193}
 194
 195static
 196void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
 197{
 198        rt_b->rt_period = ns_to_ktime(period);
 199        rt_b->rt_runtime = runtime;
 200
 201        spin_lock_init(&rt_b->rt_runtime_lock);
 202
 203        hrtimer_init(&rt_b->rt_period_timer,
 204                        CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 205        rt_b->rt_period_timer.function = sched_rt_period_timer;
 206        rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
 207}
 208
 209static inline int rt_bandwidth_enabled(void)
 210{
 211        return sysctl_sched_rt_runtime >= 0;
 212}
 213
 214static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
 215{
 216        ktime_t now;
 217
 218        if (rt_bandwidth_enabled() && rt_b->rt_runtime == RUNTIME_INF)
 219                return;
 220
 221        if (hrtimer_active(&rt_b->rt_period_timer))
 222                return;
 223
 224        spin_lock(&rt_b->rt_runtime_lock);
 225        for (;;) {
 226                if (hrtimer_active(&rt_b->rt_period_timer))
 227                        break;
 228
 229                now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
 230                hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
 231                hrtimer_start_expires(&rt_b->rt_period_timer,
 232                                HRTIMER_MODE_ABS);
 233        }
 234        spin_unlock(&rt_b->rt_runtime_lock);
 235}
 236
 237#ifdef CONFIG_RT_GROUP_SCHED
 238static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
 239{
 240        hrtimer_cancel(&rt_b->rt_period_timer);
 241}
 242#endif
 243
 244/*
 245 * sched_domains_mutex serializes calls to arch_init_sched_domains,
 246 * detach_destroy_domains and partition_sched_domains.
 247 */
 248static DEFINE_MUTEX(sched_domains_mutex);
 249
 250#ifdef CONFIG_GROUP_SCHED
 251
 252#include <linux/cgroup.h>
 253
 254struct cfs_rq;
 255
 256static LIST_HEAD(task_groups);
 257
 258/* task group related information */
 259struct task_group {
 260#ifdef CONFIG_CGROUP_SCHED
 261        struct cgroup_subsys_state css;
 262#endif
 263
 264#ifdef CONFIG_FAIR_GROUP_SCHED
 265        /* schedulable entities of this group on each cpu */
 266        struct sched_entity **se;
 267        /* runqueue "owned" by this group on each cpu */
 268        struct cfs_rq **cfs_rq;
 269        unsigned long shares;
 270#endif
 271
 272#ifdef CONFIG_RT_GROUP_SCHED
 273        struct sched_rt_entity **rt_se;
 274        struct rt_rq **rt_rq;
 275
 276        struct rt_bandwidth rt_bandwidth;
 277#endif
 278
 279        struct rcu_head rcu;
 280        struct list_head list;
 281
 282        struct task_group *parent;
 283        struct list_head siblings;
 284        struct list_head children;
 285};
 286
 287#ifdef CONFIG_USER_SCHED
 288
 289/*
 290 * Root task group.
 291 *         Every UID task group (including init_task_group aka UID-0) will
 292 *         be a child to this group.
 293 */
 294struct task_group root_task_group;
 295
 296#ifdef CONFIG_FAIR_GROUP_SCHED
 297/* Default task group's sched entity on each cpu */
 298static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
 299/* Default task group's cfs_rq on each cpu */
 300static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
 301#endif /* CONFIG_FAIR_GROUP_SCHED */
 302
 303#ifdef CONFIG_RT_GROUP_SCHED
 304static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
 305static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
 306#endif /* CONFIG_RT_GROUP_SCHED */
 307#else /* !CONFIG_USER_SCHED */
 308#define root_task_group init_task_group
 309#endif /* CONFIG_USER_SCHED */
 310
 311/* task_group_lock serializes add/remove of task groups and also changes to
 312 * a task group's cpu shares.
 313 */
 314static DEFINE_SPINLOCK(task_group_lock);
 315
 316#ifdef CONFIG_FAIR_GROUP_SCHED
 317#ifdef CONFIG_USER_SCHED
 318# define INIT_TASK_GROUP_LOAD        (2*NICE_0_LOAD)
 319#else /* !CONFIG_USER_SCHED */
 320# define INIT_TASK_GROUP_LOAD        NICE_0_LOAD
 321#endif /* CONFIG_USER_SCHED */
 322
 323/*
 324 * A weight of 0 or 1 can cause arithmetics problems.
 325 * A weight of a cfs_rq is the sum of weights of which entities
 326 * are queued on this cfs_rq, so a weight of a entity should not be
 327 * too large, so as the shares value of a task group.
 328 * (The default weight is 1024 - so there's no practical
 329 *  limitation from this.)
 330 */
 331#define MIN_SHARES        2
 332#define MAX_SHARES        (1UL << 18)
 333
 334static int init_task_group_load = INIT_TASK_GROUP_LOAD;
 335#endif
 336
 337/* Default task group.
 338 *        Every task in system belong to this group at bootup.
 339 */
 340struct task_group init_task_group;
 341
 342/* return group to which a task belongs */
 343static inline struct task_group *task_group(struct task_struct *p)
 344{
 345        struct task_group *tg;
 346
 347#ifdef CONFIG_USER_SCHED
 348        tg = p->user->tg;
 349#elif defined(CONFIG_CGROUP_SCHED)
 350        tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
 351                                struct task_group, css);
 352#else
 353        tg = &init_task_group;
 354#endif
 355        return tg;
 356}
 357
 358/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
 359static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 360{
 361#ifdef CONFIG_FAIR_GROUP_SCHED
 362        p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
 363        p->se.parent = task_group(p)->se[cpu];
 364#endif
 365
 366#ifdef CONFIG_RT_GROUP_SCHED
 367        p->rt.rt_rq  = task_group(p)->rt_rq[cpu];
 368        p->rt.parent = task_group(p)->rt_se[cpu];
 369#endif
 370}
 371
 372#else
 373
 374static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
 375static inline struct task_group *task_group(struct task_struct *p)
 376{
 377        return NULL;
 378}
 379
 380#endif        /* CONFIG_GROUP_SCHED */
 381
 382/* CFS-related fields in a runqueue */
 383struct cfs_rq {
 384        struct load_weight load;
 385        unsigned long nr_running;
 386
 387        u64 exec_clock;
 388        u64 min_vruntime;
 389
 390        struct rb_root tasks_timeline;
 391        struct rb_node *rb_leftmost;
 392
 393        struct list_head tasks;
 394        struct list_head *balance_iterator;
 395
 396        /*
 397         * 'curr' points to currently running entity on this cfs_rq.
 398         * It is set to NULL otherwise (i.e when none are currently running).
 399         */
 400        struct sched_entity *curr, *next, *last;
 401
 402        unsigned int nr_spread_over;
 403
 404#ifdef CONFIG_FAIR_GROUP_SCHED
 405        struct rq *rq;        /* cpu runqueue to which this cfs_rq is attached */
 406
 407        /*
 408         * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
 409         * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
 410         * (like users, containers etc.)
 411         *
 412         * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
 413         * list is used during load balance.
 414         */
 415        struct list_head leaf_cfs_rq_list;
 416        struct task_group *tg;        /* group that "owns" this runqueue */
 417
 418#ifdef CONFIG_SMP
 419        /*
 420         * the part of load.weight contributed by tasks
 421         */
 422        unsigned long task_weight;
 423
 424        /*
 425         *   h_load = weight * f(tg)
 426         *
 427         * Where f(tg) is the recursive weight fraction assigned to
 428         * this group.
 429         */
 430        unsigned long h_load;
 431
 432        /*
 433         * this cpu's part of tg->shares
 434         */
 435        unsigned long shares;
 436
 437        /*
 438         * load.weight at the time we set shares
 439         */
 440        unsigned long rq_weight;
 441#endif
 442#endif
 443};
 444
 445/* Real-Time classes' related field in a runqueue: */
 446struct rt_rq {
 447        struct rt_prio_array active;
 448        unsigned long rt_nr_running;
 449#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
 450        int highest_prio; /* highest queued rt task prio */
 451#endif
 452#ifdef CONFIG_SMP
 453        unsigned long rt_nr_migratory;
 454        int overloaded;
 455#endif
 456        int rt_throttled;
 457        u64 rt_time;
 458        u64 rt_runtime;
 459        /* Nests inside the rq lock: */
 460        spinlock_t rt_runtime_lock;
 461
 462#ifdef CONFIG_RT_GROUP_SCHED
 463        unsigned long rt_nr_boosted;
 464
 465        struct rq *rq;
 466        struct list_head leaf_rt_rq_list;
 467        struct task_group *tg;
 468        struct sched_rt_entity *rt_se;
 469#endif
 470};
 471
 472#ifdef CONFIG_SMP
 473
 474/*
 475 * We add the notion of a root-domain which will be used to define per-domain
 476 * variables. Each exclusive cpuset essentially defines an island domain by
 477 * fully partitioning the member cpus from any other cpuset. Whenever a new
 478 * exclusive cpuset is created, we also create and attach a new root-domain
 479 * object.
 480 *
 481 */
 482struct root_domain {
 483        atomic_t refcount;
 484        cpumask_t span;
 485        cpumask_t online;
 486
 487        /*
 488         * The "RT overload" flag: it gets set if a CPU has more than
 489         * one runnable RT task.
 490         */
 491        cpumask_t rto_mask;
 492        atomic_t rto_count;
 493#ifdef CONFIG_SMP
 494        struct cpupri cpupri;
 495#endif
 496};
 497
 498/*
 499 * By default the system creates a single root-domain with all cpus as
 500 * members (mimicking the global state we have today).
 501 */
 502static struct root_domain def_root_domain;
 503
 504#endif
 505
 506/*
 507 * This is the main, per-CPU runqueue data structure.
 508 *
 509 * Locking rule: those places that want to lock multiple runqueues
 510 * (such as the load balancing or the thread migration code), lock
 511 * acquire operations must be ordered by ascending &runqueue.
 512 */
 513struct rq {
 514        /* runqueue lock: */
 515        spinlock_t lock;
 516
 517        /*
 518         * nr_running and cpu_load should be in the same cacheline because
 519         * remote CPUs use both these fields when doing load calculation.
 520         */
 521        unsigned long nr_running;
 522        #define CPU_LOAD_IDX_MAX 5
 523        unsigned long cpu_load[CPU_LOAD_IDX_MAX];
 524        unsigned char idle_at_tick;
 525#ifdef CONFIG_NO_HZ
 526        unsigned long last_tick_seen;
 527        unsigned char in_nohz_recently;
 528#endif
 529        /* capture load from *all* tasks on this cpu: */
 530        struct load_weight load;
 531        unsigned long nr_load_updates;
 532        u64 nr_switches;
 533
 534        struct cfs_rq cfs;
 535        struct rt_rq rt;
 536
 537#ifdef CONFIG_FAIR_GROUP_SCHED
 538        /* list of leaf cfs_rq on this cpu: */
 539        struct list_head leaf_cfs_rq_list;
 540#endif
 541#ifdef CONFIG_RT_GROUP_SCHED
 542        struct list_head leaf_rt_rq_list;
 543#endif
 544
 545        /*
 546         * This is part of a global counter where only the total sum
 547         * over all CPUs matters. A task can increase this counter on
 548         * one CPU and if it got migrated afterwards it may decrease
 549         * it on another CPU. Always updated under the runqueue lock:
 550         */
 551        unsigned long nr_uninterruptible;
 552
 553        struct task_struct *curr, *idle;
 554        unsigned long next_balance;
 555        struct mm_struct *prev_mm;
 556
 557        u64 clock;
 558
 559        atomic_t nr_iowait;
 560
 561#ifdef CONFIG_SMP
 562        struct root_domain *rd;
 563        struct sched_domain *sd;
 564
 565        /* For active balancing */
 566        int active_balance;
 567        int push_cpu;
 568        /* cpu of this runqueue: */
 569        int cpu;
 570        int online;
 571
 572        unsigned long avg_load_per_task;
 573
 574        struct task_struct *migration_thread;
 575        struct list_head migration_queue;
 576#endif
 577
 578#ifdef CONFIG_SCHED_HRTICK
 579#ifdef CONFIG_SMP
 580        int hrtick_csd_pending;
 581        struct call_single_data hrtick_csd;
 582#endif
 583        struct hrtimer hrtick_timer;
 584#endif
 585
 586#ifdef CONFIG_SCHEDSTATS
 587        /* latency stats */
 588        struct sched_info rq_sched_info;
 589
 590        /* sys_sched_yield() stats */
 591        unsigned int yld_exp_empty;
 592        unsigned int yld_act_empty;
 593        unsigned int yld_both_empty;
 594        unsigned int yld_count;
 595
 596        /* schedule() stats */
 597        unsigned int sched_switch;
 598        unsigned int sched_count;
 599        unsigned int sched_goidle;
 600
 601        /* try_to_wake_up() stats */
 602        unsigned int ttwu_count;
 603        unsigned int ttwu_local;
 604
 605        /* BKL stats */
 606        unsigned int bkl_count;
 607#endif
 608};
 609
 610static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 611
 612static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
 613{
 614        rq->curr->sched_class->check_preempt_curr(rq, p, sync);
 615}
 616
 617static inline int cpu_of(struct rq *rq)
 618{
 619#ifdef CONFIG_SMP
 620        return rq->cpu;
 621#else
 622        return 0;
 623#endif
 624}
 625
 626/*
 627 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
 628 * See detach_destroy_domains: synchronize_sched for details.
 629 *
 630 * The domain tree of any CPU may only be accessed from within
 631 * preempt-disabled sections.
 632 */
 633#define for_each_domain(cpu, __sd) \
 634        for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
 635
 636#define cpu_rq(cpu)                (&per_cpu(runqueues, (cpu)))
 637#define this_rq()                (&__get_cpu_var(runqueues))
 638#define task_rq(p)                cpu_rq(task_cpu(p))
 639#define cpu_curr(cpu)                (cpu_rq(cpu)->curr)
 640
 641static inline void update_rq_clock(struct rq *rq)
 642{
 643        rq->clock = sched_clock_cpu(cpu_of(rq));
 644}
 645
 646/*
 647 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
 648 */
 649#ifdef CONFIG_SCHED_DEBUG
 650# define const_debug __read_mostly
 651#else
 652# define const_debug static const
 653#endif
 654
 655/**
 656 * runqueue_is_locked
 657 *
 658 * Returns true if the current cpu runqueue is locked.
 659 * This interface allows printk to be called with the runqueue lock
 660 * held and know whether or not it is OK to wake up the klogd.
 661 */
 662int runqueue_is_locked(void)
 663{
 664        int cpu = get_cpu();
 665        struct rq *rq = cpu_rq(cpu);
 666        int ret;
 667
 668        ret = spin_is_locked(&rq->lock);
 669        put_cpu();
 670        return ret;
 671}
 672
 673/*
 674 * Debugging: various feature bits
 675 */
 676
 677#define SCHED_FEAT(name, enabled)        \
 678        __SCHED_FEAT_##name ,
 679
 680enum {
 681#include "sched_features.h"
 682};
 683
 684#undef SCHED_FEAT
 685
 686#define SCHED_FEAT(name, enabled)        \
 687        (1UL << __SCHED_FEAT_##name) * enabled |
 688
 689const_debug unsigned int sysctl_sched_features =
 690#include "sched_features.h"
 691        0;
 692
 693#undef SCHED_FEAT
 694
 695#ifdef CONFIG_SCHED_DEBUG
 696#define SCHED_FEAT(name, enabled)        \
 697        #name ,
 698
 699static __read_mostly char *sched_feat_names[] = {
 700#include "sched_features.h"
 701        NULL
 702};
 703
 704#undef SCHED_FEAT
 705
 706static int sched_feat_open(struct inode *inode, struct file *filp)
 707{
 708        filp->private_data = inode->i_private;
 709        return 0;
 710}
 711
 712static ssize_t
 713sched_feat_read(struct file *filp, char __user *ubuf,
 714                size_t cnt, loff_t *ppos)
 715{
 716        char *buf;
 717        int r = 0;
 718        int len = 0;
 719        int i;
 720
 721        for (i = 0; sched_feat_names[i]; i++) {
 722                len += strlen(sched_feat_names[i]);
 723                len += 4;
 724        }
 725
 726        buf = kmalloc(len + 2, GFP_KERNEL);
 727        if (!buf)
 728                return -ENOMEM;
 729
 730        for (i = 0; sched_feat_names[i]; i++) {
 731                if (sysctl_sched_features & (1UL << i))
 732                        r += sprintf(buf + r, "%s ", sched_feat_names[i]);
 733                else
 734                        r += sprintf(buf + r, "NO_%s ", sched_feat_names[i]);
 735        }
 736
 737        r += sprintf(buf + r, "\n");
 738        WARN_ON(r >= len + 2);
 739
 740        r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 741
 742        kfree(buf);
 743
 744        return r;
 745}
 746
 747static ssize_t
 748sched_feat_write(struct file *filp, const char __user *ubuf,
 749                size_t cnt, loff_t *ppos)
 750{
 751        char buf[64];
 752        char *cmp = buf;
 753        int neg = 0;
 754        int i;
 755
 756        if (cnt > 63)
 757                cnt = 63;
 758
 759        if (copy_from_user(&buf, ubuf, cnt))
 760                return -EFAULT;
 761
 762        buf[cnt] = 0;
 763
 764        if (strncmp(buf, "NO_", 3) == 0) {
 765                neg = 1;
 766                cmp += 3;
 767        }
 768
 769        for (i = 0; sched_feat_names[i]; i++) {
 770                int len = strlen(sched_feat_names[i]);
 771
 772                if (strncmp(cmp, sched_feat_names[i], len) == 0) {
 773                        if (neg)
 774                                sysctl_sched_features &= ~(1UL << i);
 775                        else
 776                                sysctl_sched_features |= (1UL << i);
 777                        break;
 778                }
 779        }
 780
 781        if (!sched_feat_names[i])
 782                return -EINVAL;
 783
 784        filp->f_pos += cnt;
 785
 786        return cnt;
 787}
 788
 789static struct file_operations sched_feat_fops = {
 790        .open        = sched_feat_open,
 791        .read        = sched_feat_read,
 792        .write        = sched_feat_write,
 793};
 794
 795static __init int sched_init_debug(void)
 796{
 797        debugfs_create_file("sched_features", 0644, NULL, NULL,
 798                        &sched_feat_fops);
 799
 800        return 0;
 801}
 802late_initcall(sched_init_debug);
 803
 804#endif
 805
 806#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
 807
 808/*
 809 * Number of tasks to iterate in a single balance run.
 810 * Limited because this is done with IRQs disabled.
 811 */
 812const_debug unsigned int sysctl_sched_nr_migrate = 32;
 813
 814/*
 815 * ratelimit for updating the group shares.
 816 * default: 0.25ms
 817 */
 818unsigned int sysctl_sched_shares_ratelimit = 250000;
 819
 820/*
 821 * Inject some fuzzyness into changing the per-cpu group shares
 822 * this avoids remote rq-locks at the expense of fairness.
 823 * default: 4
 824 */
 825unsigned int sysctl_sched_shares_thresh = 4;
 826
 827/*
 828 * period over which we measure -rt task cpu usage in us.
 829 * default: 1s
 830 */
 831unsigned int sysctl_sched_rt_period = 1000000;
 832
 833static __read_mostly int scheduler_running;
 834
 835/*
 836 * part of the period that we allow rt tasks to run in us.
 837 * default: 0.95s
 838 */
 839int sysctl_sched_rt_runtime = 950000;
 840
 841static inline u64 global_rt_period(void)
 842{
 843        return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
 844}
 845
 846static inline u64 global_rt_runtime(void)
 847{
 848        if (sysctl_sched_rt_runtime < 0)
 849                return RUNTIME_INF;
 850
 851        return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
 852}
 853
 854#ifndef prepare_arch_switch
 855# define prepare_arch_switch(next)        do { } while (0)
 856#endif
 857#ifndef finish_arch_switch
 858# define finish_arch_switch(prev)        do { } while (0)
 859#endif
 860
 861static inline int task_current(struct rq *rq, struct task_struct *p)
 862{
 863        return rq->curr == p;
 864}
 865
 866#ifndef __ARCH_WANT_UNLOCKED_CTXSW
 867static inline int task_running(struct rq *rq, struct task_struct *p)
 868{
 869        return task_current(rq, p);
 870}
 871
 872static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
 873{
 874}
 875
 876static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
 877{
 878#ifdef CONFIG_DEBUG_SPINLOCK
 879        /* this is a valid case when another task releases the spinlock */
 880        rq->lock.owner = current;
 881#endif
 882        /*
 883         * If we are tracking spinlock dependencies then we have to
 884         * fix up the runqueue lock - which gets 'carried over' from
 885         * prev into current:
 886         */
 887        spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
 888
 889        spin_unlock_irq(&rq->lock);
 890}
 891
 892#else /* __ARCH_WANT_UNLOCKED_CTXSW */
 893static inline int task_running(struct rq *rq, struct task_struct *p)
 894{
 895#ifdef CONFIG_SMP
 896        return p->oncpu;
 897#else
 898        return task_current(rq, p);
 899#endif
 900}
 901
 902static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
 903{
 904#ifdef CONFIG_SMP
 905        /*
 906         * We can optimise this out completely for !SMP, because the
 907         * SMP rebalancing from interrupt is the only thing that cares
 908         * here.
 909         */
 910        next->oncpu = 1;
 911#endif
 912#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
 913        spin_unlock_irq(&rq->lock);
 914#else
 915        spin_unlock(&rq->lock);
 916#endif
 917}
 918
 919static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
 920{
 921#ifdef CONFIG_SMP
 922        /*
 923         * After ->oncpu is cleared, the task can be moved to a different CPU.
 924         * We must ensure this doesn't happen until the switch is completely
 925         * finished.
 926         */
 927        smp_wmb();
 928        prev->oncpu = 0;
 929#endif
 930#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
 931        local_irq_enable();
 932#endif
 933}
 934#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
 935
 936/*
 937 * __task_rq_lock - lock the runqueue a given task resides on.
 938 * Must be called interrupts disabled.
 939 */
 940static inline struct rq *__task_rq_lock(struct task_struct *p)
 941        __acquires(rq->lock)
 942{
 943        for (;;) {
 944                struct rq *rq = task_rq(p);
 945                spin_lock(&rq->lock);
 946                if (likely(rq == task_rq(p)))
 947                        return rq;
 948                spin_unlock(&rq->lock);
 949        }
 950}
 951
 952/*
 953 * task_rq_lock - lock the runqueue a given task resides on and disable
 954 * interrupts. Note the ordering: we can safely lookup the task_rq without
 955 * explicitly disabling preemption.
 956 */
 957static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
 958        __acquires(rq->lock)
 959{
 960        struct rq *rq;
 961
 962        for (;;) {
 963                local_irq_save(*flags);
 964                rq = task_rq(p);
 965                spin_lock(&rq->lock);
 966                if (likely(rq == task_rq(p)))
 967                        return rq;
 968                spin_unlock_irqrestore(&rq->lock, *flags);
 969        }
 970}
 971
 972void task_rq_unlock_wait(struct task_struct *p)
 973{
 974        struct rq *rq = task_rq(p);
 975
 976        smp_mb(); /* spin-unlock-wait is not a full memory barrier */
 977        spin_unlock_wait(&rq->lock);
 978}
 979
 980static void __task_rq_unlock(struct rq *rq)
 981        __releases(rq->lock)
 982{
 983        spin_unlock(&rq->lock);
 984}
 985
 986static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
 987        __releases(rq->lock)
 988{
 989        spin_unlock_irqrestore(&rq->lock, *flags);
 990}
 991
 992/*
 993 * this_rq_lock - lock this runqueue and disable interrupts.
 994 */
 995static struct rq *this_rq_lock(void)
 996        __acquires(rq->lock)
 997{
 998        struct rq *rq;
 999
1000        local_irq_disable();
1001        rq = this_rq();
1002        spin_lock(&rq->lock);
1003
1004        return rq;
1005}
1006
1007#ifdef CONFIG_SCHED_HRTICK
1008/*
1009 * Use HR-timers to deliver accurate preemption points.
1010 *
1011 * Its all a bit involved since we cannot program an hrt while holding the
1012 * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
1013 * reschedule event.
1014 *
1015 * When we get rescheduled we reprogram the hrtick_timer outside of the
1016 * rq->lock.
1017 */
1018
1019/*
1020 * Use hrtick when:
1021 *  - enabled by features
1022 *  - hrtimer is actually high res
1023 */
1024static inline int hrtick_enabled(struct rq *rq)
1025{
1026        if (!sched_feat(HRTICK))
1027                return 0;
1028        if (!cpu_active(cpu_of(rq)))
1029                return 0;
1030        return hrtimer_is_hres_active(&rq->hrtick_timer);
1031}
1032
1033static void hrtick_clear(struct rq *rq)
1034{
1035        if (hrtimer_active(&rq->hrtick_timer))
1036                hrtimer_cancel(&rq->hrtick_timer);
1037}
1038
1039/*
1040 * High-resolution timer tick.
1041 * Runs from hardirq context with interrupts disabled.
1042 */
1043static enum hrtimer_restart hrtick(struct hrtimer *timer)
1044{
1045        struct rq *rq = container_of(timer, struct rq, hrtick_timer);
1046
1047        WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
1048
1049        spin_lock(&rq->lock);
1050        update_rq_clock(rq);
1051        rq->curr->sched_class->task_tick(rq, rq->curr, 1);
1052        spin_unlock(&rq->lock);
1053
1054        return HRTIMER_NORESTART;
1055}
1056
1057#ifdef CONFIG_SMP
1058/*
1059 * called from hardirq (IPI) context
1060 */
1061static void __hrtick_start(void *arg)
1062{
1063        struct rq *rq = arg;
1064
1065        spin_lock(&rq->lock);
1066        hrtimer_restart(&rq->hrtick_timer);
1067        rq->hrtick_csd_pending = 0;
1068        spin_unlock(&rq->lock);
1069}
1070
1071/*
1072 * Called to set the hrtick timer state.
1073 *
1074 * called with rq->lock held and irqs disabled
1075 */
1076static void hrtick_start(struct rq *rq, u64 delay)
1077{
1078        struct hrtimer *timer = &rq->hrtick_timer;
1079        ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
1080
1081        hrtimer_set_expires(timer, time);
1082
1083        if (rq == this_rq()) {
1084                hrtimer_restart(timer);
1085        } else if (!rq->hrtick_csd_pending) {
1086                __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);
1087                rq->hrtick_csd_pending = 1;
1088        }
1089}
1090
1091static int
1092hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
1093{
1094        int cpu = (int)(long)hcpu;
1095
1096        switch (action) {
1097        case CPU_UP_CANCELED:
1098        case CPU_UP_CANCELED_FROZEN:
1099        case CPU_DOWN_PREPARE:
1100        case CPU_DOWN_PREPARE_FROZEN:
1101        case CPU_DEAD:
1102        case CPU_DEAD_FROZEN:
1103                hrtick_clear(cpu_rq(cpu));
1104                return NOTIFY_OK;
1105        }
1106
1107        return NOTIFY_DONE;
1108}
1109
1110static __init void init_hrtick(void)
1111{
1112        hotcpu_notifier(hotplug_hrtick, 0);
1113}
1114#else
1115/*
1116 * Called to set the hrtick timer state.
1117 *
1118 * called with rq->lock held and irqs disabled
1119 */
1120static void hrtick_start(struct rq *rq, u64 delay)
1121{
1122        hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
1123}
1124
1125static inline void init_hrtick(void)
1126{
1127}
1128#endif /* CONFIG_SMP */
1129
1130static void init_rq_hrtick(struct rq *rq)
1131{
1132#ifdef CONFIG_SMP
1133        rq->hrtick_csd_pending = 0;
1134
1135        rq->hrtick_csd.flags = 0;
1136        rq->hrtick_csd.func = __hrtick_start;
1137        rq->hrtick_csd.info = rq;
1138#endif
1139
1140        hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1141        rq->hrtick_timer.function = hrtick;
1142        rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
1143}
1144#else        /* CONFIG_SCHED_HRTICK */
1145static inline void hrtick_clear(struct rq *rq)
1146{
1147}
1148
1149static inline void init_rq_hrtick(struct rq *rq)
1150{
1151}
1152
1153static inline void init_hrtick(void)
1154{
1155}
1156#endif        /* CONFIG_SCHED_HRTICK */
1157
1158/*
1159 * resched_task - mark a task 'to be rescheduled now'.
1160 *
1161 * On UP this means the setting of the need_resched flag, on SMP it
1162 * might also involve a cross-CPU call to trigger the scheduler on
1163 * the target CPU.
1164 */
1165#ifdef CONFIG_SMP
1166
1167#ifndef tsk_is_polling
1168#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
1169#endif
1170
1171static void resched_task(struct task_struct *p)
1172{
1173        int cpu;
1174
1175        assert_spin_locked(&task_rq(p)->lock);
1176
1177        if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
1178                return;
1179
1180        set_tsk_thread_flag(p, TIF_NEED_RESCHED);
1181
1182        cpu = task_cpu(p);
1183        if (cpu == smp_processor_id())
1184                return;
1185
1186        /* NEED_RESCHED must be visible before we test polling */
1187        smp_mb();
1188        if (!tsk_is_polling(p))
1189                smp_send_reschedule(cpu);
1190}
1191
1192static void resched_cpu(int cpu)
1193{
1194        struct rq *rq = cpu_rq(cpu);
1195        unsigned long flags;
1196
1197        if (!spin_trylock_irqsave(&rq->lock, flags))
1198                return;
1199        resched_task(cpu_curr(cpu));
1200        spin_unlock_irqrestore(&rq->lock, flags);
1201}
1202
1203#ifdef CONFIG_NO_HZ
1204/*
1205 * When add_timer_on() enqueues a timer into the timer wheel of an
1206 * idle CPU then this timer might expire before the next timer event
1207 * which is scheduled to wake up that CPU. In case of a completely
1208 * idle system the next event might even be infinite time into the
1209 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
1210 * leaves the inner idle loop so the newly added timer is taken into
1211 * account when the CPU goes back to idle and evaluates the timer
1212 * wheel for the next timer event.
1213 */
1214void wake_up_idle_cpu(int cpu)
1215{
1216        struct rq *rq = cpu_rq(cpu);
1217
1218        if (cpu == smp_processor_id())
1219                return;
1220
1221        /*
1222         * This is safe, as this function is called with the timer
1223         * wheel base lock of (cpu) held. When the CPU is on the way
1224         * to idle and has not yet set rq->curr to idle then it will
1225         * be serialized on the timer wheel base lock and take the new
1226         * timer into account automatically.
1227         */
1228        if (rq->curr != rq->idle)
1229                return;
1230
1231        /*
1232         * We can set TIF_RESCHED on the idle task of the other CPU
1233         * lockless. The worst case is that the other CPU runs the
1234         * idle task through an additional NOOP schedule()
1235         */
1236        set_tsk_thread_flag(rq->idle, TIF_NEED_RESCHED);
1237
1238        /* NEED_RESCHED must be visible before we test polling */
1239        smp_mb();
1240        if (!tsk_is_polling(rq->idle))
1241                smp_send_reschedule(cpu);
1242}
1243#endif /* CONFIG_NO_HZ */
1244
1245#else /* !CONFIG_SMP */
1246static void resched_task(struct task_struct *p)
1247{
1248        assert_spin_locked(&task_rq(p)->lock);
1249        set_tsk_need_resched(p);
1250}
1251#endif /* CONFIG_SMP */
1252
1253#if BITS_PER_LONG == 32
1254# define WMULT_CONST        (~0UL)
1255#else
1256# define WMULT_CONST        (1UL << 32)
1257#endif
1258
1259#define WMULT_SHIFT        32
1260
1261/*
1262 * Shift right and round:
1263 */
1264#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
1265
1266/*
1267 * delta *= weight / lw
1268 */
1269static unsigned long
1270calc_delta_mine(unsigned long delta_exec, unsigned long weight,
1271                struct load_weight *lw)
1272{
1273        u64 tmp;
1274
1275        if (!lw->inv_weight) {
1276                if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
1277                        lw->inv_weight = 1;
1278                else
1279                        lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
1280                                / (lw->weight+1);
1281        }
1282
1283        tmp = (u64)delta_exec * weight;
1284        /*
1285         * Check whether we'd overflow the 64-bit multiplication:
1286         */
1287        if (unlikely(tmp > WMULT_CONST))
1288                tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
1289                        WMULT_SHIFT/2);
1290        else
1291                tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
1292
1293        return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
1294}
1295
1296static inline void update_load_add(struct load_weight *lw, unsigned long inc)
1297{
1298        lw->weight += inc;
1299        lw->inv_weight = 0;
1300}
1301
1302static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
1303{
1304        lw->weight -= dec;
1305        lw->inv_weight = 0;
1306}
1307
1308/*
1309 * To aid in avoiding the subversion of "niceness" due to uneven distribution
1310 * of tasks with abnormal "nice" values across CPUs the contribution that
1311 * each task makes to its run queue's load is weighted according to its
1312 * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
1313 * scaled version of the new time slice allocation that they receive on time
1314 * slice expiry etc.
1315 */
1316
1317#define WEIGHT_IDLEPRIO                2
1318#define WMULT_IDLEPRIO                (1 << 31)
1319
1320/*
1321 * Nice levels are multiplicative, with a gentle 10% change for every
1322 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
1323 * nice 1, it will get ~10% less CPU time than another CPU-bound task
1324 * that remained on nice 0.
1325 *
1326 * The "10% effect" is relative and cumulative: from _any_ nice level,
1327 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
1328 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
1329 * If a task goes up by ~10% and another task goes down by ~10% then
1330 * the relative distance between them is ~25%.)
1331 */
1332static const int prio_to_weight[40] = {
1333 /* -20 */     88761,     71755,     56483,     46273,     36291,
1334 /* -15 */     29154,     23254,     18705,     14949,     11916,
1335 /* -10 */      9548,      7620,      6100,      4904,      3906,
1336 /*  -5 */      3121,      2501,      1991,      1586,      1277,
1337 /*   0 */      1024,       820,       655,       526,       423,
1338 /*   5 */       335,       272,       215,       172,       137,
1339 /*  10 */       110,        87,        70,        56,        45,
1340 /*  15 */        36,        29,        23,        18,        15,
1341};
1342
1343/*
1344 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
1345 *
1346 * In cases where the weight does not change often, we can use the
1347 * precalculated inverse to speed up arithmetics by turning divisions
1348 * into multiplications:
1349 */
1350static const u32 prio_to_wmult[40] = {
1351 /* -20 */     48388,     59856,     76040,     92818,    118348,
1352 /* -15 */    147320,    184698,    229616,    287308,    360437,
1353 /* -10 */    449829,    563644,    704093,    875809,   1099582,
1354 /*  -5 */   1376151,   1717300,   2157191,   2708050,   3363326,
1355 /*   0 */   4194304,   5237765,   6557202,   8165337,  10153587,
1356 /*   5 */  12820798,  15790321,  19976592,  24970740,  31350126,
1357 /*  10 */  39045157,  49367440,  61356676,  76695844,  95443717,
1358 /*  15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
1359};
1360
1361static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);
1362
1363/*
1364 * runqueue iterator, to support SMP load-balancing between different
1365 * scheduling classes, without having to expose their internal data
1366 * structures to the load-balancing proper:
1367 */
1368struct rq_iterator {
1369        void *arg;
1370        struct task_struct *(*start)(void *);
1371        struct task_struct *(*next)(void *);
1372};
1373
1374#ifdef CONFIG_SMP
1375static unsigned long
1376balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
1377              unsigned long max_load_move, struct sched_domain *sd,
1378              enum cpu_idle_type idle, int *all_pinned,
1379              int *this_best_prio, struct rq_iterator *iterator);
1380
1381static int
1382iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
1383                   struct sched_domain *sd, enum cpu_idle_type idle,
1384                   struct rq_iterator *iterator);
1385#endif
1386
1387#ifdef CONFIG_CGROUP_CPUACCT
1388static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
1389#else
1390static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
1391#endif
1392
1393static inline void inc_cpu_load(struct rq *rq, unsigned long load)
1394{
1395        update_load_add(&rq->load, load);
1396}
1397
1398static inline void dec_cpu_load(struct rq *rq, unsigned long load)
1399{
1400        update_load_sub(&rq->load, load);
1401}
1402
1403#if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
1404typedef int (*tg_visitor)(struct task_group *, void *);
1405
1406/*
1407 * Iterate the full tree, calling @down when first entering a node and @up when
1408 * leaving it for the final time.
1409 */
1410static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
1411{
1412        struct task_group *parent, *child;
1413        int ret;
1414
1415        rcu_read_lock();
1416        parent = &root_task_group;
1417down:
1418        ret = (*down)(parent, data);
1419        if (ret)
1420                goto out_unlock;
1421        list_for_each_entry_rcu(child, &parent->children, siblings) {
1422                parent = child;
1423                goto down;
1424
1425up:
1426                continue;
1427        }
1428        ret = (*up)(parent, data);
1429        if (ret)
1430                goto out_unlock;
1431
1432        child = parent;
1433        parent = parent->parent;
1434        if (parent)
1435                goto up;
1436out_unlock:
1437        rcu_read_unlock();
1438
1439        return ret;
1440}
1441
1442static int tg_nop(struct task_group *tg, void *data)
1443{
1444        return 0;
1445}
1446#endif
1447
1448#ifdef CONFIG_SMP
1449static unsigned long source_load(int cpu, int type);
1450static unsigned long target_load(int cpu, int type);
1451static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
1452
1453static unsigned long cpu_avg_load_per_task(int cpu)
1454{
1455        struct rq *rq = cpu_rq(cpu);
1456        unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
1457
1458        if (nr_running)
1459                rq->avg_load_per_task = rq->load.weight / nr_running;
1460        else
1461                rq->avg_load_per_task = 0;
1462
1463        return rq->avg_load_per_task;
1464}
1465
1466#ifdef CONFIG_FAIR_GROUP_SCHED
1467
1468static void __set_se_shares(struct sched_entity *se, unsigned long shares);
1469
1470/*
1471 * Calculate and set the cpu's group shares.
1472 */
1473static void
1474update_group_shares_cpu(struct task_group *tg, int cpu,
1475                        unsigned long sd_shares, unsigned long sd_rq_weight)
1476{
1477        int boost = 0;
1478        unsigned long shares;
1479        unsigned long rq_weight;
1480
1481        if (!tg->se[cpu])
1482                return;
1483
1484        rq_weight = tg->cfs_rq[cpu]->load.weight;
1485
1486        /*
1487         * If there are currently no tasks on the cpu pretend there is one of
1488         * average load so that when a new task gets to run here it will not
1489         * get delayed by group starvation.
1490         */
1491        if (!rq_weight) {
1492                boost = 1;
1493                rq_weight = NICE_0_LOAD;
1494        }
1495
1496        if (unlikely(rq_weight > sd_rq_weight))
1497                rq_weight = sd_rq_weight;
1498
1499        /*
1500         *           \Sum shares * rq_weight
1501         * shares =  -----------------------
1502         *               \Sum rq_weight
1503         *
1504         */
1505        shares = (sd_shares * rq_weight) / (sd_rq_weight + 1);
1506        shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
1507
1508        if (abs(shares - tg->se[cpu]->load.weight) >
1509                        sysctl_sched_shares_thresh) {
1510                struct rq *rq = cpu_rq(cpu);
1511                unsigned long flags;
1512
1513                spin_lock_irqsave(&rq->lock, flags);
1514                /*
1515                 * record the actual number of shares, not the boosted amount.
1516                 */
1517                tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
1518                tg->cfs_rq[cpu]->rq_weight = rq_weight;
1519
1520                __set_se_shares(tg->se[cpu], shares);
1521                spin_unlock_irqrestore(&rq->lock, flags);
1522        }
1523}
1524
1525/*
1526 * Re-compute the task group their per cpu shares over the given domain.
1527 * This needs to be done in a bottom-up fashion because the rq weight of a
1528 * parent group depends on the shares of its child groups.
1529 */
1530static int tg_shares_up(struct task_group *tg, void *data)
1531{
1532        unsigned long rq_weight = 0;
1533        unsigned long shares = 0;
1534        struct sched_domain *sd = data;
1535        int i;
1536
1537        for_each_cpu_mask(i, sd->span) {
1538                rq_weight += tg->cfs_rq[i]->load.weight;
1539                shares += tg->cfs_rq[i]->shares;
1540        }
1541
1542        if ((!shares && rq_weight) || shares > tg->shares)
1543                shares = tg->shares;
1544
1545        if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1546                shares = tg->shares;
1547
1548        if (!rq_weight)
1549                rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
1550
1551        for_each_cpu_mask(i, sd->span)
1552                update_group_shares_cpu(tg, i, shares, rq_weight);
1553
1554        return 0;
1555}
1556
1557/*
1558 * Compute the cpu's hierarchical load factor for each task group.
1559 * This needs to be done in a top-down fashion because the load of a child
1560 * group is a fraction of its parents load.
1561 */
1562static int tg_load_down(struct task_group *tg, void *data)
1563{
1564        unsigned long load;
1565        long cpu = (long)data;
1566
1567        if (!tg->parent) {
1568                load = cpu_rq(cpu)->load.weight;
1569        } else {
1570                load = tg->parent->cfs_rq[cpu]->h_load;
1571                load *= tg->cfs_rq[cpu]->shares;
1572                load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
1573        }
1574
1575        tg->cfs_rq[cpu]->h_load = load;
1576
1577        return 0;
1578}
1579
1580static void update_shares(struct sched_domain *sd)
1581{
1582        u64 now = cpu_clock(raw_smp_processor_id());
1583        s64 elapsed = now - sd->last_update;
1584
1585        if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
1586                sd->last_update = now;
1587                walk_tg_tree(tg_nop, tg_shares_up, sd);
1588        }
1589}
1590
1591static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
1592{
1593        spin_unlock(&rq->lock);
1594        update_shares(sd);
1595        spin_lock(&rq->lock);
1596}
1597
1598static void update_h_load(long cpu)
1599{
1600        walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
1601}
1602
1603#else
1604
1605static inline void update_shares(struct sched_domain *sd)
1606{
1607}
1608
1609static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
1610{
1611}
1612
1613#endif
1614
1615#endif
1616
1617#ifdef CONFIG_FAIR_GROUP_SCHED
1618static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
1619{
1620#ifdef CONFIG_SMP
1621        cfs_rq->shares = shares;
1622#endif
1623}
1624#endif
1625
1626#include "sched_stats.h"
1627#include "sched_idletask.c"
1628#include "sched_fair.c"
1629#include "sched_rt.c"
1630#ifdef CONFIG_SCHED_DEBUG
1631# include "sched_debug.c"
1632#endif
1633
1634#define sched_class_highest (&rt_sched_class)
1635#define for_each_class(class) \
1636   for (class = sched_class_highest; class; class = class->next)
1637
1638static void inc_nr_running(struct rq *rq)
1639{
1640        rq->nr_running++;
1641}
1642
1643static void dec_nr_running(struct rq *rq)
1644{
1645        rq->nr_running--;
1646}
1647
1648static void set_load_weight(struct task_struct *p)
1649{
1650        if (task_has_rt_policy(p)) {
1651                p->se.load.weight = prio_to_weight[0] * 2;
1652                p->se.load.inv_weight = prio_to_wmult[0] >> 1;
1653                return;
1654        }
1655
1656        /*
1657         * SCHED_IDLE tasks get minimal weight:
1658         */
1659        if (p->policy == SCHED_IDLE) {
1660                p->se.load.weight = WEIGHT_IDLEPRIO;
1661                p->se.load.inv_weight = WMULT_IDLEPRIO;
1662                return;
1663        }
1664
1665        p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
1666        p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
1667}
1668
1669static void update_avg(u64 *avg, u64 sample)
1670{
1671        s64 diff = sample - *avg;
1672        *avg += diff >> 3;
1673}
1674
1675static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
1676{
1677        sched_info_queued(p);
1678        p->sched_class->enqueue_task(rq, p, wakeup);
1679        p->se.on_rq = 1;
1680}
1681
1682static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
1683{
1684        if (sleep && p->se.last_wakeup) {
1685                update_avg(&p->se.avg_overlap,
1686                           p->se.sum_exec_runtime - p->se.last_wakeup);
1687                p->se.last_wakeup = 0;
1688        }
1689
1690        sched_info_dequeued(p);
1691        p->sched_class->dequeue_task(rq, p, sleep);
1692        p->se.on_rq = 0;
1693}
1694
1695/*
1696 * __normal_prio - return the priority that is based on the static prio
1697 */
1698static inline int __normal_prio(struct task_struct *p)
1699{
1700        return p->static_prio;
1701}
1702
1703/*
1704 * Calculate the expected normal priority: i.e. priority
1705 * without taking RT-inheritance into account. Might be
1706 * boosted by interactivity modifiers. Changes upon fork,
1707 * setprio syscalls, and whenever the interactivity
1708 * estimator recalculates.
1709 */
1710static inline int normal_prio(struct task_struct *p)
1711{
1712        int prio;
1713
1714        if (task_has_rt_policy(p))
1715                prio = MAX_RT_PRIO-1 - p->rt_priority;
1716        else
1717                prio = __normal_prio(p);
1718        return prio;
1719}
1720
1721/*
1722 * Calculate the current priority, i.e. the priority
1723 * taken into account by the scheduler. This value might
1724 * be boosted by RT tasks, or might be boosted by
1725 * interactivity modifiers. Will be RT if the task got
1726 * RT-boosted. If not then it returns p->normal_prio.
1727 */
1728static int effective_prio(struct task_struct *p)
1729{
1730        p->normal_prio = normal_prio(p);
1731        /*
1732         * If we are RT tasks or we were boosted to RT priority,
1733         * keep the priority unchanged. Otherwise, update priority
1734         * to the normal priority:
1735         */
1736        if (!rt_prio(p->prio))
1737                return p->normal_prio;
1738        return p->prio;
1739}
1740
1741/*
1742 * activate_task - move a task to the runqueue.
1743 */
1744static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
1745{
1746        if (task_contributes_to_load(p))
1747                rq->nr_uninterruptible--;
1748
1749        enqueue_task(rq, p, wakeup);
1750        inc_nr_running(rq);
1751}
1752
1753/*
1754 * deactivate_task - remove a task from the runqueue.
1755 */
1756static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
1757{
1758        if (task_contributes_to_load(p))
1759                rq->nr_uninterruptible++;
1760
1761        dequeue_task(rq, p, sleep);
1762        dec_nr_running(rq);
1763}
1764
1765/**
1766 * task_curr - is this task currently executing on a CPU?
1767 * @p: the task in question.
1768 */
1769inline int task_curr(const struct task_struct *p)
1770{
1771        return cpu_curr(task_cpu(p)) == p;
1772}
1773
1774static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
1775{
1776        set_task_rq(p, cpu);
1777#ifdef CONFIG_SMP
1778        /*
1779         * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
1780         * successfuly executed on another CPU. We must ensure that updates of
1781         * per-task data have been completed by this moment.
1782         */
1783        smp_wmb();
1784        task_thread_info(p)->cpu = cpu;
1785#endif
1786}
1787
1788static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1789                                       const struct sched_class *prev_class,
1790                                       int oldprio, int running)
1791{
1792        if (prev_class != p->sched_class) {
1793                if (prev_class->switched_from)
1794                        prev_class->switched_from(rq, p, running);
1795                p->sched_class->switched_to(rq, p, running);
1796        } else
1797                p->sched_class->prio_changed(rq, p, oldprio, running);
1798}
1799
1800#ifdef CONFIG_SMP
1801
1802/* Used instead of source_load when we know the type == 0 */
1803static unsigned long weighted_cpuload(const int cpu)
1804{
1805        return cpu_rq(cpu)->load.weight;
1806}
1807
1808/*
1809 * Is this task likely cache-hot:
1810 */
1811static int
1812task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
1813{
1814        s64 delta;
1815
1816        /*
1817         * Buddy candidates are cache hot:
1818         */
1819        if (sched_feat(CACHE_HOT_BUDDY) &&
1820                        (&p->se == cfs_rq_of(&p->se)->next ||
1821                         &p->se == cfs_rq_of(&p->se)->last))
1822                return 1;
1823
1824        if (p->sched_class != &fair_sched_class)
1825                return 0;
1826
1827        if (sysctl_sched_migration_cost == -1)
1828                return 1;
1829        if (sysctl_sched_migration_cost == 0)
1830                return 0;
1831
1832        delta = now - p->se.exec_start;
1833
1834        return delta < (s64)sysctl_sched_migration_cost;
1835}
1836
1837
1838void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1839{
1840        int old_cpu = task_cpu(p);
1841        struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
1842        struct cfs_rq *old_cfsrq = task_cfs_rq(p),
1843                      *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
1844        u64 clock_offset;
1845
1846        clock_offset = old_rq->clock - new_rq->clock;
1847
1848#ifdef CONFIG_SCHEDSTATS
1849        if (p->se.wait_start)
1850                p->se.wait_start -= clock_offset;
1851        if (p->se.sleep_start)
1852                p->se.sleep_start -= clock_offset;
1853        if (p->se.block_start)
1854                p->se.block_start -= clock_offset;
1855        if (old_cpu != new_cpu) {
1856                schedstat_inc(p, se.nr_migrations);
1857                if (task_hot(p, old_rq->clock, NULL))
1858                        schedstat_inc(p, se.nr_forced2_migrations);
1859        }
1860#endif
1861        p->se.vruntime -= old_cfsrq->min_vruntime -
1862                                         new_cfsrq->min_vruntime;
1863
1864        __set_task_cpu(p, new_cpu);
1865}
1866
1867struct migration_req {
1868        struct list_head list;
1869
1870        struct task_struct *task;
1871        int dest_cpu;
1872
1873        struct completion done;
1874};
1875
1876/*
1877 * The task's runqueue lock must be held.
1878 * Returns true if you have to wait for migration thread.
1879 */
1880static int
1881migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
1882{
1883        struct rq *rq = task_rq(p);
1884
1885        /*
1886         * If the task is not on a runqueue (and not running), then
1887         * it is sufficient to simply update the task's cpu field.
1888         */
1889        if (!p->se.on_rq && !task_running(rq, p)) {
1890                set_task_cpu(p, dest_cpu);
1891                return 0;
1892        }
1893
1894        init_completion(&req->done);
1895        req->task = p;
1896        req->dest_cpu = dest_cpu;
1897        list_add(&req->list, &rq->migration_queue);
1898
1899        return 1;
1900}
1901
1902/*
1903 * wait_task_inactive - wait for a thread to unschedule.
1904 *
1905 * If @match_state is nonzero, it's the @p->state value just checked and
1906 * not expected to change.  If it changes, i.e. @p might have woken up,
1907 * then return zero.  When we succeed in waiting for @p to be off its CPU,
1908 * we return a positive number (its total switch count).  If a second call
1909 * a short while later returns the same number, the caller can be sure that
1910 * @p has remained unscheduled the whole time.
1911 *
1912 * The caller must ensure that the task *will* unschedule sometime soon,
1913 * else this function might spin for a *long* time. This function can't
1914 * be called with interrupts off, or it may introduce deadlock with
1915 * smp_call_function() if an IPI is sent by the same process we are
1916 * waiting to become inactive.
1917 */
1918unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1919{
1920        unsigned long flags;
1921        int running, on_rq;
1922        unsigned long ncsw;
1923        struct rq *rq;
1924
1925        for (;;) {
1926                /*
1927                 * We do the initial early heuristics without holding
1928                 * any task-queue locks at all. We'll only try to get
1929                 * the runqueue lock when things look like they will
1930                 * work out!
1931                 */
1932                rq = task_rq(p);
1933
1934                /*
1935                 * If the task is actively running on another CPU
1936                 * still, just relax and busy-wait without holding
1937                 * any locks.
1938                 *
1939                 * NOTE! Since we don't hold any locks, it's not
1940                 * even sure that "rq" stays as the right runqueue!
1941                 * But we don't care, since "task_running()" will
1942                 * return false if the runqueue has changed and p
1943                 * is actually now running somewhere else!
1944                 */
1945                while (task_running(rq, p)) {
1946                        if (match_state && unlikely(p->state != match_state))
1947                                return 0;
1948                        cpu_relax();
1949                }
1950
1951                /*
1952                 * Ok, time to look more closely! We need the rq
1953                 * lock now, to be *sure*. If we're wrong, we'll
1954                 * just go back and repeat.
1955                 */
1956                rq = task_rq_lock(p, &flags);
1957                trace_sched_wait_task(rq, p);
1958                running = task_running(rq, p);
1959                on_rq = p->se.on_rq;
1960                ncsw = 0;
1961                if (!match_state || p->state == match_state)
1962                        ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
1963                task_rq_unlock(rq, &flags);
1964
1965                /*
1966                 * If it changed from the expected state, bail out now.
1967                 */
1968                if (unlikely(!ncsw))
1969                        break;
1970
1971                /*
1972                 * Was it really running after all now that we
1973                 * checked with the proper locks actually held?
1974                 *
1975                 * Oops. Go back and try again..
1976                 */
1977                if (unlikely(running)) {
1978                        cpu_relax();
1979                        continue;
1980                }
1981
1982                /*
1983                 * It's not enough that it's not actively running,
1984                 * it must be off the runqueue _entirely_, and not
1985                 * preempted!
1986                 *
1987                 * So if it wa still runnable (but just not actively
1988                 * running right now), it's preempted, and we should
1989                 * yield - it could be a while.
1990                 */
1991                if (unlikely(on_rq)) {
1992                        schedule_timeout_uninterruptible(1);
1993                        continue;
1994                }
1995
1996                /*
1997                 * Ahh, all good. It wasn't running, and it wasn't
1998                 * runnable, which means that it will never become
1999                 * running in the future either. We're all done!
2000                 */
2001                break;
2002        }
2003
2004        return ncsw;
2005}
2006
2007/***
2008 * kick_process - kick a running thread to enter/exit the kernel
2009 * @p: the to-be-kicked thread
2010 *
2011 * Cause a process which is running on another CPU to enter
2012 * kernel-mode, without any delay. (to get signals handled.)
2013 *
2014 * NOTE: this function doesnt have to take the runqueue lock,
2015 * because all it wants to ensure is that the remote task enters
2016 * the kernel. If the IPI races and the task has been migrated
2017 * to another CPU then no harm is done and the purpose has been
2018 * achieved as well.
2019 */
2020void kick_process(struct task_struct *p)
2021{
2022        int cpu;
2023
2024        preempt_disable();
2025        cpu = task_cpu(p);
2026        if ((cpu != smp_processor_id()) && task_curr(p))
2027                smp_send_reschedule(cpu);
2028        preempt_enable();
2029}
2030
2031/*
2032 * Return a low guess at the load of a migration-source cpu weighted
2033 * according to the scheduling class and "nice" value.
2034 *
2035 * We want to under-estimate the load of migration sources, to
2036 * balance conservatively.
2037 */
2038static unsigned long source_load(int cpu, int type)
2039{
2040        struct rq *rq = cpu_rq(cpu);
2041        unsigned long total = weighted_cpuload(cpu);
2042
2043        if (type == 0 || !sched_feat(LB_BIAS))
2044                return total;
2045
2046        return min(rq->cpu_load[type-1], total);
2047}
2048
2049/*
2050 * Return a high guess at the load of a migration-target cpu weighted
2051 * according to the scheduling class and "nice" value.
2052 */
2053static unsigned long target_load(int cpu, int type)
2054{
2055        struct rq *rq = cpu_rq(cpu);
2056        unsigned long total = weighted_cpuload(cpu);
2057
2058        if (type == 0 || !sched_feat(LB_BIAS))
2059                return total;
2060
2061        return max(rq->cpu_load[type-1], total);
2062}
2063
2064/*
2065 * find_idlest_group finds and returns the least busy CPU group within the
2066 * domain.
2067 */
2068static struct sched_group *
2069find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2070{
2071        struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
2072        unsigned long min_load = ULONG_MAX, this_load = 0;
2073        int load_idx = sd->forkexec_idx;
2074        int imbalance = 100 + (sd->imbalance_pct-100)/2;
2075
2076        do {
2077                unsigned long load, avg_load;
2078                int local_group;
2079                int i;
2080
2081                /* Skip over this group if it has no CPUs allowed */
2082                if (!cpus_intersects(group->cpumask, p->cpus_allowed))
2083                        continue;
2084
2085                local_group = cpu_isset(this_cpu, group->cpumask);
2086
2087                /* Tally up the load of all CPUs in the group */
2088                avg_load = 0;
2089
2090                for_each_cpu_mask_nr(i, group->cpumask) {
2091                        /* Bias balancing toward cpus of our domain */
2092                        if (local_group)
2093                                load = source_load(i, load_idx);
2094                        else
2095                                load = target_load(i, load_idx);
2096
2097                        avg_load += load;
2098                }
2099
2100                /* Adjust by relative CPU power of the group */
2101                avg_load = sg_div_cpu_power(group,
2102                                avg_load * SCHED_LOAD_SCALE);
2103
2104                if (local_group) {
2105                        this_load = avg_load;
2106                        this = group;
2107                } else if (avg_load < min_load) {
2108                        min_load = avg_load;
2109                        idlest = group;
2110                }
2111        } while (group = group->next, group != sd->groups);
2112
2113        if (!idlest || 100*this_load < imbalance*min_load)
2114                return NULL;
2115        return idlest;
2116}
2117
2118/*
2119 * find_idlest_cpu - find the idlest cpu among the cpus in group.
2120 */
2121static int
2122find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu,
2123                cpumask_t *tmp)
2124{
2125        unsigned long load, min_load = ULONG_MAX;
2126        int idlest = -1;
2127        int i;
2128
2129        /* Traverse only the allowed CPUs */
2130        cpus_and(*tmp, group->cpumask, p->cpus_allowed);
2131
2132        for_each_cpu_mask_nr(i, *tmp) {
2133                load = weighted_cpuload(i);
2134
2135                if (load < min_load || (load == min_load && i == this_cpu)) {
2136                        min_load = load;
2137                        idlest = i;
2138                }
2139        }
2140
2141        return idlest;
2142}
2143
2144/*
2145 * sched_balance_self: balance the current task (running on cpu) in domains
2146 * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
2147 * SD_BALANCE_EXEC.
2148 *
2149 * Balance, ie. select the least loaded group.
2150 *
2151 * Returns the target CPU number, or the same CPU if no balancing is needed.
2152 *
2153 * preempt must be disabled.
2154 */
2155static int sched_balance_self(int cpu, int flag)
2156{
2157        struct task_struct *t = current;
2158        struct sched_domain *tmp, *sd = NULL;
2159
2160        for_each_domain(cpu, tmp) {
2161                /*
2162                 * If power savings logic is enabled for a domain, stop there.
2163                 */
2164                if (tmp->flags & SD_POWERSAVINGS_BALANCE)
2165                        break;
2166                if (tmp->flags & flag)
2167                        sd = tmp;
2168        }
2169
2170        if (sd)
2171                update_shares(sd);
2172
2173        while (sd) {
2174                cpumask_t span, tmpmask;
2175                struct sched_group *group;
2176                int new_cpu, weight;
2177
2178                if (!(sd->flags & flag)) {
2179                        sd = sd->child;
2180                        continue;
2181                }
2182
2183                span = sd->span;
2184                group = find_idlest_group(sd, t, cpu);
2185                if (!group) {
2186                        sd = sd->child;
2187                        continue;
2188                }
2189
2190                new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask);
2191                if (new_cpu == -1 || new_cpu == cpu) {
2192                        /* Now try balancing at a lower domain level of cpu */
2193                        sd = sd->child;
2194                        continue;
2195                }
2196
2197                /* Now try balancing at a lower domain level of new_cpu */
2198                cpu = new_cpu;
2199                sd = NULL;
2200                weight = cpus_weight(span);
2201                for_each_domain(cpu, tmp) {
2202                        if (weight <= cpus_weight(tmp->span))
2203                                break;
2204                        if (tmp->flags & flag)
2205                                sd = tmp;
2206                }
2207                /* while loop will break here if sd == NULL */
2208        }
2209
2210        return cpu;
2211}
2212
2213#endif /* CONFIG_SMP */
2214
2215/***
2216 * try_to_wake_up - wake up a thread
2217 * @p: the to-be-woken-up thread
2218 * @state: the mask of task states that can be woken
2219 * @sync: do a synchronous wakeup?
2220 *
2221 * Put it on the run-queue if it's not already there. The "current"
2222 * thread is always on the run-queue (except when the actual
2223 * re-schedule is in progress), and as such you're allowed to do
2224 * the simpler "current->state = TASK_RUNNING" to mark yourself
2225 * runnable without the overhead of this.
2226 *
2227 * returns failure only if the task is already active.
2228 */
2229static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2230{
2231        int cpu, orig_cpu, this_cpu, success = 0;
2232        unsigned long flags;
2233        long old_state;
2234        struct rq *rq;
2235
2236        if (!sched_feat(SYNC_WAKEUPS))
2237                sync = 0;
2238
2239#ifdef CONFIG_SMP
2240        if (sched_feat(LB_WAKEUP_UPDATE)) {
2241                struct sched_domain *sd;
2242
2243                this_cpu = raw_smp_processor_id();
2244                cpu = task_cpu(p);
2245
2246                for_each_domain(this_cpu, sd) {
2247                        if (cpu_isset(cpu, sd->span)) {
2248                                update_shares(sd);
2249                                break;
2250                        }
2251                }
2252        }
2253#endif
2254
2255        smp_wmb();
2256        rq = task_rq_lock(p, &flags);
2257        old_state = p->state;
2258        if (!(old_state & state))
2259                goto out;
2260
2261        if (p->se.on_rq)
2262                goto out_running;
2263
2264        cpu = task_cpu(p);
2265        orig_cpu = cpu;
2266        this_cpu = smp_processor_id();
2267
2268#ifdef CONFIG_SMP
2269        if (unlikely(task_running(rq, p)))
2270                goto out_activate;
2271
2272        cpu = p->sched_class->select_task_rq(p, sync);
2273        if (cpu != orig_cpu) {
2274                set_task_cpu(p, cpu);
2275                task_rq_unlock(rq, &flags);
2276                /* might preempt at this point */
2277                rq = task_rq_lock(p, &flags);
2278                old_state = p->state;
2279                if (!(old_state & state))
2280                        goto out;
2281                if (p->se.on_rq)
2282                        goto out_running;
2283
2284                this_cpu = smp_processor_id();
2285                cpu = task_cpu(p);
2286        }
2287
2288#ifdef CONFIG_SCHEDSTATS
2289        schedstat_inc(rq, ttwu_count);
2290        if (cpu == this_cpu)
2291                schedstat_inc(rq, ttwu_local);
2292        else {
2293                struct sched_domain *sd;
2294                for_each_domain(this_cpu, sd) {
2295                        if (cpu_isset(cpu, sd->span)) {
2296                                schedstat_inc(sd, ttwu_wake_remote);
2297                                break;
2298                        }
2299                }
2300        }
2301#endif /* CONFIG_SCHEDSTATS */
2302
2303out_activate:
2304#endif /* CONFIG_SMP */
2305        schedstat_inc(p, se.nr_wakeups);
2306        if (sync)
2307                schedstat_inc(p, se.nr_wakeups_sync);
2308        if (orig_cpu != cpu)
2309                schedstat_inc(p, se.nr_wakeups_migrate);
2310        if (cpu == this_cpu)
2311                schedstat_inc(p, se.nr_wakeups_local);
2312        else
2313                schedstat_inc(p, se.nr_wakeups_remote);
2314        update_rq_clock(rq);
2315        activate_task(rq, p, 1);
2316        success = 1;
2317
2318out_running:
2319        trace_sched_wakeup(rq, p);
2320        check_preempt_curr(rq, p, sync);
2321
2322        p->state = TASK_RUNNING;
2323#ifdef CONFIG_SMP
2324        if (p->sched_class->task_wake_up)
2325                p->sched_class->task_wake_up(rq, p);
2326#endif
2327out:
2328        current->se.last_wakeup = current->se.sum_exec_runtime;
2329
2330        task_rq_unlock(rq, &flags);
2331
2332        return success;
2333}
2334
2335int wake_up_process(struct task_struct *p)
2336{
2337        return try_to_wake_up(p, TASK_ALL, 0);
2338}
2339EXPORT_SYMBOL(wake_up_process);
2340
2341int wake_up_state(struct task_struct *p, unsigned int state)
2342{
2343        return try_to_wake_up(p, state, 0);
2344}
2345
2346/*
2347 * Perform scheduler related setup for a newly forked process p.
2348 * p is forked by current.
2349 *
2350 * __sched_fork() is basic setup used by init_idle() too:
2351 */
2352static void __sched_fork(struct task_struct *p)
2353{
2354        p->se.exec_start                = 0;
2355        p->se.sum_exec_runtime                = 0;
2356        p->se.prev_sum_exec_runtime        = 0;
2357        p->se.last_wakeup                = 0;
2358        p->se.avg_overlap                = 0;
2359
2360#ifdef CONFIG_SCHEDSTATS
2361        p->se.wait_start                = 0;
2362        p->se.sum_sleep_runtime                = 0;
2363        p->se.sleep_start                = 0;
2364        p->se.block_start                = 0;
2365        p->se.sleep_max                        = 0;
2366        p->se.block_max                        = 0;
2367        p->se.exec_max                        = 0;
2368        p->se.slice_max                        = 0;
2369        p->se.wait_max                        = 0;
2370#endif
2371
2372        INIT_LIST_HEAD(&p->rt.run_list);
2373        p->se.on_rq = 0;
2374        INIT_LIST_HEAD(&p->se.group_node);
2375
2376#ifdef CONFIG_PREEMPT_NOTIFIERS
2377        INIT_HLIST_HEAD(&p->preempt_notifiers);
2378#endif
2379
2380        /*
2381         * We mark the process as running here, but have not actually
2382         * inserted it onto the runqueue yet. This guarantees that
2383         * nobody will actually run it, and a signal or other external
2384         * event cannot wake it up and insert it on the runqueue either.
2385         */
2386        p->state = TASK_RUNNING;
2387}
2388
2389/*
2390 * fork()/clone()-time setup:
2391 */
2392void sched_fork(struct task_struct *p, int clone_flags)
2393{
2394        int cpu = get_cpu();
2395
2396        __sched_fork(p);
2397
2398#ifdef CONFIG_SMP
2399        cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
2400#endif
2401        set_task_cpu(p, cpu);
2402
2403        /*
2404         * Make sure we do not leak PI boosting priority to the child:
2405         */
2406        p->prio = current->normal_prio;
2407        if (!rt_prio(p->prio))
2408                p->sched_class = &fair_sched_class;
2409
2410#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
2411        if (likely(sched_info_on()))
2412                memset(&p->sched_info, 0, sizeof(p->sched_info));
2413#endif
2414#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
2415        p->oncpu = 0;
2416#endif
2417#ifdef CONFIG_PREEMPT
2418        /* Want to start with kernel preemption disabled. */
2419        task_thread_info(p)->preempt_count = 1;
2420#endif
2421        put_cpu();
2422}
2423
2424/*
2425 * wake_up_new_task - wake up a newly created task for the first time.
2426 *
2427 * This function will do some initial scheduler statistics housekeeping
2428 * that must be done for every newly created context, then puts the task
2429 * on the runqueue and wakes it.
2430 */
2431void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
2432{
2433        unsigned long flags;
2434        struct rq *rq;
2435
2436        rq = task_rq_lock(p, &flags);
2437        BUG_ON(p->state != TASK_RUNNING);
2438        update_rq_clock(rq);
2439
2440        p->prio = effective_prio(p);
2441
2442        if (!p->sched_class->task_new || !current->se.on_rq) {
2443                activate_task(rq, p, 0);
2444        } else {
2445                /*
2446                 * Let the scheduling class do new task startup
2447                 * management (if any):
2448                 */
2449                p->sched_class->task_new(rq, p);
2450                inc_nr_running(rq);
2451        }
2452        trace_sched_wakeup_new(rq, p);
2453        check_preempt_curr(rq, p, 0);
2454#ifdef CONFIG_SMP
2455        if (p->sched_class->task_wake_up)
2456                p->sched_class->task_wake_up(rq, p);
2457#endif
2458        task_rq_unlock(rq, &flags);
2459}
2460
2461#ifdef CONFIG_PREEMPT_NOTIFIERS
2462
2463/**
2464 * preempt_notifier_register - tell me when current is being being preempted & rescheduled
2465 * @notifier: notifier struct to register
2466 */
2467void preempt_notifier_register(struct preempt_notifier *notifier)
2468{
2469        hlist_add_head(&notifier->link, &current->preempt_notifiers);
2470}
2471EXPORT_SYMBOL_GPL(preempt_notifier_register);
2472
2473/**
2474 * preempt_notifier_unregister - no longer interested in preemption notifications
2475 * @notifier: notifier struct to unregister
2476 *
2477 * This is safe to call from within a preemption notifier.
2478 */
2479void preempt_notifier_unregister(struct preempt_notifier *notifier)
2480{
2481        hlist_del(&notifier->link);
2482}
2483EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2484
2485static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2486{
2487        struct preempt_notifier *notifier;
2488        struct hlist_node *node;
2489
2490        hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
2491                notifier->ops->sched_in(notifier, raw_smp_processor_id());
2492}
2493
2494static void
2495fire_sched_out_preempt_notifiers(struct task_struct *curr,
2496                                 struct task_struct *next)
2497{
2498        struct preempt_notifier *notifier;
2499        struct hlist_node *node;
2500
2501        hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
2502                notifier->ops->sched_out(notifier, next);
2503}
2504
2505#else /* !CONFIG_PREEMPT_NOTIFIERS */
2506
2507static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2508{
2509}
2510
2511static void
2512fire_sched_out_preempt_notifiers(struct task_struct *curr,
2513                                 struct task_struct *next)
2514{
2515}
2516
2517#endif /* CONFIG_PREEMPT_NOTIFIERS */
2518
2519/**
2520 * prepare_task_switch - prepare to switch tasks
2521 * @rq: the runqueue preparing to switch
2522 * @prev: the current task that is being switched out
2523 * @next: the task we are going to switch to.
2524 *
2525 * This is called with the rq lock held and interrupts off. It must
2526 * be paired with a subsequent finish_task_switch after the context
2527 * switch.
2528 *
2529 * prepare_task_switch sets up locking and calls architecture specific
2530 * hooks.
2531 */
2532static inline void
2533prepare_task_switch(struct rq *rq, struct task_struct *prev,
2534                    struct task_struct *next)
2535{
2536        fire_sched_out_preempt_notifiers(prev, next);
2537        prepare_lock_switch(rq, next);
2538        prepare_arch_switch(next);
2539}
2540
2541/**
2542 * finish_task_switch - clean up after a task-switch
2543 * @rq: runqueue associated with task-switch
2544 * @prev: the thread we just switched away from.
2545 *
2546 * finish_task_switch must be called after the context switch, paired
2547 * with a prepare_task_switch call before the context switch.
2548 * finish_task_switch will reconcile locking set up by prepare_task_switch,
2549 * and do any other architecture-specific cleanup actions.
2550 *
2551 * Note that we may have delayed dropping an mm in context_switch(). If
2552 * so, we finish that here outside of the runqueue lock. (Doing it
2553 * with the lock held can cause deadlocks; see schedule() for
2554 * details.)
2555 */
2556static void finish_task_switch(struct rq *rq, struct task_struct *prev)
2557        __releases(rq->lock)
2558{
2559        struct mm_struct *mm = rq->prev_mm;
2560        long prev_state;
2561
2562        rq->prev_mm = NULL;
2563
2564        /*
2565         * A task struct has one reference for the use as "current".
2566         * If a task dies, then it sets TASK_DEAD in tsk->state and calls
2567         * schedule one last time. The schedule call will never return, and
2568         * the scheduled task must drop that reference.
2569         * The test for TASK_DEAD must occur while the runqueue locks are
2570         * still held, otherwise prev could be scheduled on another cpu, die
2571         * there before we look at prev->state, and then the reference would
2572         * be dropped twice.
2573         *                Manfred Spraul <manfred@colorfullife.com>
2574         */
2575        prev_state = prev->state;
2576        finish_arch_switch(prev);
2577        finish_lock_switch(rq, prev);
2578#ifdef CONFIG_SMP
2579        if (current->sched_class->post_schedule)
2580                current->sched_class->post_schedule(rq);
2581#endif
2582
2583        fire_sched_in_preempt_notifiers(current);
2584        if (mm)
2585                mmdrop(mm);
2586        if (unlikely(prev_state == TASK_DEAD)) {
2587                /*
2588                 * Remove function-return probe instances associated with this
2589                 * task and put them back on the free list.
2590                 */
2591                kprobe_flush_task(prev);
2592                put_task_struct(prev);
2593        }
2594}
2595
2596/**
2597 * schedule_tail - first thing a freshly forked thread must call.
2598 * @prev: the thread we just switched away from.
2599 */
2600asmlinkage void schedule_tail(struct task_struct *prev)
2601        __releases(rq->lock)
2602{
2603        struct rq *rq = this_rq();
2604
2605        finish_task_switch(rq, prev);
2606#ifdef __ARCH_WANT_UNLOCKED_CTXSW
2607        /* In this case, finish_task_switch does not reenable preemption */
2608        preempt_enable();
2609#endif
2610        if (current->set_child_tid)
2611                put_user(task_pid_vnr(current), current->set_child_tid);
2612}
2613
2614/*
2615 * context_switch - switch to the new MM and the new
2616 * thread's register state.
2617 */
2618static inline void
2619context_switch(struct rq *rq, struct task_struct *prev,
2620               struct task_struct *next)
2621{
2622        struct mm_struct *mm, *oldmm;
2623
2624        prepare_task_switch(rq, prev, next);
2625        trace_sched_switch(rq, prev, next);
2626        mm = next->mm;
2627        oldmm = prev->active_mm;
2628        /*
2629         * For paravirt, this is coupled with an exit in switch_to to
2630         * combine the page table reload and the switch backend into
2631         * one hypercall.
2632         */
2633        arch_enter_lazy_cpu_mode();
2634
2635        if (unlikely(!mm)) {
2636                next->active_mm = oldmm;
2637                atomic_inc(&oldmm->mm_count);
2638                enter_lazy_tlb(oldmm, next);
2639        } else
2640                switch_mm(oldmm, mm, next);
2641
2642        if (unlikely(!prev->mm)) {
2643                prev->active_mm = NULL;
2644                rq->prev_mm = oldmm;
2645        }
2646        /*
2647         * Since the runqueue lock will be released by the next
2648         * task (which is an invalid locking op but in the case
2649         * of the scheduler it's an obvious special-case), so we
2650         * do an early lockdep release here:
2651         */
2652#ifndef __ARCH_WANT_UNLOCKED_CTXSW
2653        spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2654#endif
2655
2656        /* Here we just switch the register state and the stack. */
2657        switch_to(prev, next, prev);
2658
2659        barrier();
2660        /*
2661         * this_rq must be evaluated again because prev may have moved
2662         * CPUs since it called schedule(), thus the 'rq' on its stack
2663         * frame will be invalid.
2664         */
2665        finish_task_switch(this_rq(), prev);
2666}
2667
2668/*
2669 * nr_running, nr_uninterruptible and nr_context_switches:
2670 *
2671 * externally visible scheduler statistics: current number of runnable
2672 * threads, current number of uninterruptible-sleeping threads, total
2673 * number of context switches performed since bootup.
2674 */
2675unsigned long nr_running(void)
2676{
2677        unsigned long i, sum = 0;
2678
2679        for_each_online_cpu(i)
2680                sum += cpu_rq(i)->nr_running;
2681
2682        return sum;
2683}
2684
2685unsigned long nr_uninterruptible(void)
2686{
2687        unsigned long i, sum = 0;
2688
2689        for_each_possible_cpu(i)
2690                sum += cpu_rq(i)->nr_uninterruptible;
2691
2692        /*
2693         * Since we read the counters lockless, it might be slightly
2694         * inaccurate. Do not allow it to go below zero though:
2695         */
2696        if (unlikely((long)sum < 0))
2697                sum = 0;
2698
2699        return sum;
2700}
2701
2702unsigned long long nr_context_switches(void)
2703{
2704        int i;
2705        unsigned long long sum = 0;
2706
2707        for_each_possible_cpu(i)
2708                sum += cpu_rq(i)->nr_switches;
2709
2710        return sum;
2711}
2712
2713unsigned long nr_iowait(void)
2714{
2715        unsigned long i, sum = 0;
2716
2717        for_each_possible_cpu(i)
2718                sum += atomic_read(&cpu_rq(i)->nr_iowait);
2719
2720        return sum;
2721}
2722
2723unsigned long nr_active(void)
2724{
2725        unsigned long i, running = 0, uninterruptible = 0;
2726
2727        for_each_online_cpu(i) {
2728                running += cpu_rq(i)->nr_running;
2729                uninterruptible += cpu_rq(i)->nr_uninterruptible;
2730        }
2731
2732        if (unlikely((long)uninterruptible < 0))
2733                uninterruptible = 0;
2734
2735        return running + uninterruptible;
2736}
2737
2738/*
2739 * Update rq->cpu_load[] statistics. This function is usually called every
2740 * scheduler tick (TICK_NSEC).
2741 */
2742static void update_cpu_load(struct rq *this_rq)
2743{
2744        unsigned long this_load = this_rq->load.weight;
2745        int i, scale;
2746
2747        this_rq->nr_load_updates++;
2748
2749        /* Update our load: */
2750        for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
2751                unsigned long old_load, new_load;
2752
2753                /* scale is effectively 1 << i now, and >> i divides by scale */
2754
2755                old_load = this_rq->cpu_load[i];
2756                new_load = this_load;
2757                /*
2758                 * Round up the averaging division if load is increasing. This
2759                 * prevents us from getting stuck on 9 if the load is 10, for
2760                 * example.
2761                 */
2762                if (new_load > old_load)
2763                        new_load += scale-1;
2764                this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
2765        }
2766}
2767
2768#ifdef CONFIG_SMP
2769
2770/*
2771 * double_rq_lock - safely lock two runqueues
2772 *
2773 * Note this does not disable interrupts like task_rq_lock,
2774 * you need to do so manually before calling.
2775 */
2776static void double_rq_lock(struct rq *rq1, struct rq *rq2)
2777        __acquires(rq1->lock)
2778        __acquires(rq2->lock)
2779{
2780        BUG_ON(!irqs_disabled());
2781        if (rq1 == rq2) {
2782                spin_lock(&rq1->lock);
2783                __acquire(rq2->lock);        /* Fake it out ;) */
2784        } else {
2785                if (rq1 < rq2) {
2786                        spin_lock(&rq1->lock);
2787                        spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
2788                } else {
2789                        spin_lock(&rq2->lock);
2790                        spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
2791                }
2792        }
2793        update_rq_clock(rq1);
2794        update_rq_clock(rq2);
2795}
2796
2797/*
2798 * double_rq_unlock - safely unlock two runqueues
2799 *
2800 * Note this does not restore interrupts like task_rq_unlock,
2801 * you need to do so manually after calling.
2802 */
2803static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
2804        __releases(rq1->lock)
2805        __releases(rq2->lock)
2806{
2807        spin_unlock(&rq1->lock);
2808        if (rq1 != rq2)
2809                spin_unlock(&rq2->lock);
2810        else
2811                __release(rq2->lock);
2812}
2813
2814/*
2815 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
2816 */
2817static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
2818        __releases(this_rq->lock)
2819        __acquires(busiest->lock)
2820        __acquires(this_rq->lock)
2821{
2822        int ret = 0;
2823
2824        if (unlikely(!irqs_disabled())) {
2825                /* printk() doesn't work good under rq->lock */
2826                spin_unlock(&this_rq->lock);
2827                BUG_ON(1);
2828        }
2829        if (unlikely(!spin_trylock(&busiest->lock))) {
2830                if (busiest < this_rq) {
2831                        spin_unlock(&this_rq->lock);
2832                        spin_lock(&busiest->lock);
2833                        spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
2834                        ret = 1;
2835                } else
2836                        spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
2837        }
2838        return ret;
2839}
2840
2841static void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
2842        __releases(busiest->lock)
2843{
2844        spin_unlock(&busiest->lock);
2845        lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
2846}
2847
2848/*
2849 * If dest_cpu is allowed for this process, migrate the task to it.
2850 * This is accomplished by forcing the cpu_allowed mask to only
2851 * allow dest_cpu, which will force the cpu onto dest_cpu. Then
2852 * the cpu_allowed mask is restored.
2853 */
2854static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2855{
2856        struct migration_req req;
2857        unsigned long flags;
2858        struct rq *rq;
2859
2860        rq = task_rq_lock(p, &flags);
2861        if (!cpu_isset(dest_cpu, p->cpus_allowed)
2862            || unlikely(!cpu_active(dest_cpu)))
2863                goto out;
2864
2865        trace_sched_migrate_task(rq, p, dest_cpu);
2866        /* force the process onto the specified CPU */
2867        if (migrate_task(p, dest_cpu, &req)) {
2868                /* Need to wait for migration thread (might exit: take ref). */
2869                struct task_struct *mt = rq->migration_thread;
2870
2871                get_task_struct(mt);
2872                task_rq_unlock(rq, &flags);
2873                wake_up_process(mt);
2874                put_task_struct(mt);
2875                wait_for_completion(&req.done);
2876
2877                return;
2878        }
2879out:
2880        task_rq_unlock(rq, &flags);
2881}
2882
2883/*
2884 * sched_exec - execve() is a valuable balancing opportunity, because at
2885 * this point the task has the smallest effective memory and cache footprint.
2886 */
2887void sched_exec(void)
2888{
2889        int new_cpu, this_cpu = get_cpu();
2890        new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
2891        put_cpu();
2892        if (new_cpu != this_cpu)
2893                sched_migrate_task(current, new_cpu);
2894}
2895
2896/*
2897 * pull_task - move a task from a remote runqueue to the local runqueue.
2898 * Both runqueues must be locked.
2899 */
2900static void pull_task(struct rq *src_rq, struct task_struct *p,
2901                      struct rq *this_rq, int this_cpu)
2902{
2903        deactivate_task(src_rq, p, 0);
2904        set_task_cpu(p, this_cpu);
2905        activate_task(this_rq, p, 0);
2906        /*
2907         * Note that idle threads have a prio of MAX_PRIO, for this test
2908         * to be always true for them.
2909         */
2910        check_preempt_curr(this_rq, p, 0);
2911}
2912
2913/*
2914 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
2915 */
2916static
2917int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2918                     struct sched_domain *sd, enum cpu_idle_type idle,
2919                     int *all_pinned)
2920{
2921        /*
2922         * We do not migrate tasks that are:
2923         * 1) running (obviously), or
2924         * 2) cannot be migrated to this CPU due to cpus_allowed, or
2925         * 3) are cache-hot on their current CPU.
2926         */
2927        if (!cpu_isset(this_cpu, p->cpus_allowed)) {
2928                schedstat_inc(p, se.nr_failed_migrations_affine);
2929                return 0;
2930        }
2931        *all_pinned = 0;
2932
2933        if (task_running(rq, p)) {
2934                schedstat_inc(p, se.nr_failed_migrations_running);
2935                return 0;
2936        }
2937
2938        /*
2939         * Aggressive migration if:
2940         * 1) task is cache cold, or
2941         * 2) too many balance attempts have failed.
2942         */
2943
2944        if (!task_hot(p, rq->clock, sd) ||
2945                        sd->nr_balance_failed > sd->cache_nice_tries) {
2946#ifdef CONFIG_SCHEDSTATS
2947                if (task_hot(p, rq->clock, sd)) {
2948                        schedstat_inc(sd, lb_hot_gained[idle]);
2949                        schedstat_inc(p, se.nr_forced_migrations);
2950                }
2951#endif
2952                return 1;
2953        }
2954
2955        if (task_hot(p, rq->clock, sd)) {
2956                schedstat_inc(p, se.nr_failed_migrations_hot);
2957                return 0;
2958        }
2959        return 1;
2960}
2961
2962static unsigned long
2963balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2964              unsigned long max_load_move, struct sched_domain *sd,
2965              enum cpu_idle_type idle, int *all_pinned,
2966              int *this_best_prio, struct rq_iterator *iterator)
2967{
2968        int loops = 0, pulled = 0, pinned = 0;
2969        struct task_struct *p;
2970        long rem_load_move = max_load_move;
2971
2972        if (max_load_move == 0)
2973                goto out;
2974
2975        pinned = 1;
2976
2977        /*
2978         * Start the load-balancing iterator:
2979         */
2980        p = iterator->start(iterator->arg);
2981next:
2982        if (!p || loops++ > sysctl_sched_nr_migrate)
2983                goto out;
2984
2985        if ((p->se.load.weight >> 1) > rem_load_move ||
2986            !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
2987                p = iterator->next(iterator->arg);
2988                goto next;
2989        }
2990
2991        pull_task(busiest, p, this_rq, this_cpu);
2992        pulled++;
2993        rem_load_move -= p->se.load.weight;
2994
2995        /*
2996         * We only want to steal up to the prescribed amount of weighted load.
2997         */
2998        if (rem_load_move > 0) {
2999                if (p->prio < *this_best_prio)
3000                        *this_best_prio = p->prio;
3001                p = iterator->next(iterator->arg);
3002                goto next;
3003        }
3004out:
3005        /*
3006         * Right now, this is one of only two places pull_task() is called,
3007         * so we can safely collect pull_task() stats here rather than
3008         * inside pull_task().
3009         */
3010        schedstat_add(sd, lb_gained[idle], pulled);
3011
3012        if (all_pinned)
3013                *all_pinned = pinned;
3014
3015        return max_load_move - rem_load_move;
3016}
3017
3018/*
3019 * move_tasks tries to move up to max_load_move weighted load from busiest to
3020 * this_rq, as part of a balancing operation within domain "sd".
3021 * Returns 1 if successful and 0 otherwise.
3022 *
3023 * Called with both runqueues locked.
3024 */
3025static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
3026                      unsigned long max_load_move,
3027                      struct sched_domain *sd, enum cpu_idle_type idle,
3028                      int *all_pinned)
3029{
3030        const struct sched_class *class = sched_class_highest;
3031        unsigned long total_load_moved = 0;
3032        int this_best_prio = this_rq->curr->prio;
3033
3034        do {
3035                total_load_moved +=
3036                        class->load_balance(this_rq, this_cpu, busiest,
3037                                max_load_move - total_load_moved,
3038                                sd, idle, all_pinned, &this_best_prio);
3039                class = class->next;
3040
3041                if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
3042                        break;
3043
3044        } while (class && max_load_move > total_load_moved);
3045
3046        return total_load_moved > 0;
3047}
3048
3049static int
3050iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3051                   struct sched_domain *sd, enum cpu_idle_type idle,
3052                   struct rq_iterator *iterator)
3053{
3054        struct task_struct *p = iterator->start(iterator->arg);
3055        int pinned = 0;
3056
3057        while (p) {
3058                if (can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
3059                        pull_task(busiest, p, this_rq, this_cpu);
3060                        /*
3061                         * Right now, this is only the second place pull_task()
3062                         * is called, so we can safely collect pull_task()
3063                         * stats here rather than inside pull_task().
3064                         */
3065                        schedstat_inc(sd, lb_gained[idle]);
3066
3067                        return 1;
3068                }
3069                p = iterator->next(iterator->arg);
3070        }
3071
3072        return 0;
3073}
3074
3075/*
3076 * move_one_task tries to move exactly one task from busiest to this_rq, as
3077 * part of active balancing operations within "domain".
3078 * Returns 1 if successful and 0 otherwise.
3079 *
3080 * Called with both runqueues locked.
3081 */
3082static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3083                         struct sched_domain *sd, enum cpu_idle_type idle)
3084{
3085        const struct sched_class *class;
3086
3087        for (class = sched_class_highest; class; class = class->next)
3088                if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
3089                        return 1;
3090
3091        return 0;
3092}
3093
3094/*
3095 * find_busiest_group finds and returns the busiest CPU group within the
3096 * domain. It calculates and returns the amount of weighted load which
3097 * should be moved to restore balance via the imbalance parameter.
3098 */
3099static struct sched_group *
3100find_busiest_group(struct sched_domain *sd, int this_cpu,
3101                   unsigned long *imbalance, enum cpu_idle_type idle,
3102                   int *sd_idle, const cpumask_t *cpus, int *balance)
3103{
3104        struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
3105        unsigned long max_load, avg_load, total_load, this_load, total_pwr;
3106        unsigned long max_pull;
3107        unsigned long busiest_load_per_task, busiest_nr_running;
3108        unsigned long this_load_per_task, this_nr_running;
3109        int load_idx, group_imb = 0;
3110#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
3111        int power_savings_balance = 1;
3112        unsigned long leader_nr_running = 0, min_load_per_task = 0;
3113        unsigned long min_nr_running = ULONG_MAX;
3114        struct sched_group *group_min = NULL, *group_leader = NULL;
3115#endif
3116
3117        max_load = this_load = total_load = total_pwr = 0;
3118        busiest_load_per_task = busiest_nr_running = 0;
3119        this_load_per_task = this_nr_running = 0;
3120
3121        if (idle == CPU_NOT_IDLE)
3122                load_idx = sd->busy_idx;
3123        else if (idle == CPU_NEWLY_IDLE)
3124                load_idx = sd->newidle_idx;
3125        else
3126                load_idx = sd->idle_idx;
3127
3128        do {
3129                unsigned long load, group_capacity, max_cpu_load, min_cpu_load;
3130                int local_group;
3131                int i;
3132                int __group_imb = 0;
3133                unsigned int balance_cpu = -1, first_idle_cpu = 0;
3134                unsigned long sum_nr_running, sum_weighted_load;
3135                unsigned long sum_avg_load_per_task;
3136                unsigned long avg_load_per_task;
3137
3138                local_group = cpu_isset(this_cpu, group->cpumask);
3139
3140                if (local_group)
3141                        balance_cpu = first_cpu(group->cpumask);
3142
3143                /* Tally up the load of all CPUs in the group */
3144                sum_weighted_load = sum_nr_running = avg_load = 0;
3145                sum_avg_load_per_task = avg_load_per_task = 0;
3146
3147                max_cpu_load = 0;
3148                min_cpu_load = ~0UL;
3149
3150                for_each_cpu_mask_nr(i, group->cpumask) {
3151                        struct rq *rq;
3152
3153                        if (!cpu_isset(i, *cpus))
3154                                continue;
3155
3156                        rq = cpu_rq(i);
3157
3158                        if (*sd_idle && rq->nr_running)
3159                                *sd_idle = 0;
3160
3161                        /* Bias balancing toward cpus of our domain */
3162                        if (local_group) {
3163                                if (idle_cpu(i) && !first_idle_cpu) {
3164                                        first_idle_cpu = 1;
3165                                        balance_cpu = i;
3166                                }
3167
3168                                load = target_load(i, load_idx);
3169                        } else {
3170                                load = source_load(i, load_idx);
3171                                if (load > max_cpu_load)
3172                                        max_cpu_load = load;
3173                                if (min_cpu_load > load)
3174                                        min_cpu_load = load;
3175                        }
3176
3177                        avg_load += load;
3178                        sum_nr_running += rq->nr_running;
3179                        sum_weighted_load += weighted_cpuload(i);
3180
3181                        sum_avg_load_per_task += cpu_avg_load_per_task(i);
3182                }
3183
3184                /*
3185                 * First idle cpu or the first cpu(busiest) in this sched group
3186                 * is eligible for doing load balancing at this and above
3187                 * domains. In the newly idle case, we will allow all the cpu's
3188                 * to do the newly idle load balance.
3189                 */
3190                if (idle != CPU_NEWLY_IDLE && local_group &&
3191                    balance_cpu != this_cpu && balance) {
3192                        *balance = 0;
3193                        goto ret;
3194                }
3195
3196                total_load += avg_load;
3197                total_pwr += group->__cpu_power;
3198
3199                /* Adjust by relative CPU power of the group */
3200                avg_load = sg_div_cpu_power(group,
3201                                avg_load * SCHED_LOAD_SCALE);
3202
3203
3204                /*
3205                 * Consider the group unbalanced when the imbalance is larger
3206                 * than the average weight of two tasks.
3207                 *
3208                 * APZ: with cgroup the avg task weight can vary wildly and
3209                 *      might not be a suitable number - should we keep a
3210                 *      normalized nr_running number somewhere that negates
3211                 *      the hierarchy?
3212                 */
3213                avg_load_per_task = sg_div_cpu_power(group,
3214                                sum_avg_load_per_task * SCHED_LOAD_SCALE);
3215
3216                if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
3217                        __group_imb = 1;
3218
3219                group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
3220
3221                if (local_group) {
3222                        this_load = avg_load;
3223                        this = group;
3224                        this_nr_running = sum_nr_running;
3225                        this_load_per_task = sum_weighted_load;
3226                } else if (avg_load > max_load &&
3227                           (sum_nr_running > group_capacity || __group_imb)) {
3228                        max_load = avg_load;
3229                        busiest = group;
3230                        busiest_nr_running = sum_nr_running;
3231                        busiest_load_per_task = sum_weighted_load;
3232                        group_imb = __group_imb;
3233                }
3234
3235#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
3236                /*
3237                 * Busy processors will not participate in power savings
3238                 * balance.
3239                 */
3240                if (idle == CPU_NOT_IDLE ||
3241                                !(sd->flags & SD_POWERSAVINGS_BALANCE))
3242                        goto group_next;
3243
3244                /*
3245                 * If the local group is idle or completely loaded
3246                 * no need to do power savings balance at this domain
3247                 */
3248                if (local_group && (this_nr_running >= group_capacity ||
3249                                    !this_nr_running))
3250                        power_savings_balance = 0;
3251
3252                /*
3253                 * If a group is already running at full capacity or idle,
3254                 * don't include that group in power savings calculations
3255                 */
3256                if (!power_savings_balance || sum_nr_running >= group_capacity
3257                    || !sum_nr_running)
3258                        goto group_next;
3259
3260                /*
3261                 * Calculate the group which has the least non-idle load.
3262                 * This is the group from where we need to pick up the load
3263                 * for saving power
3264                 */
3265                if ((sum_nr_running < min_nr_running) ||
3266                    (sum_nr_running == min_nr_running &&
3267                     first_cpu(group->cpumask) <
3268                     first_cpu(group_min->cpumask))) {
3269                        group_min = group;
3270                        min_nr_running = sum_nr_running;
3271                        min_load_per_task = sum_weighted_load /
3272                                                sum_nr_running;
3273                }
3274
3275                /*
3276                 * Calculate the group which is almost near its
3277                 * capacity but still has some space to pick up some load
3278                 * from other group and save more power
3279                 */
3280                if (sum_nr_running <= group_capacity - 1) {
3281                        if (sum_nr_running > leader_nr_running ||
3282                            (sum_nr_running == leader_nr_running &&
3283                             first_cpu(group->cpumask) >
3284                              first_cpu(group_leader->cpumask))) {
3285                                group_leader = group;
3286                                leader_nr_running = sum_nr_running;
3287                        }
3288                }
3289group_next:
3290#endif
3291                group = group->next;
3292        } while (group != sd->groups);
3293
3294        if (!busiest || this_load >= max_load || busiest_nr_running == 0)
3295                goto out_balanced;
3296
3297        avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr;
3298
3299        if (this_load >= avg_load ||
3300                        100*max_load <= sd->imbalance_pct*this_load)
3301                goto out_balanced;
3302
3303        busiest_load_per_task /= busiest_nr_running;
3304        if (group_imb)
3305                busiest_load_per_task = min(busiest_load_per_task, avg_load);
3306
3307        /*
3308         * We're trying to get all the cpus to the average_load, so we don't
3309         * want to push ourselves above the average load, nor do we wish to
3310         * reduce the max loaded cpu below the average load, as either of these
3311         * actions would just result in more rebalancing later, and ping-pong
3312         * tasks around. Thus we look for the minimum possible imbalance.
3313         * Negative imbalances (*we* are more loaded than anyone else) will
3314         * be counted as no imbalance for these purposes -- we can't fix that
3315         * by pulling tasks to us. Be careful of negative numbers as they'll
3316         * appear as very large values with unsigned longs.
3317         */
3318        if (max_load <= busiest_load_per_task)
3319                goto out_balanced;
3320
3321        /*
3322         * In the presence of smp nice balancing, certain scenarios can have
3323         * max load less than avg load(as we skip the groups at or below
3324         * its cpu_power, while calculating max_load..)
3325         */
3326        if (max_load < avg_load) {
3327                *imbalance = 0;
3328                goto small_imbalance;
3329        }
3330
3331        /* Don't want to pull so many tasks that a group would go idle */
3332        max_pull = min(max_load - avg_load, max_load - busiest_load_per_task);
3333
3334        /* How much load to actually move to equalise the imbalance */
3335        *imbalance = min(max_pull * busiest->__cpu_power,
3336                                (avg_load - this_load) * this->__cpu_power)
3337                        / SCHED_LOAD_SCALE;
3338
3339        /*
3340         * if *imbalance is less than the average load per runnable task
3341         * there is no gaurantee that any tasks will be moved so we'll have
3342         * a think about bumping its value to force at least one task to be
3343         * moved
3344         */
3345        if (*imbalance < busiest_load_per_task) {
3346                unsigned long tmp, pwr_now, pwr_move;
3347                unsigned int imbn;
3348
3349small_imbalance:
3350                pwr_move = pwr_now = 0;
3351                imbn = 2;
3352                if (this_nr_running) {
3353                        this_load_per_task /= this_nr_running;
3354                        if (busiest_load_per_task > this_load_per_task)
3355                                imbn = 1;
3356                } else
3357                        this_load_per_task = cpu_avg_load_per_task(this_cpu);
3358
3359                if (max_load - this_load + busiest_load_per_task >=
3360                                        busiest_load_per_task * imbn) {
3361                        *imbalance = busiest_load_per_task;
3362                        return busiest;
3363                }
3364
3365                /*
3366                 * OK, we don't have enough imbalance to justify moving tasks,
3367                 * however we may be able to increase total CPU power used by
3368                 * moving them.
3369                 */
3370
3371                pwr_now += busiest->__cpu_power *
3372                                min(busiest_load_per_task, max_load);
3373                pwr_now += this->__cpu_power *
3374                                min(this_load_per_task, this_load);
3375                pwr_now /= SCHED_LOAD_SCALE;
3376
3377                /* Amount of load we'd subtract */
3378                tmp = sg_div_cpu_power(busiest,
3379                                busiest_load_per_task * SCHED_LOAD_SCALE);
3380                if (max_load > tmp)
3381                        pwr_move += busiest->__cpu_power *
3382                                min(busiest_load_per_task, max_load - tmp);
3383
3384                /* Amount of load we'd add */
3385                if (max_load * busiest->__cpu_power <
3386                                busiest_load_per_task * SCHED_LOAD_SCALE)
3387                        tmp = sg_div_cpu_power(this,
3388                                        max_load * busiest->__cpu_power);
3389                else
3390                        tmp = sg_div_cpu_power(this,
3391                                busiest_load_per_task * SCHED_LOAD_SCALE);
3392                pwr_move += this->__cpu_power *
3393                                min(this_load_per_task, this_load + tmp);
3394                pwr_move /= SCHED_LOAD_SCALE;
3395
3396                /* Move if we gain throughput */
3397                if (pwr_move > pwr_now)
3398                        *imbalance = busiest_load_per_task;
3399        }
3400
3401        return busiest;
3402
3403out_balanced:
3404#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
3405        if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
3406                goto ret;
3407
3408        if (this == group_leader && group_leader != group_min) {
3409                *imbalance = min_load_per_task;
3410                return group_min;
3411        }
3412#endif
3413ret:
3414        *imbalance = 0;
3415        return NULL;
3416}
3417
3418/*
3419 * find_busiest_queue - find the busiest runqueue among the cpus in group.
3420 */
3421static struct rq *
3422find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3423                   unsigned long imbalance, const cpumask_t *cpus)
3424{
3425        struct rq *busiest = NULL, *rq;
3426        unsigned long max_load = 0;
3427        int i;
3428
3429        for_each_cpu_mask_nr(i, group->cpumask) {
3430                unsigned long wl;
3431
3432                if (!cpu_isset(i, *cpus))
3433                        continue;
3434
3435                rq = cpu_rq(i);
3436                wl = weighted_cpuload(i);
3437
3438                if (rq->nr_running == 1 && wl > imbalance)
3439                        continue;
3440
3441                if (wl > max_load) {
3442                        max_load = wl;
3443                        busiest = rq;
3444                }
3445        }
3446
3447        return busiest;
3448}
3449
3450/*
3451 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
3452 * so long as it is large enough.
3453 */
3454#define MAX_PINNED_INTERVAL        512
3455
3456/*
3457 * Check this_cpu to ensure it is balanced within domain. Attempt to move
3458 * tasks if there is an imbalance.
3459 */
3460static int load_balance(int this_cpu, struct rq *this_rq,
3461                        struct sched_domain *sd, enum cpu_idle_type idle,
3462                        int *balance, cpumask_t *cpus)
3463{
3464        int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
3465        struct sched_group *group;
3466        unsigned long imbalance;
3467        struct rq *busiest;
3468        unsigned long flags;
3469
3470        cpus_setall(*cpus);
3471
3472        /*
3473         * When power savings policy is enabled for the parent domain, idle
3474         * sibling can pick up load irrespective of busy siblings. In this case,
3475         * let the state of idle sibling percolate up as CPU_IDLE, instead of
3476         * portraying it as CPU_NOT_IDLE.
3477         */
3478        if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
3479            !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3480                sd_idle = 1;
3481
3482        schedstat_inc(sd, lb_count[idle]);
3483
3484redo:
3485        update_shares(sd);
3486        group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
3487                                   cpus, balance);
3488
3489        if (*balance == 0)
3490                goto out_balanced;
3491
3492        if (!group) {
3493                schedstat_inc(sd, lb_nobusyg[idle]);
3494                goto out_balanced;
3495        }
3496
3497        busiest = find_busiest_queue(group, idle, imbalance, cpus);
3498        if (!busiest) {
3499                schedstat_inc(sd, lb_nobusyq[idle]);
3500                goto out_balanced;
3501        }
3502
3503        BUG_ON(busiest == this_rq);
3504
3505        schedstat_add(sd, lb_imbalance[idle], imbalance);
3506
3507        ld_moved = 0;
3508        if (busiest->nr_running > 1) {
3509                /*
3510                 * Attempt to move tasks. If find_busiest_group has found
3511                 * an imbalance but busiest->nr_running <= 1, the group is
3512                 * still unbalanced. ld_moved simply stays zero, so it is
3513                 * correctly treated as an imbalance.
3514                 */
3515                local_irq_save(flags);
3516                double_rq_lock(this_rq, busiest);
3517                ld_moved = move_tasks(this_rq, this_cpu, busiest,
3518                                      imbalance, sd, idle, &all_pinned);
3519                double_rq_unlock(this_rq, busiest);
3520                local_irq_restore(flags);
3521
3522                /*
3523                 * some other cpu did the load balance for us.
3524                 */
3525                if (ld_moved && this_cpu != smp_processor_id())
3526                        resched_cpu(this_cpu);
3527
3528                /* All tasks on this runqueue were pinned by CPU affinity */
3529                if (unlikely(all_pinned)) {
3530                        cpu_clear(cpu_of(busiest), *cpus);
3531                        if (!cpus_empty(*cpus))
3532                                goto redo;
3533                        goto out_balanced;
3534                }
3535        }
3536
3537        if (!ld_moved) {
3538                schedstat_inc(sd, lb_failed[idle]);
3539                sd->nr_balance_failed++;
3540
3541                if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {
3542
3543                        spin_lock_irqsave(&busiest->lock, flags);
3544
3545                        /* don't kick the migration_thread, if the curr
3546                         * task on busiest cpu can't be moved to this_cpu
3547                         */
3548                        if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) {
3549                                spin_unlock_irqrestore(&busiest->lock, flags);
3550                                all_pinned = 1;
3551                                goto out_one_pinned;
3552                        }
3553
3554                        if (!busiest->active_balance) {
3555                                busiest->active_balance = 1;
3556                                busiest->push_cpu = this_cpu;
3557                                active_balance = 1;
3558                        }
3559                        spin_unlock_irqrestore(&busiest->lock, flags);
3560                        if (active_balance)
3561                                wake_up_process(busiest->migration_thread);
3562
3563                        /*
3564                         * We've kicked active balancing, reset the failure
3565                         * counter.
3566                         */
3567                        sd->nr_balance_failed = sd->cache_nice_tries+1;
3568                }
3569        } else
3570                sd->nr_balance_failed = 0;
3571
3572        if (likely(!active_balance)) {
3573                /* We were unbalanced, so reset the balancing interval */
3574                sd->balance_interval = sd->min_interval;
3575        } else {
3576                /*
3577                 * If we've begun active balancing, start to back off. This
3578                 * case may not be covered by the all_pinned logic if there
3579                 * is only 1 task on the busy runqueue (because we don't call
3580                 * move_tasks).
3581                 */
3582                if (sd->balance_interval < sd->max_interval)
3583                        sd->balance_interval *= 2;
3584        }
3585
3586        if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3587            !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3588                ld_moved = -1;
3589
3590        goto out;
3591
3592out_balanced:
3593        schedstat_inc(sd, lb_balanced[idle]);
3594
3595        sd->nr_balance_failed = 0;
3596
3597out_one_pinned:
3598        /* tune up the balancing interval */
3599        if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
3600                        (sd->balance_interval < sd->max_interval))
3601                sd->balance_interval *= 2;
3602
3603        if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3604            !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3605                ld_moved = -1;
3606        else
3607                ld_moved = 0;
3608out:
3609        if (ld_moved)
3610                update_shares(sd);
3611        return ld_moved;
3612}
3613
3614/*
3615 * Check this_cpu to ensure it is balanced within domain. Attempt to move
3616 * tasks if there is an imbalance.
3617 *
3618 * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
3619 * this_rq is locked.
3620 */
3621static int
3622load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3623                        cpumask_t *cpus)
3624{
3625        struct sched_group *group;
3626        struct rq *busiest = NULL;
3627        unsigned long imbalance;
3628        int ld_moved = 0;
3629        int sd_idle = 0;
3630        int all_pinned = 0;
3631
3632        cpus_setall(*cpus);
3633
3634        /*
3635         * When power savings policy is enabled for the parent domain, idle
3636         * sibling can pick up load irrespective of busy siblings. In this case,
3637         * let the state of idle sibling percolate up as IDLE, instead of
3638         * portraying it as CPU_NOT_IDLE.
3639         */
3640        if (sd->flags & SD_SHARE_CPUPOWER &&
3641            !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3642                sd_idle = 1;
3643
3644        schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
3645redo:
3646        update_shares_locked(this_rq, sd);
3647        group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
3648                                   &sd_idle, cpus, NULL);
3649        if (!group) {
3650                schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
3651                goto out_balanced;
3652        }
3653
3654        busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance, cpus);
3655        if (!busiest) {
3656                schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
3657                goto out_balanced;
3658        }
3659
3660        BUG_ON(busiest == this_rq);
3661
3662        schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
3663
3664        ld_moved = 0;
3665        if (busiest->nr_running > 1) {
3666                /* Attempt to move tasks */
3667                double_lock_balance(this_rq, busiest);
3668                /* this_rq->clock is already updated */
3669                update_rq_clock(busiest);
3670                ld_moved = move_tasks(this_rq, this_cpu, busiest,
3671                                        imbalance, sd, CPU_NEWLY_IDLE,
3672                                        &all_pinned);
3673                double_unlock_balance(this_rq, busiest);
3674
3675                if (unlikely(all_pinned)) {
3676                        cpu_clear(cpu_of(busiest), *cpus);
3677                        if (!cpus_empty(*cpus))
3678                                goto redo;
3679                }
3680        }
3681
3682        if (!ld_moved) {
3683                schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
3684                if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3685                    !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3686                        return -1;
3687        } else
3688                sd->nr_balance_failed = 0;
3689
3690        update_shares_locked(this_rq, sd);
3691        return ld_moved;
3692
3693out_balanced:
3694        schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
3695        if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3696            !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3697                return -1;
3698        sd->nr_balance_failed = 0;
3699
3700        return 0;
3701}
3702
3703/*
3704 * idle_balance is called by schedule() if this_cpu is about to become
3705 * idle. Attempts to pull tasks from other CPUs.
3706 */
3707static void idle_balance(int this_cpu, struct rq *this_rq)
3708{
3709        struct sched_domain *sd;
3710        int pulled_task = -1;
3711        unsigned long next_balance = jiffies + HZ;
3712        cpumask_t tmpmask;
3713
3714        for_each_domain(this_cpu, sd) {
3715                unsigned long interval;
3716
3717                if (!(sd->flags & SD_LOAD_BALANCE))
3718                        continue;
3719
3720                if (sd->flags & SD_BALANCE_NEWIDLE)
3721                        /* If we've pulled tasks over stop searching: */
3722                        pulled_task = load_balance_newidle(this_cpu, this_rq,
3723                                                           sd, &tmpmask);
3724
3725                interval = msecs_to_jiffies(sd->balance_interval);
3726                if (time_after(next_balance, sd->last_balance + interval))
3727                        next_balance = sd->last_balance + interval;
3728                if (pulled_task)
3729                        break;
3730        }
3731        if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
3732                /*
3733                 * We are going idle. next_balance may be set based on
3734                 * a busy processor. So reset next_balance.
3735                 */
3736                this_rq->next_balance = next_balance;
3737        }
3738}
3739
3740/*
3741 * active_load_balance is run by migration threads. It pushes running tasks
3742 * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
3743 * running on each physical CPU where possible, and avoids physical /
3744 * logical imbalances.
3745 *
3746 * Called with busiest_rq locked.
3747 */
3748static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3749{
3750        int target_cpu = busiest_rq->push_cpu;
3751        struct sched_domain *sd;
3752        struct rq *target_rq;
3753
3754        /* Is there any task to move? */
3755        if (busiest_rq->nr_running <= 1)
3756                return;
3757
3758        target_rq = cpu_rq(target_cpu);
3759
3760        /*
3761         * This condition is "impossible", if it occurs
3762         * we need to fix it. Originally reported by
3763         * Bjorn Helgaas on a 128-cpu setup.
3764         */
3765        BUG_ON(busiest_rq == target_rq);
3766
3767        /* move a task from busiest_rq to target_rq */
3768        double_lock_balance(busiest_rq, target_rq);
3769        update_rq_clock(busiest_rq);
3770        update_rq_clock(target_rq);
3771
3772        /* Search for an sd spanning us and the target CPU. */
3773        for_each_domain(target_cpu, sd) {
3774                if ((sd->flags & SD_LOAD_BALANCE) &&
3775                    cpu_isset(busiest_cpu, sd->span))
3776                                break;
3777        }
3778
3779        if (likely(sd)) {
3780                schedstat_inc(sd, alb_count);
3781
3782                if (move_one_task(target_rq, target_cpu, busiest_rq,
3783                                  sd, CPU_IDLE))
3784                        schedstat_inc(sd, alb_pushed);
3785                else
3786                        schedstat_inc(sd, alb_failed);
3787        }
3788        double_unlock_balance(busiest_rq, target_rq);
3789}
3790
3791#ifdef CONFIG_NO_HZ
3792static struct {
3793        atomic_t load_balancer;
3794        cpumask_t cpu_mask;
3795} nohz ____cacheline_aligned = {
3796        .load_balancer = ATOMIC_INIT(-1),
3797        .cpu_mask = CPU_MASK_NONE,
3798};
3799
3800/*
3801 * This routine will try to nominate the ilb (idle load balancing)
3802 * owner among the cpus whose ticks are stopped. ilb owner will do the idle
3803 * load balancing on behalf of all those cpus. If all the cpus in the system
3804 * go into this tickless mode, then there will be no ilb owner (as there is
3805 * no need for one) and all the cpus will sleep till the next wakeup event
3806 * arrives...
3807 *
3808 * For the ilb owner, tick is not stopped. And this tick will be used
3809 * for idle load balancing. ilb owner will still be part of
3810 * nohz.cpu_mask..
3811 *
3812 * While stopping the tick, this cpu will become the ilb owner if there
3813 * is no other owner. And will be the owner till that cpu becomes busy
3814 * or if all cpus in the system stop their ticks at which point
3815 * there is no need for ilb owner.
3816 *
3817 * When the ilb owner becomes busy, it nominates another owner, during the
3818 * next busy scheduler_tick()
3819 */
3820int select_nohz_load_balancer(int stop_tick)
3821{
3822        int cpu = smp_processor_id();
3823
3824        if (stop_tick) {
3825                cpu_set(cpu, nohz.cpu_mask);
3826                cpu_rq(cpu)->in_nohz_recently = 1;
3827
3828                /*
3829                 * If we are going offline and still the leader, give up!
3830                 */
3831                if (!cpu_active(cpu) &&
3832                    atomic_read(&nohz.load_balancer) == cpu) {
3833                        if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
3834                                BUG();
3835                        return 0;
3836                }
3837
3838                /* time for ilb owner also to sleep */
3839                if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
3840                        if (atomic_read(&nohz.load_balancer) == cpu)
3841                                atomic_set(&nohz.load_balancer, -1);
3842                        return 0;
3843                }
3844
3845                if (atomic_read(&nohz.load_balancer) == -1) {
3846                        /* make me the ilb owner */
3847                        if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
3848                                return 1;
3849                } else if (atomic_read(&nohz.load_balancer) == cpu)
3850                        return 1;
3851        } else {
3852                if (!cpu_isset(cpu, nohz.cpu_mask))
3853                        return 0;
3854
3855                cpu_clear(cpu, nohz.cpu_mask);
3856
3857                if (atomic_read(&nohz.load_balancer) == cpu)
3858                        if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
3859                                BUG();
3860        }
3861        return 0;
3862}
3863#endif
3864
3865static DEFINE_SPINLOCK(balancing);
3866
3867/*
3868 * It checks each scheduling domain to see if it is due to be balanced,
3869 * and initiates a balancing operation if so.
3870 *
3871 * Balancing parameters are set up in arch_init_sched_domains.
3872 */
3873static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3874{
3875        int balance = 1;
3876        struct rq *rq = cpu_rq(cpu);
3877        unsigned long interval;
3878        struct sched_domain *sd;
3879        /* Earliest time when we have to do rebalance again */
3880        unsigned long next_balance = jiffies + 60*HZ;
3881        int update_next_balance = 0;
3882        int need_serialize;
3883        cpumask_t tmp;
3884
3885        for_each_domain(cpu, sd) {
3886                if (!(sd->flags & SD_LOAD_BALANCE))
3887                        continue;
3888
3889                interval = sd->balance_interval;
3890                if (idle != CPU_IDLE)
3891                        interval *= sd->busy_factor;
3892
3893                /* scale ms to jiffies */
3894                interval = msecs_to_jiffies(interval);
3895                if (unlikely(!interval))
3896                        interval = 1;
3897                if (interval > HZ*NR_CPUS/10)
3898                        interval = HZ*NR_CPUS/10;
3899
3900                need_serialize = sd->flags & SD_SERIALIZE;
3901
3902                if (need_serialize) {
3903                        if (!spin_trylock(&balancing))
3904                                goto out;
3905                }
3906
3907                if (time_after_eq(jiffies, sd->last_balance + interval)) {
3908                        if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) {
3909                                /*
3910                                 * We've pulled tasks over so either we're no
3911                                 * longer idle, or one of our SMT siblings is
3912                                 * not idle.
3913                                 */
3914                                idle = CPU_NOT_IDLE;
3915                        }
3916                        sd->last_balance = jiffies;
3917                }
3918                if (need_serialize)
3919                        spin_unlock(&balancing);
3920out:
3921                if (time_after(next_balance, sd->last_balance + interval)) {
3922                        next_balance = sd->last_balance + interval;
3923                        update_next_balance = 1;
3924                }
3925
3926                /*
3927                 * Stop the load balance at this level. There is another
3928                 * CPU in our sched group which is doing load balancing more
3929                 * actively.
3930                 */
3931                if (!balance)
3932                        break;
3933        }
3934
3935        /*
3936         * next_balance will be updated only when there is a need.
3937         * When the cpu is attached to null domain for ex, it will not be
3938         * updated.
3939         */
3940        if (likely(update_next_balance))
3941                rq->next_balance = next_balance;
3942}
3943
3944/*
3945 * run_rebalance_domains is triggered when needed from the scheduler tick.
3946 * In CONFIG_NO_HZ case, the idle load balance owner will do the
3947 * rebalancing for all the cpus for whom scheduler ticks are stopped.
3948 */
3949static void run_rebalance_domains(struct softirq_action *h)
3950{
3951        int this_cpu = smp_processor_id();
3952        struct rq *this_rq = cpu_rq(this_cpu);
3953        enum cpu_idle_type idle = this_rq->idle_at_tick ?
3954                                                CPU_IDLE : CPU_NOT_IDLE;
3955
3956        rebalance_domains(this_cpu, idle);
3957
3958#ifdef CONFIG_NO_HZ
3959        /*
3960         * If this cpu is the owner for idle load balancing, then do the
3961         * balancing on behalf of the other idle cpus whose ticks are
3962         * stopped.
3963         */
3964        if (this_rq->idle_at_tick &&
3965            atomic_read(&nohz.load_balancer) == this_cpu) {
3966                cpumask_t cpus = nohz.cpu_mask;
3967                struct rq *rq;
3968                int balance_cpu;
3969
3970                cpu_clear(this_cpu, cpus);
3971                for_each_cpu_mask_nr(balance_cpu, cpus) {
3972                        /*
3973                         * If this cpu gets work to do, stop the load balancing
3974                         * work being done for other cpus. Next load
3975                         * balancing owner will pick it up.
3976                         */
3977                        if (need_resched())
3978                                break;
3979
3980                        rebalance_domains(balance_cpu, CPU_IDLE);
3981
3982                        rq = cpu_rq(balance_cpu);
3983                        if (time_after(this_rq->next_balance, rq->next_balance))
3984                                this_rq->next_balance = rq->next_balance;
3985                }
3986        }
3987#endif
3988}
3989
3990/*
3991 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
3992 *
3993 * In case of CONFIG_NO_HZ, this is the place where we nominate a new
3994 * idle load balancing owner or decide to stop the periodic load balancing,
3995 * if the whole system is idle.
3996 */
3997static inline void trigger_load_balance(struct rq *rq, int cpu)
3998{
3999#ifdef CONFIG_NO_HZ
4000        /*
4001         * If we were in the nohz mode recently and busy at the current
4002         * scheduler tick, then check if we need to nominate new idle
4003         * load balancer.
4004         */
4005        if (rq->in_nohz_recently && !rq->idle_at_tick) {
4006                rq->in_nohz_recently = 0;
4007
4008                if (atomic_read(&nohz.load_balancer) == cpu) {
4009                        cpu_clear(cpu, nohz.cpu_mask);
4010                        atomic_set(&nohz.load_balancer, -1);
4011                }
4012
4013                if (atomic_read(&nohz.load_balancer) == -1) {
4014                        /*
4015                         * simple selection for now: Nominate the
4016                         * first cpu in the nohz list to be the next
4017                         * ilb owner.
4018                         *
4019                         * TBD: Traverse the sched domains and nominate
4020                         * the nearest cpu in the nohz.cpu_mask.
4021                         */
4022                        int ilb = first_cpu(nohz.cpu_mask);
4023
4024                        if (ilb < nr_cpu_ids)
4025                                resched_cpu(ilb);
4026                }
4027        }
4028
4029        /*
4030         * If this cpu is idle and doing idle load balancing for all the
4031         * cpus with ticks stopped, is it time for that to stop?
4032         */
4033        if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4034            cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
4035                resched_cpu(cpu);
4036                return;
4037        }
4038
4039        /*
4040         * If this cpu is idle and the idle load balancing is done by
4041         * someone else, then no need raise the SCHED_SOFTIRQ
4042         */
4043        if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4044            cpu_isset(cpu, nohz.cpu_mask))
4045                return;
4046#endif
4047        if (time_after_eq(jiffies, rq->next_balance))
4048                raise_softirq(SCHED_SOFTIRQ);
4049}
4050
4051#else        /* CONFIG_SMP */
4052
4053/*
4054 * on UP we do not need to balance between CPUs:
4055 */
4056static inline void idle_balance(int cpu, struct rq *rq)
4057{
4058}
4059
4060#endif
4061
4062DEFINE_PER_CPU(struct kernel_stat, kstat);
4063
4064EXPORT_PER_CPU_SYMBOL(kstat);
4065
4066/*
4067 * Return any ns on the sched_clock that have not yet been banked in
4068 * @p in case that task is currently running.
4069 */
4070unsigned long long task_delta_exec(struct task_struct *p)
4071{
4072        unsigned long flags;
4073        struct rq *rq;
4074        u64 ns = 0;
4075
4076        rq = task_rq_lock(p, &flags);
4077
4078        if (task_current(rq, p)) {
4079                u64 delta_exec;
4080
4081                update_rq_clock(rq);
4082                delta_exec = rq->clock - p->se.exec_start;
4083                if ((s64)delta_exec > 0)
4084                        ns = delta_exec;
4085        }
4086
4087        task_rq_unlock(rq, &flags);
4088
4089        return ns;
4090}
4091
4092/*
4093 * Account user cpu time to a process.
4094 * @p: the process that the cpu time gets accounted to
4095 * @cputime: the cpu time spent in user space since the last update
4096 */
4097void account_user_time(struct task_struct *p, cputime_t cputime)
4098{
4099        struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4100        cputime64_t tmp;
4101
4102        p->utime = cputime_add(p->utime, cputime);
4103        account_group_user_time(p, cputime);
4104
4105        /* Add user time to cpustat. */
4106        tmp = cputime_to_cputime64(cputime);
4107        if (TASK_NICE(p) > 0)
4108                cpustat->nice = cputime64_add(cpustat->nice, tmp);
4109        else
4110                cpustat->user = cputime64_add(cpustat->user, tmp);
4111        /* Account for user time used */
4112        acct_update_integrals(p);
4113}
4114
4115/*
4116 * Account guest cpu time to a process.
4117 * @p: the process that the cpu time gets accounted to
4118 * @cputime: the cpu time spent in virtual machine since the last update
4119 */
4120static void account_guest_time(struct task_struct *p, cputime_t cputime)
4121{
4122        cputime64_t tmp;
4123        struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4124
4125        tmp = cputime_to_cputime64(cputime);
4126
4127        p->utime = cputime_add(p->utime, cputime);
4128        account_group_user_time(p, cputime);
4129        p->gtime = cputime_add(p->gtime, cputime);
4130
4131        cpustat->user = cputime64_add(cpustat->user, tmp);
4132        cpustat->guest = cputime64_add(cpustat->guest, tmp);
4133}
4134
4135/*
4136 * Account scaled user cpu time to a process.
4137 * @p: the process that the cpu time gets accounted to
4138 * @cputime: the cpu time spent in user space since the last update
4139 */
4140void account_user_time_scaled(struct task_struct *p, cputime_t cputime)
4141{
4142        p->utimescaled = cputime_add(p->utimescaled, cputime);
4143}
4144
4145/*
4146 * Account system cpu time to a process.
4147 * @p: the process that the cpu time gets accounted to
4148 * @hardirq_offset: the offset to subtract from hardirq_count()
4149 * @cputime: the cpu time spent in kernel space since the last update
4150 */
4151void account_system_time(struct task_struct *p, int hardirq_offset,
4152                         cputime_t cputime)
4153{
4154        struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4155        struct rq *rq = this_rq();
4156        cputime64_t tmp;
4157
4158        if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
4159                account_guest_time(p, cputime);
4160                return;
4161        }
4162
4163        p->stime = cputime_add(p->stime, cputime);
4164        account_group_system_time(p, cputime);
4165
4166        /* Add system time to cpustat. */
4167        tmp = cputime_to_cputime64(cputime);
4168        if (hardirq_count() - hardirq_offset)
4169                cpustat->irq = cputime64_add(cpustat->irq, tmp);
4170        else if (softirq_count())
4171                cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
4172        else if (p != rq->idle)
4173                cpustat->system = cputime64_add(cpustat->system, tmp);
4174        else if (atomic_read(&rq->nr_iowait) > 0)
4175                cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4176        else
4177                cpustat->idle = cputime64_add(cpustat->idle, tmp);
4178        /* Account for system time used */
4179        acct_update_integrals(p);
4180}
4181
4182/*
4183 * Account scaled system cpu time to a process.
4184 * @p: the process that the cpu time gets accounted to
4185 * @hardirq_offset: the offset to subtract from hardirq_count()
4186 * @cputime: the cpu time spent in kernel space since the last update
4187 */
4188void account_system_time_scaled(struct task_struct *p, cputime_t cputime)
4189{
4190        p->stimescaled = cputime_add(p->stimescaled, cputime);
4191}
4192
4193/*
4194 * Account for involuntary wait time.
4195 * @p: the process from which the cpu time has been stolen
4196 * @steal: the cpu time spent in involuntary wait
4197 */
4198void account_steal_time(struct task_struct *p, cputime_t steal)
4199{
4200        struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4201        cputime64_t tmp = cputime_to_cputime64(steal);
4202        struct rq *rq = this_rq();
4203
4204        if (p == rq->idle) {
4205                p->stime = cputime_add(p->stime, steal);
4206                account_group_system_time(p, steal);
4207                if (atomic_read(&rq->nr_iowait) > 0)
4208                        cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4209                else
4210                        cpustat->idle = cputime64_add(cpustat->idle, tmp);
4211        } else
4212                cpustat->steal = cputime64_add(cpustat->steal, tmp);
4213}
4214
4215/*
4216 * Use precise platform statistics if available:
4217 */
4218#ifdef CONFIG_VIRT_CPU_ACCOUNTING
4219cputime_t task_utime(struct task_struct *p)
4220{
4221        return p->utime;
4222}
4223
4224cputime_t task_stime(struct task_struct *p)
4225{
4226        return p->stime;
4227}
4228#else
4229cputime_t task_utime(struct task_struct *p)
4230{
4231        clock_t utime = cputime_to_clock_t(p->utime),
4232                total = utime + cputime_to_clock_t(p->stime);
4233        u64 temp;
4234
4235        /*
4236         * Use CFS's precise accounting:
4237         */
4238        temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime);
4239
4240        if (total) {
4241                temp *= utime;
4242                do_div(temp, total);
4243        }
4244        utime = (clock_t)temp;
4245
4246        p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime));
4247        return p->prev_utime;
4248}
4249
4250cputime_t task_stime(struct task_struct *p)
4251{
4252        clock_t stime;
4253
4254        /*
4255         * Use CFS's precise accounting. (we subtract utime from
4256         * the total, to make sure the total observed by userspace
4257         * grows monotonically - apps rely on that):
4258         */
4259        stime = nsec_to_clock_t(p->se.sum_exec_runtime) -
4260                        cputime_to_clock_t(task_utime(p));
4261
4262        if (stime >= 0)
4263                p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime));
4264
4265        return p->prev_stime;
4266}
4267#endif
4268
4269inline cputime_t task_gtime(struct task_struct *p)
4270{
4271        return p->gtime;
4272}
4273
4274/*
4275 * This function gets called by the timer code, with HZ frequency.
4276 * We call it with interrupts disabled.
4277 *
4278 * It also gets called by the fork code, when changing the parent's
4279 * timeslices.
4280 */
4281void scheduler_tick(void)
4282{
4283        int cpu = smp_processor_id();
4284        struct rq *rq = cpu_rq(cpu);
4285        struct task_struct *curr = rq->curr;
4286
4287        sched_clock_tick();
4288
4289        spin_lock(&rq->lock);
4290        update_rq_clock(rq);
4291        update_cpu_load(rq);
4292        curr->sched_class->task_tick(rq, curr, 0);
4293        spin_unlock(&rq->lock);
4294
4295#ifdef CONFIG_SMP
4296        rq->idle_at_tick = idle_cpu(cpu);
4297        trigger_load_balance(rq, cpu);
4298#endif
4299}
4300
4301#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
4302                                defined(CONFIG_PREEMPT_TRACER))
4303
4304static inline unsigned long get_parent_ip(unsigned long addr)
4305{
4306        if (in_lock_functions(addr)) {
4307                addr = CALLER_ADDR2;
4308                if (in_lock_functions(addr))
4309                        addr = CALLER_ADDR3;
4310        }
4311        return addr;
4312}
4313
4314void __kprobes add_preempt_count(int val)
4315{
4316#ifdef CONFIG_DEBUG_PREEMPT
4317        /*
4318         * Underflow?
4319         */
4320        if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
4321                return;
4322#endif
4323        preempt_count() += val;
4324#ifdef CONFIG_DEBUG_PREEMPT
4325        /*
4326         * Spinlock count overflowing soon?
4327         */
4328        DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
4329                                PREEMPT_MASK - 10);
4330#endif
4331        if (preempt_count() == val)
4332                trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
4333}
4334EXPORT_SYMBOL(add_preempt_count);
4335
4336void __kprobes sub_preempt_count(int val)
4337{
4338#ifdef CONFIG_DEBUG_PREEMPT
4339        /*
4340         * Underflow?
4341         */
4342        if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
4343                return;
4344        /*
4345         * Is the spinlock portion underflowing?
4346         */
4347        if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
4348                        !(preempt_count() & PREEMPT_MASK)))
4349                return;
4350#endif
4351
4352        if (preempt_count() == val)
4353                trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
4354        preempt_count() -= val;
4355}
4356EXPORT_SYMBOL(sub_preempt_count);
4357
4358#endif
4359
4360/*
4361 * Print scheduling while atomic bug:
4362 */
4363static noinline void __schedule_bug(struct task_struct *prev)
4364{
4365        struct pt_regs *regs = get_irq_regs();
4366
4367        printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
4368                prev->comm, prev->pid, preempt_count());
4369
4370        debug_show_held_locks(prev);
4371        print_modules();
4372        if (irqs_disabled())
4373                print_irqtrace_events(prev);
4374
4375        if (regs)
4376                show_regs(regs);
4377        else
4378                dump_stack();
4379}
4380
4381/*
4382 * Various schedule()-time debugging checks and statistics:
4383 */
4384static inline void schedule_debug(struct task_struct *prev)
4385{
4386        /*
4387         * Test if we are atomic. Since do_exit() needs to call into
4388         * schedule() atomically, we ignore that path for now.
4389         * Otherwise, whine if we are scheduling when we should not be.
4390         */
4391        if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
4392                __schedule_bug(prev);
4393
4394        profile_hit(SCHED_PROFILING, __builtin_return_address(0));
4395
4396        schedstat_inc(this_rq(), sched_count);
4397#ifdef CONFIG_SCHEDSTATS
4398        if (unlikely(prev->lock_depth >= 0)) {
4399                schedstat_inc(this_rq(), bkl_count);
4400                schedstat_inc(prev, sched_info.bkl_count);
4401        }
4402#endif
4403}
4404
4405/*
4406 * Pick up the highest-prio task:
4407 */
4408static inline struct task_struct *
4409pick_next_task(struct rq *rq, struct task_struct *prev)
4410{
4411        const struct sched_class *class;
4412        struct task_struct *p;
4413
4414        /*
4415         * Optimization: we know that if all tasks are in
4416         * the fair class we can call that function directly:
4417         */
4418        if (likely(rq->nr_running == rq->cfs.nr_running)) {
4419                p = fair_sched_class.pick_next_task(rq);
4420                if (likely(p))
4421                        return p;
4422        }
4423
4424        class = sched_class_highest;
4425        for ( ; ; ) {
4426                p = class->pick_next_task(rq);
4427                if (p)
4428                        return p;
4429                /*
4430                 * Will never be NULL as the idle class always
4431                 * returns a non-NULL p:
4432                 */
4433                class = class->next;
4434        }
4435}
4436
4437/*
4438 * schedule() is the main scheduler function.
4439 */
4440asmlinkage void __sched schedule(void)
4441{
4442        struct task_struct *prev, *next;
4443        unsigned long *switch_count;
4444        struct rq *rq;
4445        int cpu;
4446
4447need_resched:
4448        preempt_disable();
4449        cpu = smp_processor_id();
4450        rq = cpu_rq(cpu);
4451        rcu_qsctr_inc(cpu);
4452        prev = rq->curr;
4453        switch_count = &prev->nivcsw;
4454
4455        release_kernel_lock(prev);
4456need_resched_nonpreemptible:
4457
4458        schedule_debug(prev);
4459
4460        if (sched_feat(HRTICK))
4461                hrtick_clear(rq);
4462
4463        spin_lock_irq(&rq->lock);
4464        update_rq_clock(rq);
4465        clear_tsk_need_resched(prev);
4466
4467        if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
4468                if (unlikely(signal_pending_state(prev->state, prev)))
4469                        prev->state = TASK_RUNNING;
4470                else
4471                        deactivate_task(rq, prev, 1);
4472                switch_count = &prev->nvcsw;
4473        }
4474
4475#ifdef CONFIG_SMP
4476        if (prev->sched_class->pre_schedule)
4477                prev->sched_class->pre_schedule(rq, prev);
4478#endif
4479
4480        if (unlikely(!rq->nr_running))
4481                idle_balance(cpu, rq);
4482
4483        prev->sched_class->put_prev_task(rq, prev);
4484        next = pick_next_task(rq, prev);
4485
4486        if (likely(prev != next)) {
4487                sched_info_switch(prev, next);
4488
4489                rq->nr_switches++;
4490                rq->curr = next;
4491                ++*switch_count;
4492
4493                context_switch(rq, prev, next); /* unlocks the rq */
4494                /*
4495                 * the context switch might have flipped the stack from under
4496                 * us, hence refresh the local variables.
4497                 */
4498                cpu = smp_processor_id();
4499                rq = cpu_rq(cpu);
4500        } else
4501                spin_unlock_irq(&rq->lock);
4502
4503        if (unlikely(reacquire_kernel_lock(current) < 0))
4504                goto need_resched_nonpreemptible;
4505
4506        preempt_enable_no_resched();
4507        if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
4508                goto need_resched;
4509}
4510EXPORT_SYMBOL(schedule);
4511
4512#ifdef CONFIG_PREEMPT
4513/*
4514 * this is the entry point to schedule() from in-kernel preemption
4515 * off of preempt_enable. Kernel preemptions off return from interrupt
4516 * occur there and call schedule directly.
4517 */
4518asmlinkage void __sched preempt_schedule(void)
4519{
4520        struct thread_info *ti = current_thread_info();
4521
4522        /*
4523         * If there is a non-zero preempt_count or interrupts are disabled,
4524         * we do not want to preempt the current task. Just return..
4525         */
4526        if (likely(ti->preempt_count || irqs_disabled()))
4527                return;
4528
4529        do {
4530                add_preempt_count(PREEMPT_ACTIVE);
4531                schedule();
4532                sub_preempt_count(PREEMPT_ACTIVE);
4533
4534                /*
4535                 * Check again in case we missed a preemption opportunity
4536                 * between schedule and now.
4537                 */
4538                barrier();
4539        } while (unlikely(test_thread_flag(TIF_NEED_RESCHED)));
4540}
4541EXPORT_SYMBOL(preempt_schedule);
4542
4543/*
4544 * this is the entry point to schedule() from kernel preemption
4545 * off of irq context.
4546 * Note, that this is called and return with irqs disabled. This will
4547 * protect us against recursive calling from irq.
4548 */
4549asmlinkage void __sched preempt_schedule_irq(void)
4550{
4551        struct thread_info *ti = current_thread_info();
4552
4553        /* Catch callers which need to be fixed */
4554        BUG_ON(ti->preempt_count || !irqs_disabled());
4555
4556        do {
4557                add_preempt_count(PREEMPT_ACTIVE);
4558                local_irq_enable();
4559                schedule();
4560                local_irq_disable();
4561                sub_preempt_count(PREEMPT_ACTIVE);
4562
4563                /*
4564                 * Check again in case we missed a preemption opportunity
4565                 * between schedule and now.
4566                 */
4567                barrier();
4568        } while (unlikely(test_thread_flag(TIF_NEED_RESCHED)));
4569}
4570
4571#endif /* CONFIG_PREEMPT */
4572
4573int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
4574                          void *key)
4575{
4576        return try_to_wake_up(curr->private, mode, sync);
4577}
4578EXPORT_SYMBOL(default_wake_function);
4579
4580/*
4581 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
4582 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
4583 * number) then we wake all the non-exclusive tasks and one exclusive task.
4584 *
4585 * There are circumstances in which we can try to wake a task which has already
4586 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
4587 * zero in this (rare) case, and we handle it by continuing to scan the queue.
4588 */
4589static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
4590                             int nr_exclusive, int sync, void *key)
4591{
4592        wait_queue_t *curr, *next;
4593
4594        list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
4595                unsigned flags = curr->flags;
4596
4597                if (curr->func(curr, mode, sync, key) &&
4598                                (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
4599                        break;
4600        }
4601}
4602
4603/**
4604 * __wake_up - wake up threads blocked on a waitqueue.
4605 * @q: the waitqueue
4606 * @mode: which threads
4607 * @nr_exclusive: how many wake-one or wake-many threads to wake up
4608 * @key: is directly passed to the wakeup function
4609 */
4610void __wake_up(wait_queue_head_t *q, unsigned int mode,
4611                        int nr_exclusive, void *key)
4612{
4613        unsigned long flags;
4614
4615        spin_lock_irqsave(&q->lock, flags);
4616        __wake_up_common(q, mode, nr_exclusive, 0, key);
4617        spin_unlock_irqrestore(&q->lock, flags);
4618}
4619EXPORT_SYMBOL(__wake_up);
4620
4621/*
4622 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
4623 */
4624void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
4625{
4626        __wake_up_common(q, mode, 1, 0, NULL);
4627}
4628
4629/**
4630 * __wake_up_sync - wake up threads blocked on a waitqueue.
4631 * @q: the waitqueue
4632 * @mode: which threads
4633 * @nr_exclusive: how many wake-one or wake-many threads to wake up
4634 *
4635 * The sync wakeup differs that the waker knows that it will schedule
4636 * away soon, so while the target thread will be woken up, it will not
4637 * be migrated to another CPU - ie. the two threads are 'synchronized'
4638 * with each other. This can prevent needless bouncing between CPUs.
4639 *
4640 * On UP it can prevent extra preemption.
4641 */
4642void
4643__wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
4644{
4645        unsigned long flags;
4646        int sync = 1;
4647
4648        if (unlikely(!q))
4649                return;
4650
4651        if (unlikely(!nr_exclusive))
4652                sync = 0;
4653
4654        spin_lock_irqsave(&q->lock, flags);
4655        __wake_up_common(q, mode, nr_exclusive, sync, NULL);
4656        spin_unlock_irqrestore(&q->lock, flags);
4657}
4658EXPORT_SYMBOL_GPL(__wake_up_sync);        /* For internal use only */
4659
4660/**
4661 * complete: - signals a single thread waiting on this completion
4662 * @x:  holds the state of this particular completion
4663 *
4664 * This will wake up a single thread waiting on this completion. Threads will be
4665 * awakened in the same order in which they were queued.
4666 *
4667 * See also complete_all(), wait_for_completion() and related routines.
4668 */
4669void complete(struct completion *x)
4670{
4671        unsigned long flags;
4672
4673        spin_lock_irqsave(&x->wait.lock, flags);
4674        x->done++;
4675        __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
4676        spin_unlock_irqrestore(&x->wait.lock, flags);
4677}
4678EXPORT_SYMBOL(complete);
4679
4680/**
4681 * complete_all: - signals all threads waiting on this completion
4682 * @x:  holds the state of this particular completion
4683 *
4684 * This will wake up all threads waiting on this particular completion event.
4685 */
4686void complete_all(struct completion *x)
4687{
4688        unsigned long flags;
4689
4690        spin_lock_irqsave(&x->wait.lock, flags);
4691        x->done += UINT_MAX/2;
4692        __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
4693        spin_unlock_irqrestore(&x->wait.lock, flags);
4694}
4695EXPORT_SYMBOL(complete_all);
4696
4697static inline long __sched
4698do_wait_for_common(struct completion *x, long timeout, int state)
4699{
4700        if (!x->done) {
4701                DECLARE_WAITQUEUE(wait, current);
4702
4703                wait.flags |= WQ_FLAG_EXCLUSIVE;
4704                __add_wait_queue_tail(&x->wait, &wait);
4705                do {
4706                        if (signal_pending_state(state, current)) {
4707                                timeout = -ERESTARTSYS;
4708                                break;
4709                        }
4710                        __set_current_state(state);
4711                        spin_unlock_irq(&x->wait.lock);
4712                        timeout = schedule_timeout(timeout);
4713                        spin_lock_irq(&x->wait.lock);
4714                } while (!x->done && timeout);
4715                __remove_wait_queue(&x->wait, &wait);
4716                if (!x->done)
4717                        return timeout;
4718        }
4719        x->done--;
4720        return timeout ?: 1;
4721}
4722
4723static long __sched
4724wait_for_common(struct completion *x, long timeout, int state)
4725{
4726        might_sleep();
4727
4728        spin_lock_irq(&x->wait.lock);
4729        timeout = do_wait_for_common(x, timeout, state);
4730        spin_unlock_irq(&x->wait.lock);
4731        return timeout;
4732}
4733
4734/**
4735 * wait_for_completion: - waits for completion of a task
4736 * @x:  holds the state of this particular completion
4737 *
4738 * This waits to be signaled for completion of a specific task. It is NOT
4739 * interruptible and there is no timeout.
4740 *
4741 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
4742 * and interrupt capability. Also see complete().
4743 */
4744void __sched wait_for_completion(struct completion *x)
4745{
4746        wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
4747}
4748EXPORT_SYMBOL(wait_for_completion);
4749
4750/**
4751 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
4752 * @x:  holds the state of this particular completion
4753 * @timeout:  timeout value in jiffies
4754 *
4755 * This waits for either a completion of a specific task to be signaled or for a
4756 * specified timeout to expire. The timeout is in jiffies. It is not
4757 * interruptible.
4758 */
4759unsigned long __sched
4760wait_for_completion_timeout(struct completion *x, unsigned long timeout)
4761{
4762        return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
4763}
4764EXPORT_SYMBOL(wait_for_completion_timeout);
4765
4766/**
4767 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
4768 * @x:  holds the state of this particular completion
4769 *
4770 * This waits for completion of a specific task to be signaled. It is
4771 * interruptible.
4772 */
4773int __sched wait_for_completion_interruptible(struct completion *x)
4774{
4775        long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
4776        if (t == -ERESTARTSYS)
4777                return t;
4778        return 0;
4779}
4780EXPORT_SYMBOL(wait_for_completion_interruptible);
4781
4782/**
4783 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
4784 * @x:  holds the state of this particular completion
4785 * @timeout:  timeout value in jiffies
4786 *
4787 * This waits for either a completion of a specific task to be signaled or for a
4788 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
4789 */
4790unsigned long __sched
4791wait_for_completion_interruptible_timeout(struct completion *x,
4792                                          unsigned long timeout)
4793{
4794        return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
4795}
4796EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
4797
4798/**
4799 * wait_for_completion_killable: - waits for completion of a task (killable)
4800 * @x:  holds the state of this particular completion
4801 *
4802 * This waits to be signaled for completion of a specific task. It can be
4803 * interrupted by a kill signal.
4804 */
4805int __sched wait_for_completion_killable(struct completion *x)
4806{
4807        long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
4808        if (t == -ERESTARTSYS)
4809                return t;
4810        return 0;
4811}
4812EXPORT_SYMBOL(wait_for_completion_killable);
4813
4814/**
4815 *        try_wait_for_completion - try to decrement a completion without blocking
4816 *        @x:        completion structure
4817 *
4818 *        Returns: 0 if a decrement cannot be done without blocking
4819 *                 1 if a decrement succeeded.
4820 *
4821 *        If a completion is being used as a counting completion,
4822 *        attempt to decrement the counter without blocking. This
4823 *        enables us to avoid waiting if the resource the completion
4824 *        is protecting is not available.
4825 */
4826bool try_wait_for_completion(struct completion *x)
4827{
4828        int ret = 1;
4829
4830        spin_lock_irq(&x->wait.lock);
4831        if (!x->done)
4832                ret = 0;
4833        else
4834                x->done--;
4835        spin_unlock_irq(&x->wait.lock);
4836        return ret;
4837}
4838EXPORT_SYMBOL(try_wait_for_completion);
4839
4840/**
4841 *        completion_done - Test to see if a completion has any waiters
4842 *        @x:        completion structure
4843 *
4844 *        Returns: 0 if there are waiters (wait_for_completion() in progress)
4845 *                 1 if there are no waiters.
4846 *
4847 */
4848bool completion_done(struct completion *x)
4849{
4850        int ret = 1;
4851
4852        spin_lock_irq(&x->wait.lock);
4853        if (!x->done)
4854                ret = 0;
4855        spin_unlock_irq(&x->wait.lock);
4856        return ret;
4857}
4858EXPORT_SYMBOL(completion_done);
4859
4860static long __sched
4861sleep_on_common(wait_queue_head_t *q, int state, long timeout)
4862{
4863        unsigned long flags;
4864        wait_queue_t wait;
4865
4866        init_waitqueue_entry(&wait, current);
4867
4868        __set_current_state(state);
4869
4870        spin_lock_irqsave(&q->lock, flags);
4871        __add_wait_queue(q, &wait);
4872        spin_unlock(&q->lock);
4873        timeout = schedule_timeout(timeout);
4874        spin_lock_irq(&q->lock);
4875        __remove_wait_queue(q, &wait);
4876        spin_unlock_irqrestore(&q->lock, flags);
4877
4878        return timeout;
4879}
4880
4881void __sched interruptible_sleep_on(wait_queue_head_t *q)
4882{
4883        sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
4884}
4885EXPORT_SYMBOL(interruptible_sleep_on);
4886
4887long __sched
4888interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
4889{
4890        return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
4891}
4892EXPORT_SYMBOL(interruptible_sleep_on_timeout);
4893
4894void __sched sleep_on(wait_queue_head_t *q)
4895{
4896        sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
4897}
4898EXPORT_SYMBOL(sleep_on);
4899
4900long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
4901{
4902        return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
4903}
4904EXPORT_SYMBOL(sleep_on_timeout);
4905
4906#ifdef CONFIG_RT_MUTEXES
4907
4908/*
4909 * rt_mutex_setprio - set the current priority of a task
4910 * @p: task
4911 * @prio: prio value (kernel-internal form)
4912 *
4913 * This function changes the 'effective' priority of a task. It does
4914 * not touch ->normal_prio like __setscheduler().
4915 *
4916 * Used by the rt_mutex code to implement priority inheritance logic.
4917 */
4918void rt_mutex_setprio(struct task_struct *p, int prio)
4919{
4920        unsigned long flags;
4921        int oldprio, on_rq, running;
4922        struct rq *rq;
4923        const struct sched_class *prev_class = p->sched_class;
4924
4925        BUG_ON(prio < 0 || prio > MAX_PRIO);
4926
4927        rq = task_rq_lock(p, &flags);
4928        update_rq_clock(rq);
4929
4930        oldprio = p->prio;
4931        on_rq = p->se.on_rq;
4932        running = task_current(rq, p);
4933        if (on_rq)
4934                dequeue_task(rq, p, 0);
4935        if (running)
4936                p->sched_class->put_prev_task(rq, p);
4937
4938        if (rt_prio(prio))
4939                p->sched_class = &rt_sched_class;
4940        else
4941                p->sched_class = &fair_sched_class;
4942
4943        p->prio = prio;
4944
4945        if (running)
4946                p->sched_class->set_curr_task(rq);
4947        if (on_rq) {
4948                enqueue_task(rq, p, 0);
4949
4950                check_class_changed(rq, p, prev_class, oldprio, running);
4951        }
4952        task_rq_unlock(rq, &flags);
4953}
4954
4955#endif
4956
4957void set_user_nice(struct task_struct *p, long nice)
4958{
4959        int old_prio, delta, on_rq;
4960        unsigned long flags;
4961        struct rq *rq;
4962
4963        if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
4964                return;
4965        /*
4966         * We have to be careful, if called from sys_setpriority(),
4967         * the task might be in the middle of scheduling on another CPU.
4968         */
4969        rq = task_rq_lock(p, &flags);
4970        update_rq_clock(rq);
4971        /*
4972         * The RT priorities are set via sched_setscheduler(), but we still
4973         * allow the 'normal' nice value to be set - but as expected
4974         * it wont have any effect on scheduling until the task is
4975         * SCHED_FIFO/SCHED_RR:
4976         */
4977        if (task_has_rt_policy(p)) {
4978                p->static_prio = NICE_TO_PRIO(nice);
4979                goto out_unlock;
4980        }
4981        on_rq = p->se.on_rq;
4982        if (on_rq)
4983                dequeue_task(rq, p, 0);
4984
4985        p->static_prio = NICE_TO_PRIO(nice);
4986        set_load_weight(p);
4987        old_prio = p->prio;
4988        p->prio = effective_prio(p);
4989        delta = p->prio - old_prio;
4990
4991        if (on_rq) {
4992                enqueue_task(rq, p, 0);
4993                /*
4994                 * If the task increased its priority or is running and
4995                 * lowered its priority, then reschedule its CPU:
4996                 */
4997                if (delta < 0 || (delta > 0 && task_running(rq, p)))
4998                        resched_task(rq->curr);
4999        }
5000out_unlock:
5001        task_rq_unlock(rq, &flags);
5002}
5003EXPORT_SYMBOL(set_user_nice);
5004
5005/*
5006 * can_nice - check if a task can reduce its nice value
5007 * @p: task
5008 * @nice: nice value
5009 */
5010int can_nice(const struct task_struct *p, const int nice)
5011{
5012        /* convert nice value [19,-20] to rlimit style value [1,40] */
5013        int nice_rlim = 20 - nice;
5014
5015        return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
5016                capable(CAP_SYS_NICE));
5017}
5018
5019#ifdef __ARCH_WANT_SYS_NICE
5020
5021/*
5022 * sys_nice - change the priority of the current process.
5023 * @increment: priority increment
5024 *
5025 * sys_setpriority is a more generic, but much slower function that
5026 * does similar things.
5027 */
5028asmlinkage long sys_nice(int increment)
5029{
5030        long nice, retval;
5031
5032        /*
5033         * Setpriority might change our priority at the same moment.
5034         * We don't have to worry. Conceptually one call occurs first
5035         * and we have a single winner.
5036         */
5037        if (increment < -40)
5038                increment = -40;
5039        if (increment > 40)
5040                increment = 40;
5041
5042        nice = PRIO_TO_NICE(current->static_prio) + increment;
5043        if (nice < -20)
5044                nice = -20;
5045        if (nice > 19)
5046                nice = 19;
5047
5048        if (increment < 0 && !can_nice(current, nice))
5049                return -EPERM;
5050
5051        retval = security_task_setnice(current, nice);
5052        if (retval)
5053                return retval;
5054
5055        set_user_nice(current, nice);
5056        return 0;
5057}
5058
5059#endif
5060
5061/**
5062 * task_prio - return the priority value of a given task.
5063 * @p: the task in question.
5064 *
5065 * This is the priority value as seen by users in /proc.
5066 * RT tasks are offset by -200. Normal tasks are centered
5067 * around 0, value goes from -16 to +15.
5068 */
5069int task_prio(const struct task_struct *p)
5070{
5071        return p->prio - MAX_RT_PRIO;
5072}
5073
5074/**
5075 * task_nice - return the nice value of a given task.
5076 * @p: the task in question.
5077 */
5078int task_nice(const struct task_struct *p)
5079{
5080        return TASK_NICE(p);
5081}
5082EXPORT_SYMBOL(task_nice);
5083
5084/**
5085 * idle_cpu - is a given cpu idle currently?
5086 * @cpu: the processor in question.
5087 */
5088int idle_cpu(int cpu)
5089{
5090        return cpu_curr(cpu) == cpu_rq(cpu)->idle;
5091}
5092
5093/**
5094 * idle_task - return the idle task for a given cpu.
5095 * @cpu: the processor in question.
5096 */
5097struct task_struct *idle_task(int cpu)
5098{
5099        return cpu_rq(cpu)->idle;
5100}
5101
5102/**
5103 * find_process_by_pid - find a process with a matching PID value.
5104 * @pid: the pid in question.
5105 */
5106static struct task_struct *find_process_by_pid(pid_t pid)
5107{
5108        return pid ? find_task_by_vpid(pid) : current;
5109}
5110
5111/* Actually do priority change: must hold rq lock. */
5112static void
5113__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
5114{
5115        BUG_ON(p->se.on_rq);
5116
5117        p->policy = policy;
5118        switch (p->policy) {
5119        case SCHED_NORMAL:
5120        case SCHED_BATCH:
5121        case SCHED_IDLE:
5122                p->sched_class = &fair_sched_class;
5123                break;
5124        case SCHED_FIFO:
5125        case SCHED_RR:
5126                p->sched_class = &rt_sched_class;
5127                break;
5128        }
5129
5130        p->rt_priority = prio;
5131        p->normal_prio = normal_prio(p);
5132        /* we are holding p->pi_lock already */
5133        p->prio = rt_mutex_getprio(p);
5134        set_load_weight(p);
5135}
5136
5137static int __sched_setscheduler(struct task_struct *p, int policy,
5138                                struct sched_param *param, bool user)
5139{
5140        int retval, oldprio, oldpolicy = -1, on_rq, running;
5141        unsigned long flags;
5142        const struct sched_class *prev_class = p->sched_class;
5143        struct rq *rq;
5144
5145        /* may grab non-irq protected spin_locks */
5146        BUG_ON(in_interrupt());
5147recheck:
5148        /* double check policy once rq lock held */
5149        if (policy < 0)
5150                policy = oldpolicy = p->policy;
5151        else if (policy != SCHED_FIFO && policy != SCHED_RR &&
5152                        policy != SCHED_NORMAL && policy != SCHED_BATCH &&
5153                        policy != SCHED_IDLE)
5154                return -EINVAL;
5155        /*
5156         * Valid priorities for SCHED_FIFO and SCHED_RR are
5157         * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
5158         * SCHED_BATCH and SCHED_IDLE is 0.
5159         */
5160        if (param->sched_priority < 0 ||
5161            (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
5162            (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
5163                return -EINVAL;
5164        if (rt_policy(policy) != (param->sched_priority != 0))
5165                return -EINVAL;
5166
5167        /*
5168         * Allow unprivileged RT tasks to decrease priority:
5169         */
5170        if (user && !capable(CAP_SYS_NICE)) {
5171                if (rt_policy(policy)) {
5172                        unsigned long rlim_rtprio;
5173
5174                        if (!lock_task_sighand(p, &flags))
5175                                return -ESRCH;
5176                        rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
5177                        unlock_task_sighand(p, &flags);
5178
5179                        /* can't set/change the rt policy */
5180                        if (policy != p->policy && !rlim_rtprio)
5181                                return -EPERM;
5182
5183                        /* can't increase priority */
5184                        if (param->sched_priority > p->rt_priority &&
5185                            param->sched_priority > rlim_rtprio)
5186                                return -EPERM;
5187                }
5188                /*
5189                 * Like positive nice levels, dont allow tasks to
5190                 * move out of SCHED_IDLE either:
5191                 */
5192                if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
5193                        return -EPERM;
5194
5195                /* can't change other user's priorities */
5196                if ((current->euid != p->euid) &&
5197                    (current->euid != p->uid))
5198                        return -EPERM;
5199        }
5200
5201        if (user) {
5202#ifdef CONFIG_RT_GROUP_SCHED
5203                /*
5204                 * Do not allow realtime tasks into groups that have no runtime
5205                 * assigned.
5206                 */
5207                if (rt_bandwidth_enabled() && rt_policy(policy) &&
5208                                task_group(p)->rt_bandwidth.rt_runtime == 0)
5209                        return -EPERM;
5210#endif
5211
5212                retval = security_task_setscheduler(p, policy, param);
5213                if (retval)
5214                        return retval;
5215        }
5216
5217        /*
5218         * make sure no PI-waiters arrive (or leave) while we are
5219         * changing the priority of the task:
5220         */
5221        spin_lock_irqsave(&p->pi_lock, flags);
5222        /*
5223         * To be able to change p->policy safely, the apropriate
5224         * runqueue lock must be held.
5225         */
5226        rq = __task_rq_lock(p);
5227        /* recheck policy now with rq lock held */
5228        if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
5229                policy = oldpolicy = -1;
5230                __task_rq_unlock(rq);
5231                spin_unlock_irqrestore(&p->pi_lock, flags);
5232                goto recheck;
5233        }
5234        update_rq_clock(rq);
5235        on_rq = p->se.on_rq;
5236        running = task_current(rq, p);
5237        if (on_rq)
5238                deactivate_task(rq, p, 0);
5239        if (running)
5240                p->sched_class->put_prev_task(rq, p);
5241
5242        oldprio = p->prio;
5243        __setscheduler(rq, p, policy, param->sched_priority);
5244
5245        if (running)
5246                p->sched_class->set_curr_task(rq);
5247        if (on_rq) {
5248                activate_task(rq, p, 0);
5249
5250                check_class_changed(rq, p, prev_class, oldprio, running);
5251        }
5252        __task_rq_unlock(rq);
5253        spin_unlock_irqrestore(&p->pi_lock, flags);
5254
5255        rt_mutex_adjust_pi(p);
5256
5257        return 0;
5258}
5259
5260/**
5261 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
5262 * @p: the task in question.
5263 * @policy: new policy.
5264 * @param: structure containing the new RT priority.
5265 *
5266 * NOTE that the task may be already dead.
5267 */
5268int sched_setscheduler(struct task_struct *p, int policy,
5269                       struct sched_param *param)
5270{
5271        return __sched_setscheduler(p, policy, param, true);
5272}
5273EXPORT_SYMBOL_GPL(sched_setscheduler);
5274
5275/**
5276 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
5277 * @p: the task in question.
5278 * @policy: new policy.
5279 * @param: structure containing the new RT priority.
5280 *
5281 * Just like sched_setscheduler, only don't bother checking if the
5282 * current context has permission.  For example, this is needed in
5283 * stop_machine(): we create temporary high priority worker threads,
5284 * but our caller might not have that capability.
5285 */
5286int sched_setscheduler_nocheck(struct task_struct *p, int policy,
5287                               struct sched_param *param)
5288{
5289        return __sched_setscheduler(p, policy, param, false);
5290}
5291
5292static int
5293do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
5294{
5295        struct sched_param lparam;
5296        struct task_struct *p;
5297        int retval;
5298
5299        if (!param || pid < 0)
5300                return -EINVAL;
5301        if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
5302                return -EFAULT;
5303
5304        rcu_read_lock();
5305        retval = -ESRCH;
5306        p = find_process_by_pid(pid);
5307        if (p != NULL)
5308                retval = sched_setscheduler(p, policy, &lparam);
5309        rcu_read_unlock();
5310
5311        return retval;
5312}
5313
5314/**
5315 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
5316 * @pid: the pid in question.
5317 * @policy: new policy.
5318 * @param: structure containing the new RT priority.
5319 */
5320asmlinkage long
5321sys_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
5322{
5323        /* negative values for policy are not valid */
5324        if (policy < 0)
5325                return -EINVAL;
5326
5327        return do_sched_setscheduler(pid, policy, param);
5328}
5329
5330/**
5331 * sys_sched_setparam - set/change the RT priority of a thread
5332 * @pid: the pid in question.
5333 * @param: structure containing the new RT priority.
5334 */
5335asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
5336{
5337        return do_sched_setscheduler(pid, -1, param);
5338}
5339
5340/**
5341 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
5342 * @pid: the pid in question.
5343 */
5344asmlinkage long sys_sched_getscheduler(pid_t pid)
5345{
5346        struct task_struct *p;
5347        int retval;
5348
5349        if (pid < 0)
5350                return -EINVAL;
5351
5352        retval = -ESRCH;
5353        read_lock(&tasklist_lock);
5354        p = find_process_by_pid(pid);
5355        if (p) {
5356                retval = security_task_getscheduler(p);
5357                if (!retval)
5358                        retval = p->policy;
5359        }
5360        read_unlock(&tasklist_lock);
5361        return retval;
5362}
5363
5364/**
5365 * sys_sched_getscheduler - get the RT priority of a thread
5366 * @pid: the pid in question.
5367 * @param: structure containing the RT priority.
5368 */
5369asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param)
5370{
5371        struct sched_param lp;
5372        struct task_struct *p;
5373        int retval;
5374
5375        if (!param || pid < 0)
5376                return -EINVAL;
5377
5378        read_lock(&tasklist_lock);
5379        p = find_process_by_pid(pid);
5380        retval = -ESRCH;
5381        if (!p)
5382                goto out_unlock;
5383
5384        retval = security_task_getscheduler(p);
5385        if (retval)
5386                goto out_unlock;
5387
5388        lp.sched_priority = p->rt_priority;
5389        read_unlock(&tasklist_lock);
5390
5391        /*
5392         * This one might sleep, we cannot do it with a spinlock held ...
5393         */
5394        retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
5395
5396        return retval;
5397
5398out_unlock:
5399        read_unlock(&tasklist_lock);
5400        return retval;
5401}
5402
5403long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5404{
5405        cpumask_t cpus_allowed;
5406        cpumask_t new_mask = *in_mask;
5407        struct task_struct *p;
5408        int retval;
5409
5410        get_online_cpus();
5411        read_lock(&tasklist_lock);
5412
5413        p = find_process_by_pid(pid);
5414        if (!p) {
5415                read_unlock(&tasklist_lock);
5416                put_online_cpus();
5417                return -ESRCH;
5418        }
5419
5420        /*
5421         * It is not safe to call set_cpus_allowed with the
5422         * tasklist_lock held. We will bump the task_struct's
5423         * usage count and then drop tasklist_lock.
5424         */
5425        get_task_struct(p);
5426        read_unlock(&tasklist_lock);
5427
5428        retval = -EPERM;
5429        if ((current->euid != p->euid) && (current->euid != p->uid) &&
5430                        !capable(CAP_SYS_NICE))
5431                goto out_unlock;
5432
5433        retval = security_task_setscheduler(p, 0, NULL);
5434        if (retval)
5435                goto out_unlock;
5436
5437        cpuset_cpus_allowed(p, &cpus_allowed);
5438        cpus_and(new_mask, new_mask, cpus_allowed);
5439 again:
5440        retval = set_cpus_allowed_ptr(p, &new_mask);
5441
5442        if (!retval) {
5443                cpuset_cpus_allowed(p, &cpus_allowed);
5444                if (!cpus_subset(new_mask, cpus_allowed)) {
5445                        /*
5446                         * We must have raced with a concurrent cpuset
5447                         * update. Just reset the cpus_allowed to the
5448                         * cpuset's cpus_allowed
5449                         */
5450                        new_mask = cpus_allowed;
5451                        goto again;
5452                }
5453        }
5454out_unlock:
5455        put_task_struct(p);
5456        put_online_cpus();
5457        return retval;
5458}
5459
5460static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5461                             cpumask_t *new_mask)
5462{
5463        if (len < sizeof(cpumask_t)) {
5464                memset(new_mask, 0, sizeof(cpumask_t));
5465        } else if (len > sizeof(cpumask_t)) {
5466                len = sizeof(cpumask_t);
5467        }
5468        return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5469}
5470
5471/**
5472 * sys_sched_setaffinity - set the cpu affinity of a process
5473 * @pid: pid of the process
5474 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
5475 * @user_mask_ptr: user-space pointer to the new cpu mask
5476 */
5477asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5478                                      unsigned long __user *user_mask_ptr)
5479{
5480        cpumask_t new_mask;
5481        int retval;
5482
5483        retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask);
5484        if (retval)
5485                return retval;
5486
5487        return sched_setaffinity(pid, &new_mask);
5488}
5489
5490long sched_getaffinity(pid_t pid, cpumask_t *mask)
5491{
5492        struct task_struct *p;
5493        int retval;
5494
5495        get_online_cpus();
5496        read_lock(&tasklist_lock);
5497
5498        retval = -ESRCH;
5499        p = find_process_by_pid(pid);
5500        if (!p)
5501                goto out_unlock;
5502
5503        retval = security_task_getscheduler(p);
5504        if (retval)
5505                goto out_unlock;
5506
5507        cpus_and(*mask, p->cpus_allowed, cpu_online_map);
5508
5509out_unlock:
5510        read_unlock(&tasklist_lock);
5511        put_online_cpus();
5512
5513        return retval;
5514}
5515
5516/**
5517 * sys_sched_getaffinity - get the cpu affinity of a process
5518 * @pid: pid of the process
5519 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
5520 * @user_mask_ptr: user-space pointer to hold the current cpu mask
5521 */
5522asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
5523                                      unsigned long __user *user_mask_ptr)
5524{
5525        int ret;
5526        cpumask_t mask;
5527
5528        if (len < sizeof(cpumask_t))
5529                return -EINVAL;
5530
5531        ret = sched_getaffinity(pid, &mask);
5532        if (ret < 0)
5533                return ret;
5534
5535        if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t)))
5536                return -EFAULT;
5537
5538        return sizeof(cpumask_t);
5539}
5540
5541/**
5542 * sys_sched_yield - yield the current processor to other threads.
5543 *
5544 * This function yields the current CPU to other tasks. If there are no
5545 * other threads running on this CPU then this function will return.
5546 */
5547asmlinkage long sys_sched_yield(void)
5548{
5549        struct rq *rq = this_rq_lock();
5550
5551        schedstat_inc(rq, yld_count);
5552        current->sched_class->yield_task(rq);
5553
5554        /*
5555         * Since we are going to call schedule() anyway, there's
5556         * no need to preempt or enable interrupts:
5557         */
5558        __release(rq->lock);
5559        spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
5560        _raw_spin_unlock(&rq->lock);
5561        preempt_enable_no_resched();
5562
5563        schedule();
5564
5565        return 0;
5566}
5567
5568static void __cond_resched(void)
5569{
5570#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
5571        __might_sleep(__FILE__, __LINE__);
5572#endif
5573        /*
5574         * The BKS might be reacquired before we have dropped
5575         * PREEMPT_ACTIVE, which could trigger a second
5576         * cond_resched() call.
5577         */
5578        do {
5579                add_preempt_count(PREEMPT_ACTIVE);
5580                schedule();
5581                sub_preempt_count(PREEMPT_ACTIVE);
5582        } while (need_resched());
5583}
5584
5585int __sched _cond_resched(void)
5586{
5587        if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
5588                                        system_state == SYSTEM_RUNNING) {
5589                __cond_resched();
5590                return 1;
5591        }
5592        return 0;
5593}
5594EXPORT_SYMBOL(_cond_resched);
5595
5596/*
5597 * cond_resched_lock() - if a reschedule is pending, drop the given lock,
5598 * call schedule, and on return reacquire the lock.
5599 *
5600 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
5601 * operations here to prevent schedule() from being called twice (once via
5602 * spin_unlock(), once by hand).
5603 */
5604int cond_resched_lock(spinlock_t *lock)
5605{
5606        int resched = need_resched() && system_state == SYSTEM_RUNNING;
5607        int ret = 0;
5608
5609        if (spin_needbreak(lock) || resched) {
5610                spin_unlock(lock);
5611                if (resched && need_resched())
5612                        __cond_resched();
5613                else
5614                        cpu_relax();
5615                ret = 1;
5616                spin_lock(lock);
5617        }
5618        return ret;
5619}
5620EXPORT_SYMBOL(cond_resched_lock);
5621
5622int __sched cond_resched_softirq(void)
5623{
5624        BUG_ON(!in_softirq());
5625
5626        if (need_resched() && system_state == SYSTEM_RUNNING) {
5627                local_bh_enable();
5628                __cond_resched();
5629                local_bh_disable();
5630                return 1;
5631        }
5632        return 0;
5633}
5634EXPORT_SYMBOL(cond_resched_softirq);
5635
5636/**
5637 * yield - yield the current processor to other threads.
5638 *
5639 * This is a shortcut for kernel-space yielding - it marks the
5640 * thread runnable and calls sys_sched_yield().
5641 */
5642void __sched yield(void)
5643{
5644        set_current_state(TASK_RUNNING);
5645        sys_sched_yield();
5646}
5647EXPORT_SYMBOL(yield);
5648
5649/*
5650 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
5651 * that process accounting knows that this is a task in IO wait state.
5652 *
5653 * But don't do that if it is a deliberate, throttling IO wait (this task
5654 * has set its backing_dev_info: the queue against which it should throttle)
5655 */
5656void __sched io_schedule(void)
5657{
5658        struct rq *rq = &__raw_get_cpu_var(runqueues);
5659
5660        delayacct_blkio_start();
5661        atomic_inc(&rq->nr_iowait);
5662        schedule();
5663        atomic_dec(&rq->nr_iowait);
5664        delayacct_blkio_end();
5665}
5666EXPORT_SYMBOL(io_schedule);
5667
5668long __sched io_schedule_timeout(long timeout)
5669{
5670        struct rq *rq = &__raw_get_cpu_var(runqueues);
5671        long ret;
5672
5673        delayacct_blkio_start();
5674        atomic_inc(&rq->nr_iowait);
5675        ret = schedule_timeout(timeout);
5676        atomic_dec(&rq->nr_iowait);
5677        delayacct_blkio_end();
5678        return ret;
5679}
5680
5681/**
5682 * sys_sched_get_priority_max - return maximum RT priority.
5683 * @policy: scheduling class.
5684 *
5685 * this syscall returns the maximum rt_priority that can be used
5686 * by a given scheduling class.
5687 */
5688asmlinkage long sys_sched_get_priority_max(int policy)
5689{
5690        int ret = -EINVAL;
5691
5692        switch (policy) {
5693        case SCHED_FIFO:
5694        case SCHED_RR:
5695                ret = MAX_USER_RT_PRIO-1;
5696                break;
5697        case SCHED_NORMAL:
5698        case SCHED_BATCH:
5699        case SCHED_IDLE:
5700                ret = 0;
5701                break;
5702        }
5703        return ret;
5704}
5705
5706/**
5707 * sys_sched_get_priority_min - return minimum RT priority.
5708 * @policy: scheduling class.
5709 *
5710 * this syscall returns the minimum rt_priority that can be used
5711 * by a given scheduling class.
5712 */
5713asmlinkage long sys_sched_get_priority_min(int policy)
5714{
5715        int ret = -EINVAL;
5716
5717        switch (policy) {
5718        case SCHED_FIFO:
5719        case SCHED_RR:
5720                ret = 1;
5721                break;
5722        case SCHED_NORMAL:
5723        case SCHED_BATCH:
5724        case SCHED_IDLE:
5725                ret = 0;
5726        }
5727        return ret;
5728}
5729
5730/**
5731 * sys_sched_rr_get_interval - return the default timeslice of a process.
5732 * @pid: pid of the process.
5733 * @interval: userspace pointer to the timeslice value.
5734 *
5735 * this syscall writes the default timeslice value of a given process
5736 * into the user-space timespec buffer. A value of '0' means infinity.
5737 */
5738asmlinkage
5739long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval)
5740{
5741        struct task_struct *p;
5742        unsigned int time_slice;
5743        int retval;
5744        struct timespec t;
5745
5746        if (pid < 0)
5747                return -EINVAL;
5748
5749        retval = -ESRCH;
5750        read_lock(&tasklist_lock);
5751        p = find_process_by_pid(pid);
5752        if (!p)
5753                goto out_unlock;
5754
5755        retval = security_task_getscheduler(p);
5756        if (retval)
5757                goto out_unlock;
5758
5759        /*
5760         * Time slice is 0 for SCHED_FIFO tasks and for SCHED_OTHER
5761         * tasks that are on an otherwise idle runqueue:
5762         */
5763        time_slice = 0;
5764        if (p->policy == SCHED_RR) {
5765                time_slice = DEF_TIMESLICE;
5766        } else if (p->policy != SCHED_FIFO) {
5767                struct sched_entity *se = &p->se;
5768                unsigned long flags;
5769                struct rq *rq;
5770
5771                rq = task_rq_lock(p, &flags);
5772                if (rq->cfs.load.weight)
5773                        time_slice = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
5774                task_rq_unlock(rq, &flags);
5775        }
5776        read_unlock(&tasklist_lock);
5777        jiffies_to_timespec(time_slice, &t);
5778        retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
5779        return retval;
5780
5781out_unlock:
5782        read_unlock(&tasklist_lock);
5783        return retval;
5784}
5785
5786static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5787
5788void sched_show_task(struct task_struct *p)
5789{
5790        unsigned long free = 0;
5791        unsigned state;
5792
5793        state = p->state ? __ffs(p->state) + 1 : 0;
5794        printk(KERN_INFO "%-13.13s %c", p->comm,
5795                state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
5796#if BITS_PER_LONG == 32
5797        if (state == TASK_RUNNING)
5798                printk(KERN_CONT " running  ");
5799        else
5800                printk(KERN_CONT " %08lx ", thread_saved_pc(p));
5801#else
5802        if (state == TASK_RUNNING)
5803                printk(KERN_CONT "  running task    ");
5804        else
5805                printk(KERN_CONT " %016lx ", thread_saved_pc(p));
5806#endif
5807#ifdef CONFIG_DEBUG_STACK_USAGE
5808        {
5809                unsigned long *n = end_of_stack(p);
5810                while (!*n)
5811                        n++;
5812                free = (unsigned long)n - (unsigned long)end_of_stack(p);
5813        }
5814#endif
5815        printk(KERN_CONT "%5lu %5d %6d\n", free,
5816                task_pid_nr(p), task_pid_nr(p->real_parent));
5817
5818        show_stack(p, NULL);
5819}
5820
5821void show_state_filter(unsigned long state_filter)
5822{
5823        struct task_struct *g, *p;
5824
5825#if BITS_PER_LONG == 32
5826        printk(KERN_INFO
5827                "  task                PC stack   pid father\n");
5828#else
5829        printk(KERN_INFO
5830                "  task                        PC stack   pid father\n");
5831#endif
5832        read_lock(&tasklist_lock);
5833        do_each_thread(g, p) {
5834                /*
5835                 * reset the NMI-timeout, listing all files on a slow
5836                 * console might take alot of time:
5837                 */
5838                touch_nmi_watchdog();
5839                if (!state_filter || (p->state & state_filter))
5840                        sched_show_task(p);
5841        } while_each_thread(g, p);
5842
5843        touch_all_softlockup_watchdogs();
5844
5845#ifdef CONFIG_SCHED_DEBUG
5846        sysrq_sched_debug_show();
5847#endif
5848        read_unlock(&tasklist_lock);
5849        /*
5850         * Only show locks if all tasks are dumped:
5851         */
5852        if (state_filter == -1)
5853                debug_show_all_locks();
5854}
5855
5856void __cpuinit init_idle_bootup_task(struct task_struct *idle)
5857{
5858        idle->sched_class = &idle_sched_class;
5859}
5860
5861/**
5862 * init_idle - set up an idle thread for a given CPU
5863 * @idle: task in question
5864 * @cpu: cpu the idle task belongs to
5865 *
5866 * NOTE: this function does not set the idle thread's NEED_RESCHED
5867 * flag, to make booting more robust.
5868 */
5869void __cpuinit init_idle(struct task_struct *idle, int cpu)
5870{
5871        struct rq *rq = cpu_rq(cpu);
5872        unsigned long flags;
5873
5874        spin_lock_irqsave(&rq->lock, flags);
5875
5876        __sched_fork(idle);
5877        idle->se.exec_start = sched_clock();
5878
5879        idle->prio = idle->normal_prio = MAX_PRIO;
5880        idle->cpus_allowed = cpumask_of_cpu(cpu);
5881        __set_task_cpu(idle, cpu);
5882
5883        rq->curr = rq->idle = idle;
5884#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
5885        idle->oncpu = 1;
5886#endif
5887        spin_unlock_irqrestore(&rq->lock, flags);
5888
5889        /* Set the preempt count _outside_ the spinlocks! */
5890#if defined(CONFIG_PREEMPT)
5891        task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
5892#else
5893        task_thread_info(idle)->preempt_count = 0;
5894#endif
5895        /*
5896         * The idle tasks have their own, simple scheduling class:
5897         */
5898        idle->sched_class = &idle_sched_class;
5899}
5900
5901/*
5902 * In a system that switches off the HZ timer nohz_cpu_mask
5903 * indicates which cpus entered this state. This is used
5904 * in the rcu update to wait only for active cpus. For system
5905 * which do not switch off the HZ timer nohz_cpu_mask should
5906 * always be CPU_MASK_NONE.
5907 */
5908cpumask_t nohz_cpu_mask = CPU_MASK_NONE;
5909
5910/*
5911 * Increase the granularity value when there are more CPUs,
5912 * because with more CPUs the 'effective latency' as visible
5913 * to users decreases. But the relationship is not linear,
5914 * so pick a second-best guess by going with the log2 of the
5915 * number of CPUs.
5916 *
5917 * This idea comes from the SD scheduler of Con Kolivas:
5918 */
5919static inline void sched_init_granularity(void)
5920{
5921        unsigned int factor = 1 + ilog2(num_online_cpus());
5922        const unsigned long limit = 200000000;
5923
5924        sysctl_sched_min_granularity *= factor;
5925        if (sysctl_sched_min_granularity > limit)
5926                sysctl_sched_min_granularity = limit;
5927
5928        sysctl_sched_latency *= factor;
5929        if (sysctl_sched_latency > limit)
5930                sysctl_sched_latency = limit;
5931
5932        sysctl_sched_wakeup_granularity *= factor;
5933
5934        sysctl_sched_shares_ratelimit *= factor;
5935}
5936
5937#ifdef CONFIG_SMP
5938/*
5939 * This is how migration works:
5940 *
5941 * 1) we queue a struct migration_req structure in the source CPU's
5942 *    runqueue and wake up that CPU's migration thread.
5943 * 2) we down() the locked semaphore => thread blocks.
5944 * 3) migration thread wakes up (implicitly it forces the migrated
5945 *    thread off the CPU)
5946 * 4) it gets the migration request and checks whether the migrated
5947 *    task is still in the wrong runqueue.
5948 * 5) if it's in the wrong runqueue then the migration thread removes
5949 *    it and puts it into the right queue.
5950 * 6) migration thread up()s the semaphore.
5951 * 7) we wake up and the migration is done.
5952 */
5953
5954/*
5955 * Change a given task's CPU affinity. Migrate the thread to a
5956 * proper CPU and schedule it away if the CPU it's executing on
5957 * is removed from the allowed bitmask.
5958 *
5959 * NOTE: the caller must have a valid reference to the task, the
5960 * task must not exit() & deallocate itself prematurely. The
5961 * call is not atomic; no spinlocks may be held.
5962 */
5963int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5964{
5965        struct migration_req req;
5966        unsigned long flags;
5967        struct rq *rq;
5968        int ret = 0;
5969
5970        rq = task_rq_lock(p, &flags);
5971        if (!cpus_intersects(*new_mask, cpu_online_map)) {
5972                ret = -EINVAL;
5973                goto out;
5974        }
5975
5976        if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5977                     !cpus_equal(p->cpus_allowed, *new_mask))) {
5978                ret = -EINVAL;
5979                goto out;
5980        }
5981
5982        if (p->sched_class->set_cpus_allowed)
5983                p->sched_class->set_cpus_allowed(p, new_mask);
5984        else {
5985                p->cpus_allowed = *new_mask;
5986                p->rt.nr_cpus_allowed = cpus_weight(*new_mask);
5987        }
5988
5989        /* Can the task run on the task's current CPU? If so, we're done */
5990        if (cpu_isset(task_cpu(p), *new_mask))
5991                goto out;
5992
5993        if (migrate_task(p, any_online_cpu(*new_mask), &req)) {
5994                /* Need help from migration thread: drop lock and wait. */
5995                task_rq_unlock(rq, &flags);
5996                wake_up_process(rq->migration_thread);
5997                wait_for_completion(&req.done);
5998                tlb_migrate_finish(p->mm);
5999                return 0;
6000        }
6001out:
6002        task_rq_unlock(rq, &flags);
6003
6004        return ret;
6005}
6006EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
6007
6008/*
6009 * Move (not current) task off this cpu, onto dest cpu. We're doing
6010 * this because either it can't run here any more (set_cpus_allowed()
6011 * away from this CPU, or CPU going down), or because we're
6012 * attempting to rebalance this task on exec (sched_exec).
6013 *
6014 * So we race with normal scheduler movements, but that's OK, as long
6015 * as the task is no longer on this CPU.
6016 *
6017 * Returns non-zero if task was successfully migrated.
6018 */
6019static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
6020{
6021        struct rq *rq_dest, *rq_src;
6022        int ret = 0, on_rq;
6023
6024        if (unlikely(!cpu_active(dest_cpu)))
6025                return ret;
6026
6027        rq_src = cpu_rq(src_cpu);
6028        rq_dest = cpu_rq(dest_cpu);
6029
6030        double_rq_lock(rq_src, rq_dest);
6031        /* Already moved. */
6032        if (task_cpu(p) != src_cpu)
6033                goto done;
6034        /* Affinity changed (again). */
6035        if (!cpu_isset(dest_cpu, p->cpus_allowed))
6036                goto fail;
6037
6038        on_rq = p->se.on_rq;
6039        if (on_rq)
6040                deactivate_task(rq_src, p, 0);
6041
6042        set_task_cpu(p, dest_cpu);
6043        if (on_rq) {
6044                activate_task(rq_dest, p, 0);
6045                check_preempt_curr(rq_dest, p, 0);
6046        }
6047done:
6048        ret = 1;
6049fail:
6050        double_rq_unlock(rq_src, rq_dest);
6051        return ret;
6052}
6053
6054/*
6055 * migration_thread - this is a highprio system thread that performs
6056 * thread migration by bumping thread off CPU then 'pushing' onto
6057 * another runqueue.
6058 */
6059static int migration_thread(void *data)
6060{
6061        int cpu = (long)data;
6062        struct rq *rq;
6063
6064        rq = cpu_rq(cpu);
6065        BUG_ON(rq->migration_thread != current);
6066
6067        set_current_state(TASK_INTERRUPTIBLE);
6068        while (!kthread_should_stop()) {
6069                struct migration_req *req;
6070                struct list_head *head;
6071
6072                spin_lock_irq(&rq->lock);
6073
6074                if (cpu_is_offline(cpu)) {
6075                        spin_unlock_irq(&rq->lock);
6076                        goto wait_to_die;
6077                }
6078
6079                if (rq->active_balance) {
6080                        active_load_balance(rq, cpu);
6081                        rq->active_balance = 0;
6082                }
6083
6084                head = &rq->migration_queue;
6085
6086                if (list_empty(head)) {
6087                        spin_unlock_irq(&rq->lock);
6088                        schedule();
6089                        set_current_state(TASK_INTERRUPTIBLE);
6090                        continue;
6091                }
6092                req = list_entry(head->next, struct migration_req, list);
6093                list_del_init(head->next);
6094
6095                spin_unlock(&rq->lock);
6096                __migrate_task(req->task, cpu, req->dest_cpu);
6097                local_irq_enable();
6098
6099                complete(&req->done);
6100        }
6101        __set_current_state(TASK_RUNNING);
6102        return 0;
6103
6104wait_to_die:
6105        /* Wait for kthread_stop */
6106        set_current_state(TASK_INTERRUPTIBLE);
6107        while (!kthread_should_stop()) {
6108                schedule();
6109                set_current_state(TASK_INTERRUPTIBLE);
6110        }
6111        __set_current_state(TASK_RUNNING);
6112        return 0;
6113}
6114
6115#ifdef CONFIG_HOTPLUG_CPU
6116
6117static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6118{
6119        int ret;
6120
6121        local_irq_disable();
6122        ret = __migrate_task(p, src_cpu, dest_cpu);
6123        local_irq_enable();
6124        return ret;
6125}
6126
6127/*
6128 * Figure out where task on dead CPU should go, use force if necessary.
6129 * NOTE: interrupts should be disabled by the caller
6130 */
6131static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6132{
6133        unsigned long flags;
6134        cpumask_t mask;
6135        struct rq *rq;
6136        int dest_cpu;
6137
6138        do {
6139                /* On same node? */
6140                mask = node_to_cpumask(cpu_to_node(dead_cpu));
6141                cpus_and(mask, mask, p->cpus_allowed);
6142                dest_cpu = any_online_cpu(mask);
6143
6144                /* On any allowed CPU? */
6145                if (dest_cpu >= nr_cpu_ids)
6146                        dest_cpu = any_online_cpu(p->cpus_allowed);
6147
6148                /* No more Mr. Nice Guy. */
6149                if (dest_cpu >= nr_cpu_ids) {
6150                        cpumask_t cpus_allowed;
6151
6152                        cpuset_cpus_allowed_locked(p, &cpus_allowed);
6153                        /*
6154                         * Try to stay on the same cpuset, where the
6155                         * current cpuset may be a subset of all cpus.
6156                         * The cpuset_cpus_allowed_locked() variant of
6157                         * cpuset_cpus_allowed() will not block. It must be
6158                         * called within calls to cpuset_lock/cpuset_unlock.
6159                         */
6160                        rq = task_rq_lock(p, &flags);
6161                        p->cpus_allowed = cpus_allowed;
6162                        dest_cpu = any_online_cpu(p->cpus_allowed);
6163                        task_rq_unlock(rq, &flags);
6164
6165                        /*
6166                         * Don't tell them about moving exiting tasks or
6167                         * kernel threads (both mm NULL), since they never
6168                         * leave kernel.
6169                         */
6170                        if (p->mm && printk_ratelimit()) {
6171                                printk(KERN_INFO "process %d (%s) no "
6172                                       "longer affine to cpu%d\n",
6173                                        task_pid_nr(p), p->comm, dead_cpu);
6174                        }
6175                }
6176        } while (!__migrate_task_irq(p, dead_cpu, dest_cpu));
6177}
6178
6179/*
6180 * While a dead CPU has no uninterruptible tasks queued at this point,
6181 * it might still have a nonzero ->nr_uninterruptible counter, because
6182 * for performance reasons the counter is not stricly tracking tasks to
6183 * their home CPUs. So we just add the counter to another CPU's counter,
6184 * to keep the global sum constant after CPU-down:
6185 */
6186static void migrate_nr_uninterruptible(struct rq *rq_src)
6187{
6188        struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR));
6189        unsigned long flags;
6190
6191        local_irq_save(flags);
6192        double_rq_lock(rq_src, rq_dest);
6193        rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
6194        rq_src->nr_uninterruptible = 0;
6195        double_rq_unlock(rq_src, rq_dest);
6196        local_irq_restore(flags);
6197}
6198
6199/* Run through task list and migrate tasks from the dead cpu. */
6200static void migrate_live_tasks(int src_cpu)
6201{
6202        struct task_struct *p, *t;
6203
6204        read_lock(&tasklist_lock);
6205
6206        do_each_thread(t, p) {
6207                if (p == current)
6208                        continue;
6209
6210                if (task_cpu(p) == src_cpu)
6211                        move_task_off_dead_cpu(src_cpu, p);
6212        } while_each_thread(t, p);
6213
6214        read_unlock(&tasklist_lock);
6215}
6216
6217/*
6218 * Schedules idle task to be the next runnable task on current CPU.
6219 * It does so by boosting its priority to highest possible.
6220 * Used by CPU offline code.
6221 */
6222void sched_idle_next(void)
6223{
6224        int this_cpu = smp_processor_id();
6225        struct rq *rq = cpu_rq(this_cpu);
6226        struct task_struct *p = rq->idle;
6227        unsigned long flags;
6228
6229        /* cpu has to be offline */
6230        BUG_ON(cpu_online(this_cpu));
6231
6232        /*
6233         * Strictly not necessary since rest of the CPUs are stopped by now
6234         * and interrupts disabled on the current cpu.
6235         */
6236        spin_lock_irqsave(&rq->lock, flags);
6237
6238        __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
6239
6240        update_rq_clock(rq);
6241        activate_task(rq, p, 0);
6242
6243        spin_unlock_irqrestore(&rq->lock, flags);
6244}
6245
6246/*
6247 * Ensures that the idle task is using init_mm right before its cpu goes
6248 * offline.
6249 */
6250void idle_task_exit(void)
6251{
6252        struct mm_struct *mm = current->active_mm;
6253
6254        BUG_ON(cpu_online(smp_processor_id()));
6255
6256        if (mm != &init_mm)
6257                switch_mm(mm, &init_mm, current);
6258        mmdrop(mm);
6259}
6260
6261/* called under rq->lock with disabled interrupts */
6262static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
6263{
6264        struct rq *rq = cpu_rq(dead_cpu);
6265
6266        /* Must be exiting, otherwise would be on tasklist. */
6267        BUG_ON(!p->exit_state);
6268
6269        /* Cannot have done final schedule yet: would have vanished. */
6270        BUG_ON(p->state == TASK_DEAD);
6271
6272        get_task_struct(p);
6273
6274        /*
6275         * Drop lock around migration; if someone else moves it,
6276         * that's OK. No task can be added to this CPU, so iteration is
6277         * fine.
6278         */
6279        spin_unlock_irq(&rq->lock);
6280        move_task_off_dead_cpu(dead_cpu, p);
6281        spin_lock_irq(&rq->lock);
6282
6283        put_task_struct(p);
6284}
6285
6286/* release_task() removes task from tasklist, so we won't find dead tasks. */
6287static void migrate_dead_tasks(unsigned int dead_cpu)
6288{
6289        struct rq *rq = cpu_rq(dead_cpu);
6290        struct task_struct *next;
6291
6292        for ( ; ; ) {
6293                if (!rq->nr_running)
6294                        break;
6295                update_rq_clock(rq);
6296                next = pick_next_task(rq, rq->curr);
6297                if (!next)
6298                        break;
6299                next->sched_class->put_prev_task(rq, next);
6300                migrate_dead(dead_cpu, next);
6301
6302        }
6303}
6304#endif /* CONFIG_HOTPLUG_CPU */
6305
6306#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
6307
6308static struct ctl_table sd_ctl_dir[] = {
6309        {
6310                .procname        = "sched_domain",
6311                .mode                = 0555,
6312        },
6313        {0, },
6314};
6315
6316static struct ctl_table sd_ctl_root[] = {
6317        {
6318                .ctl_name        = CTL_KERN,
6319                .procname        = "kernel",
6320                .mode                = 0555,
6321                .child                = sd_ctl_dir,
6322        },
6323        {0, },
6324};
6325
6326static struct ctl_table *sd_alloc_ctl_entry(int n)
6327{
6328        struct ctl_table *entry =
6329                kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
6330
6331        return entry;
6332}
6333
6334static void sd_free_ctl_entry(struct ctl_table **tablep)
6335{
6336        struct ctl_table *entry;
6337
6338        /*
6339         * In the intermediate directories, both the child directory and
6340         * procname are dynamically allocated and could fail but the mode
6341         * will always be set. In the lowest directory the names are
6342         * static strings and all have proc handlers.
6343         */
6344        for (entry = *tablep; entry->mode; entry++) {
6345                if (entry->child)
6346                        sd_free_ctl_entry(&entry->child);
6347                if (entry->proc_handler == NULL)
6348                        kfree(entry->procname);
6349        }
6350
6351        kfree(*tablep);
6352        *tablep = NULL;
6353}
6354
6355static void
6356set_table_entry(struct ctl_table *entry,
6357                const char *procname, void *data, int maxlen,
6358                mode_t mode, proc_handler *proc_handler)
6359{
6360        entry->procname = procname;
6361        entry->data = data;
6362        entry->maxlen = maxlen;
6363        entry->mode = mode;
6364        entry->proc_handler = proc_handler;
6365}
6366
6367static struct ctl_table *
6368sd_alloc_ctl_domain_table(struct sched_domain *sd)
6369{
6370        struct ctl_table *table = sd_alloc_ctl_entry(13);
6371
6372        if (table == NULL)
6373                return NULL;
6374
6375        set_table_entry(&table[0], "min_interval", &sd->min_interval,
6376                sizeof(long), 0644, proc_doulongvec_minmax);
6377        set_table_entry(&table[1], "max_interval", &sd->max_interval,
6378                sizeof(long), 0644, proc_doulongvec_minmax);
6379        set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
6380                sizeof(int), 0644, proc_dointvec_minmax);
6381        set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
6382                sizeof(int), 0644, proc_dointvec_minmax);
6383        set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
6384                sizeof(int), 0644, proc_dointvec_minmax);
6385        set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
6386                sizeof(int), 0644, proc_dointvec_minmax);
6387        set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
6388                sizeof(int), 0644, proc_dointvec_minmax);
6389        set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
6390                sizeof(int), 0644, proc_dointvec_minmax);
6391        set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
6392                sizeof(int), 0644, proc_dointvec_minmax);
6393        set_table_entry(&table[9], "cache_nice_tries",
6394                &sd->cache_nice_tries,
6395                sizeof(int), 0644, proc_dointvec_minmax);
6396        set_table_entry(&table[10], "flags", &sd->flags,
6397                sizeof(int), 0644, proc_dointvec_minmax);
6398        set_table_entry(&table[11], "name", sd->name,
6399                CORENAME_MAX_SIZE, 0444, proc_dostring);
6400        /* &table[12] is terminator */
6401
6402        return table;
6403}
6404
6405static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
6406{
6407        struct ctl_table *entry, *table;
6408        struct sched_domain *sd;
6409        int domain_num = 0, i;
6410        char buf[32];
6411
6412        for_each_domain(cpu, sd)
6413                domain_num++;
6414        entry = table = sd_alloc_ctl_entry(domain_num + 1);
6415        if (table == NULL)
6416                return NULL;
6417
6418        i = 0;
6419        for_each_domain(cpu, sd) {
6420                snprintf(buf, 32, "domain%d", i);
6421                entry->procname = kstrdup(buf, GFP_KERNEL);
6422                entry->mode = 0555;
6423                entry->child = sd_alloc_ctl_domain_table(sd);
6424                entry++;
6425                i++;
6426        }
6427        return table;
6428}
6429
6430static struct ctl_table_header *sd_sysctl_header;
6431static void register_sched_domain_sysctl(void)
6432{
6433        int i, cpu_num = num_online_cpus();
6434        struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
6435        char buf[32];
6436
6437        WARN_ON(sd_ctl_dir[0].child);
6438        sd_ctl_dir[0].child = entry;
6439
6440        if (entry == NULL)
6441                return;
6442
6443        for_each_online_cpu(i) {
6444                snprintf(buf, 32, "cpu%d", i);
6445                entry->procname = kstrdup(buf, GFP_KERNEL);
6446                entry->mode = 0555;
6447                entry->child = sd_alloc_ctl_cpu_table(i);
6448                entry++;
6449        }
6450
6451        WARN_ON(sd_sysctl_header);
6452        sd_sysctl_header = register_sysctl_table(sd_ctl_root);
6453}
6454
6455/* may be called multiple times per register */
6456static void unregister_sched_domain_sysctl(void)
6457{
6458        if (sd_sysctl_header)
6459                unregister_sysctl_table(sd_sysctl_header);
6460        sd_sysctl_header = NULL;
6461        if (sd_ctl_dir[0].child)
6462                sd_free_ctl_entry(&sd_ctl_dir[0].child);
6463}
6464#else
6465static void register_sched_domain_sysctl(void)
6466{
6467}
6468static void unregister_sched_domain_sysctl(void)
6469{
6470}
6471#endif
6472
6473static void set_rq_online(struct rq *rq)
6474{
6475        if (!rq->online) {
6476                const struct sched_class *class;
6477
6478                cpu_set(rq->cpu, rq->rd->online);
6479                rq->online = 1;
6480
6481                for_each_class(class) {
6482                        if (class->rq_online)
6483                                class->rq_online(rq);
6484                }
6485        }
6486}
6487
6488static void set_rq_offline(struct rq *rq)
6489{
6490        if (rq->online) {
6491                const struct sched_class *class;
6492
6493                for_each_class(class) {
6494                        if (class->rq_offline)
6495                                class->rq_offline(rq);
6496                }
6497
6498                cpu_clear(rq->cpu, rq->rd->online);
6499                rq->online = 0;
6500        }
6501}
6502
6503/*
6504 * migration_call - callback that gets triggered when a CPU is added.
6505 * Here we can start up the necessary migration thread for the new CPU.
6506 */
6507static int __cpuinit
6508migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6509{
6510        struct task_struct *p;
6511        int cpu = (long)hcpu;
6512        unsigned long flags;
6513        struct rq *rq;
6514
6515        switch (action) {
6516
6517        case CPU_UP_PREPARE:
6518        case CPU_UP_PREPARE_FROZEN:
6519                p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
6520                if (IS_ERR(p))
6521                        return NOTIFY_BAD;
6522                kthread_bind(p, cpu);
6523                /* Must be high prio: stop_machine expects to yield to it. */
6524                rq = task_rq_lock(p, &flags);
6525                __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
6526                task_rq_unlock(rq, &flags);
6527                cpu_rq(cpu)->migration_thread = p;
6528                break;
6529
6530        case CPU_ONLINE:
6531        case CPU_ONLINE_FROZEN:
6532                /* Strictly unnecessary, as first user will wake it. */
6533                wake_up_process(cpu_rq(cpu)->migration_thread);
6534
6535                /* Update our root-domain */
6536                rq = cpu_rq(cpu);
6537                spin_lock_irqsave(&rq->lock, flags);
6538                if (rq->rd) {
6539                        BUG_ON(!cpu_isset(cpu, rq->rd->span));
6540
6541                        set_rq_online(rq);
6542                }
6543                spin_unlock_irqrestore(&rq->lock, flags);
6544                break;
6545
6546#ifdef CONFIG_HOTPLUG_CPU
6547        case CPU_UP_CANCELED:
6548        case CPU_UP_CANCELED_FROZEN:
6549                if (!cpu_rq(cpu)->migration_thread)
6550                        break;
6551                /* Unbind it from offline cpu so it can run. Fall thru. */
6552                kthread_bind(cpu_rq(cpu)->migration_thread,
6553                             any_online_cpu(cpu_online_map));
6554                kthread_stop(cpu_rq(cpu)->migration_thread);
6555                cpu_rq(cpu)->migration_thread = NULL;
6556                break;
6557
6558        case CPU_DEAD:
6559        case CPU_DEAD_FROZEN:
6560                cpuset_lock(); /* around calls to cpuset_cpus_allowed_lock() */
6561                migrate_live_tasks(cpu);
6562                rq = cpu_rq(cpu);
6563                kthread_stop(rq->migration_thread);
6564                rq->migration_thread = NULL;
6565                /* Idle task back to normal (off runqueue, low prio) */
6566                spin_lock_irq(&rq->lock);
6567                update_rq_clock(rq);
6568                deactivate_task(rq, rq->idle, 0);
6569                rq->idle->static_prio = MAX_PRIO;
6570                __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
6571                rq->idle->sched_class = &idle_sched_class;
6572                migrate_dead_tasks(cpu);
6573                spin_unlock_irq(&rq->lock);
6574                cpuset_unlock();
6575                migrate_nr_uninterruptible(rq);
6576                BUG_ON(rq->nr_running != 0);
6577
6578                /*
6579                 * No need to migrate the tasks: it was best-effort if
6580                 * they didn't take sched_hotcpu_mutex. Just wake up
6581                 * the requestors.
6582                 */
6583                spin_lock_irq(&rq->lock);
6584                while (!list_empty(&rq->migration_queue)) {
6585                        struct migration_req *req;
6586
6587                        req = list_entry(rq->migration_queue.next,
6588                                         struct migration_req, list);
6589                        list_del_init(&req->list);
6590                        spin_unlock_irq(&rq->lock);
6591                        complete(&req->done);
6592                        spin_lock_irq(&rq->lock);
6593                }
6594                spin_unlock_irq(&rq->lock);
6595                break;
6596
6597        case CPU_DYING:
6598        case CPU_DYING_FROZEN:
6599                /* Update our root-domain */
6600                rq = cpu_rq(cpu);
6601                spin_lock_irqsave(&rq->lock, flags);
6602                if (rq->rd) {
6603                        BUG_ON(!cpu_isset(cpu, rq->rd->span));
6604                        set_rq_offline(rq);
6605                }
6606                spin_unlock_irqrestore(&rq->lock, flags);
6607                break;
6608#endif
6609        }
6610        return NOTIFY_OK;
6611}
6612
6613/* Register at highest priority so that task migration (migrate_all_tasks)
6614 * happens before everything else.
6615 */
6616static struct notifier_block __cpuinitdata migration_notifier = {
6617        .notifier_call = migration_call,
6618        .priority = 10
6619};
6620
6621static int __init migration_init(void)
6622{
6623        void *cpu = (void *)(long)smp_processor_id();
6624        int err;
6625
6626        /* Start one for the boot CPU: */
6627        err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
6628        BUG_ON(err == NOTIFY_BAD);
6629        migration_call(&migration_notifier, CPU_ONLINE, cpu);
6630        register_cpu_notifier(&migration_notifier);
6631
6632        return err;
6633}
6634early_initcall(migration_init);
6635#endif
6636
6637#ifdef CONFIG_SMP
6638
6639#ifdef CONFIG_SCHED_DEBUG
6640
6641static inline const char *sd_level_to_string(enum sched_domain_level lvl)
6642{
6643        switch (lvl) {
6644        case SD_LV_NONE:
6645                        return "NONE";
6646        case SD_LV_SIBLING:
6647                        return "SIBLING";
6648        case SD_LV_MC:
6649                        return "MC";
6650        case SD_LV_CPU:
6651                        return "CPU";
6652        case SD_LV_NODE:
6653                        return "NODE";
6654        case SD_LV_ALLNODES:
6655                        return "ALLNODES";
6656        case SD_LV_MAX:
6657                        return "MAX";
6658
6659        }
6660        return "MAX";
6661}
6662
6663static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6664                                  cpumask_t *groupmask)
6665{
6666        struct sched_group *group = sd->groups;
6667        char str[256];
6668
6669        cpulist_scnprintf(str, sizeof(str), sd->span);
6670        cpus_clear(*groupmask);
6671
6672        printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
6673
6674        if (!(sd->flags & SD_LOAD_BALANCE)) {
6675                printk("does not load-balance\n");
6676                if (sd->parent)
6677                        printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
6678                                        " has parent");
6679                return -1;
6680        }
6681
6682        printk(KERN_CONT "span %s level %s\n",
6683                str, sd_level_to_string(sd->level));
6684
6685        if (!cpu_isset(cpu, sd->span)) {
6686                printk(KERN_ERR "ERROR: domain->span does not contain "
6687                                "CPU%d\n", cpu);
6688        }
6689        if (!cpu_isset(cpu, group->cpumask)) {
6690                printk(KERN_ERR "ERROR: domain->groups does not contain"
6691                                " CPU%d\n", cpu);
6692        }
6693
6694        printk(KERN_DEBUG "%*s groups:", level + 1, "");
6695        do {
6696                if (!group) {
6697                        printk("\n");
6698                        printk(KERN_ERR "ERROR: group is NULL\n");
6699                        break;
6700                }
6701
6702                if (!group->__cpu_power) {
6703                        printk(KERN_CONT "\n");
6704                        printk(KERN_ERR "ERROR: domain->cpu_power not "
6705                                        "set\n");
6706                        break;
6707                }
6708
6709                if (!cpus_weight(group->cpumask)) {
6710                        printk(KERN_CONT "\n");
6711                        printk(KERN_ERR "ERROR: empty group\n");
6712                        break;
6713                }
6714
6715                if (cpus_intersects(*groupmask, group->cpumask)) {
6716                        printk(KERN_CONT "\n");
6717                        printk(KERN_ERR "ERROR: repeated CPUs\n");
6718                        break;
6719                }
6720
6721                cpus_or(*groupmask, *groupmask, group->cpumask);
6722
6723                cpulist_scnprintf(str, sizeof(str), group->cpumask);
6724                printk(KERN_CONT " %s", str);
6725
6726                group = group->next;
6727        } while (group != sd->groups);
6728        printk(KERN_CONT "\n");
6729
6730        if (!cpus_equal(sd->span, *groupmask))
6731                printk(KERN_ERR "ERROR: groups don't span domain->span\n");
6732
6733        if (sd->parent && !cpus_subset(*groupmask, sd->parent->span))
6734                printk(KERN_ERR "ERROR: parent span is not a superset "
6735                        "of domain->span\n");
6736        return 0;
6737}
6738
6739static void sched_domain_debug(struct sched_domain *sd, int cpu)
6740{
6741        cpumask_t *groupmask;
6742        int level = 0;
6743
6744        if (!sd) {
6745                printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
6746                return;
6747        }
6748
6749        printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
6750
6751        groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
6752        if (!groupmask) {
6753                printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
6754                return;
6755        }
6756
6757        for (;;) {
6758                if (sched_domain_debug_one(sd, cpu, level, groupmask))
6759                        break;
6760                level++;
6761                sd = sd->parent;
6762                if (!sd)
6763                        break;
6764        }
6765        kfree(groupmask);
6766}
6767#else /* !CONFIG_SCHED_DEBUG */
6768# define sched_domain_debug(sd, cpu) do { } while (0)
6769#endif /* CONFIG_SCHED_DEBUG */
6770
6771static int sd_degenerate(struct sched_domain *sd)
6772{
6773        if (cpus_weight(sd->span) == 1)
6774                return 1;
6775
6776        /* Following flags need at least 2 groups */
6777        if (sd->flags & (SD_LOAD_BALANCE |
6778                         SD_BALANCE_NEWIDLE |
6779                         SD_BALANCE_FORK |
6780                         SD_BALANCE_EXEC |
6781                         SD_SHARE_CPUPOWER |
6782                         SD_SHARE_PKG_RESOURCES)) {
6783                if (sd->groups != sd->groups->next)
6784                        return 0;
6785        }
6786
6787        /* Following flags don't use groups */
6788        if (sd->flags & (SD_WAKE_IDLE |
6789                         SD_WAKE_AFFINE |
6790                         SD_WAKE_BALANCE))
6791                return 0;
6792
6793        return 1;
6794}
6795
6796static int
6797sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6798{
6799        unsigned long cflags = sd->flags, pflags = parent->flags;
6800
6801        if (sd_degenerate(parent))
6802                return 1;
6803
6804        if (!cpus_equal(sd->span, parent->span))
6805                return 0;
6806
6807        /* Does parent contain flags not in child? */
6808        /* WAKE_BALANCE is a subset of WAKE_AFFINE */
6809        if (cflags & SD_WAKE_AFFINE)
6810                pflags &= ~SD_WAKE_BALANCE;
6811        /* Flags needing groups don't count if only 1 group in parent */
6812        if (parent->groups == parent->groups->next) {
6813                pflags &= ~(SD_LOAD_BALANCE |
6814                                SD_BALANCE_NEWIDLE |
6815                                SD_BALANCE_FORK |
6816                                SD_BALANCE_EXEC |
6817                                SD_SHARE_CPUPOWER |
6818                                SD_SHARE_PKG_RESOURCES);
6819        }
6820        if (~cflags & pflags)
6821                return 0;
6822
6823        return 1;
6824}
6825
6826static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6827{
6828        unsigned long flags;
6829
6830        spin_lock_irqsave(&rq->lock, flags);
6831
6832        if (rq->rd) {
6833                struct root_domain *old_rd = rq->rd;
6834
6835                if (cpu_isset(rq->cpu, old_rd->online))
6836                        set_rq_offline(rq);
6837
6838                cpu_clear(rq->cpu, old_rd->span);
6839
6840                if (atomic_dec_and_test(&old_rd->refcount))
6841                        kfree(old_rd);
6842        }
6843
6844        atomic_inc(&rd->refcount);
6845        rq->rd = rd;
6846
6847        cpu_set(rq->cpu, rd->span);
6848        if (cpu_isset(rq->cpu, cpu_online_map))
6849                set_rq_online(rq);
6850
6851        spin_unlock_irqrestore(&rq->lock, flags);
6852}
6853
6854static void init_rootdomain(struct root_domain *rd)
6855{
6856        memset(rd, 0, sizeof(*rd));
6857
6858        cpus_clear(rd->span);
6859        cpus_clear(rd->online);
6860
6861        cpupri_init(&rd->cpupri);
6862}
6863
6864static void init_defrootdomain(void)
6865{
6866        init_rootdomain(&def_root_domain);
6867        atomic_set(&def_root_domain.refcount, 1);
6868}
6869
6870static struct root_domain *alloc_rootdomain(void)
6871{
6872        struct root_domain *rd;
6873
6874        rd = kmalloc(sizeof(*rd), GFP_KERNEL);
6875        if (!rd)
6876                return NULL;
6877
6878        init_rootdomain(rd);
6879
6880        return rd;
6881}
6882
6883/*
6884 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
6885 * hold the hotplug lock.
6886 */
6887static void
6888cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6889{
6890        struct rq *rq = cpu_rq(cpu);
6891        struct sched_domain *tmp;
6892
6893        /* Remove the sched domains which do not contribute to scheduling. */
6894        for (tmp = sd; tmp; ) {
6895                struct sched_domain *parent = tmp->parent;
6896                if (!parent)
6897                        break;
6898
6899                if (sd_parent_degenerate(tmp, parent)) {
6900                        tmp->parent = parent->parent;
6901                        if (parent->parent)
6902                                parent->parent->child = tmp;
6903                } else
6904                        tmp = tmp->parent;
6905        }
6906
6907        if (sd && sd_degenerate(sd)) {
6908                sd = sd->parent;
6909                if (sd)
6910                        sd->child = NULL;
6911        }
6912
6913        sched_domain_debug(sd, cpu);
6914
6915        rq_attach_root(rq, rd);
6916        rcu_assign_pointer(rq->sd, sd);
6917}
6918
6919/* cpus with isolated domains */
6920static cpumask_t cpu_isolated_map = CPU_MASK_NONE;
6921
6922/* Setup the mask of cpus configured for isolated domains */
6923static int __init isolated_cpu_setup(char *str)
6924{
6925        static int __initdata ints[NR_CPUS];
6926        int i;
6927
6928        str = get_options(str, ARRAY_SIZE(ints), ints);
6929        cpus_clear(cpu_isolated_map);
6930        for (i = 1; i <= ints[0]; i++)
6931                if (ints[i] < NR_CPUS)
6932                        cpu_set(ints[i], cpu_isolated_map);
6933        return 1;
6934}
6935
6936__setup("isolcpus=", isolated_cpu_setup);
6937
6938/*
6939 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6940 * to a function which identifies what group(along with sched group) a CPU
6941 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS
6942 * (due to the fact that we keep track of groups covered with a cpumask_t).
6943 *
6944 * init_sched_build_groups will build a circular linked list of the groups
6945 * covered by the given span, and will set each group's ->cpumask correctly,
6946 * and ->cpu_power to 0.
6947 */
6948static void
6949init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map,
6950                        int (*group_fn)(int cpu, const cpumask_t *cpu_map,
6951                                        struct sched_group **sg,
6952                                        cpumask_t *tmpmask),
6953                        cpumask_t *covered, cpumask_t *tmpmask)
6954{
6955        struct sched_group *first = NULL, *last = NULL;
6956        int i;
6957
6958        cpus_clear(*covered);
6959
6960        for_each_cpu_mask_nr(i, *span) {
6961                struct sched_group *sg;
6962                int group = group_fn(i, cpu_map, &sg, tmpmask);
6963                int j;
6964
6965                if (cpu_isset(i, *covered))
6966                        continue;
6967
6968                cpus_clear(sg->cpumask);
6969                sg->__cpu_power = 0;
6970
6971                for_each_cpu_mask_nr(j, *span) {
6972                        if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6973                                continue;
6974
6975                        cpu_set(j, *covered);
6976                        cpu_set(j, sg->cpumask);
6977                }
6978                if (!first)
6979                        first = sg;
6980                if (last)
6981                        last->next = sg;
6982                last = sg;
6983        }
6984        last->next = first;
6985}
6986
6987#define SD_NODES_PER_DOMAIN 16
6988
6989#ifdef CONFIG_NUMA
6990
6991/**
6992 * find_next_best_node - find the next node to include in a sched_domain
6993 * @node: node whose sched_domain we're building
6994 * @used_nodes: nodes already in the sched_domain
6995 *
6996 * Find the next node to include in a given scheduling domain. Simply
6997 * finds the closest node not already in the @used_nodes map.
6998 *
6999 * Should use nodemask_t.
7000 */
7001static int find_next_best_node(int node, nodemask_t *used_nodes)
7002{
7003        int i, n, val, min_val, best_node = 0;
7004
7005        min_val = INT_MAX;
7006
7007        for (i = 0; i < nr_node_ids; i++) {
7008                /* Start at @node */
7009                n = (node + i) % nr_node_ids;
7010
7011                if (!nr_cpus_node(n))
7012                        continue;
7013
7014                /* Skip already used nodes */
7015                if (node_isset(n, *used_nodes))
7016                        continue;
7017
7018                /* Simple min distance search */
7019                val = node_distance(node, n);
7020
7021                if (val < min_val) {
7022                        min_val = val;
7023                        best_node = n;
7024                }
7025        }
7026
7027        node_set(best_node, *used_nodes);
7028        return best_node;
7029}
7030
7031/**
7032 * sched_domain_node_span - get a cpumask for a node's sched_domain
7033 * @node: node whose cpumask we're constructing
7034 * @span: resulting cpumask
7035 *
7036 * Given a node, construct a good cpumask for its sched_domain to span. It
7037 * should be one that prevents unnecessary balancing, but also spreads tasks
7038 * out optimally.
7039 */
7040static void sched_domain_node_span(int node, cpumask_t *span)
7041{
7042        nodemask_t used_nodes;
7043        node_to_cpumask_ptr(nodemask, node);
7044        int i;
7045
7046        cpus_clear(*span);
7047        nodes_clear(used_nodes);
7048
7049        cpus_or(*span, *span, *nodemask);
7050        node_set(node, used_nodes);
7051
7052        for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
7053                int next_node = find_next_best_node(node, &used_nodes);
7054
7055                node_to_cpumask_ptr_next(nodemask, next_node);
7056                cpus_or(*span, *span, *nodemask);
7057        }
7058}
7059#endif /* CONFIG_NUMA */
7060
7061int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7062
7063/*
7064 * SMT sched-domains:
7065 */
7066#ifdef CONFIG_SCHED_SMT
7067static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
7068static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);
7069
7070static int
7071cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7072                 cpumask_t *unused)
7073{
7074        if (sg)
7075                *sg = &per_cpu(sched_group_cpus, cpu);
7076        return cpu;
7077}
7078#endif /* CONFIG_SCHED_SMT */
7079
7080/*
7081 * multi-core sched-domains:
7082 */
7083#ifdef CONFIG_SCHED_MC
7084static DEFINE_PER_CPU(struct sched_domain, core_domains);
7085static DEFINE_PER_CPU(struct sched_group, sched_group_core);
7086#endif /* CONFIG_SCHED_MC */
7087
7088#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
7089static int
7090cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7091                  cpumask_t *mask)
7092{
7093        int group;
7094
7095        *mask = per_cpu(cpu_sibling_map, cpu);
7096        cpus_and(*mask, *mask, *cpu_map);
7097        group = first_cpu(*mask);
7098        if (sg)
7099                *sg = &per_cpu(sched_group_core, group);
7100        return group;
7101}
7102#elif defined(CONFIG_SCHED_MC)
7103static int
7104cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7105                  cpumask_t *unused)
7106{
7107        if (sg)
7108                *sg = &per_cpu(sched_group_core, cpu);
7109        return cpu;
7110}
7111#endif
7112
7113static DEFINE_PER_CPU(struct sched_domain, phys_domains);
7114static DEFINE_PER_CPU(struct sched_group, sched_group_phys);
7115
7116static int
7117cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7118                  cpumask_t *mask)
7119{
7120        int group;
7121#ifdef CONFIG_SCHED_MC
7122        *mask = cpu_coregroup_map(cpu);
7123        cpus_and(*mask, *mask, *cpu_map);
7124        group = first_cpu(*mask);
7125#elif defined(CONFIG_SCHED_SMT)
7126        *mask = per_cpu(cpu_sibling_map, cpu);
7127        cpus_and(*mask, *mask, *cpu_map);
7128        group = first_cpu(*mask);
7129#else
7130        group = cpu;
7131#endif
7132        if (sg)
7133                *sg = &per_cpu(sched_group_phys, group);
7134        return group;
7135}
7136
7137#ifdef CONFIG_NUMA
7138/*
7139 * The init_sched_build_groups can't handle what we want to do with node
7140 * groups, so roll our own. Now each node has its own list of groups which
7141 * gets dynamically allocated.
7142 */
7143static DEFINE_PER_CPU(struct sched_domain, node_domains);
7144static struct sched_group ***sched_group_nodes_bycpu;
7145
7146static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
7147static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);
7148
7149static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,
7150                                 struct sched_group **sg, cpumask_t *nodemask)
7151{
7152        int group;
7153
7154        *nodemask = node_to_cpumask(cpu_to_node(cpu));
7155        cpus_and(*nodemask, *nodemask, *cpu_map);
7156        group = first_cpu(*nodemask);
7157
7158        if (sg)
7159                *sg = &per_cpu(sched_group_allnodes, group);
7160        return group;
7161}
7162
7163static void init_numa_sched_groups_power(struct sched_group *group_head)
7164{
7165        struct sched_group *sg = group_head;
7166        int j;
7167
7168        if (!sg)
7169                return;
7170        do {
7171                for_each_cpu_mask_nr(j, sg->cpumask) {
7172                        struct sched_domain *sd;
7173
7174                        sd = &per_cpu(phys_domains, j);
7175                        if (j != first_cpu(sd->groups->cpumask)) {
7176                                /*
7177                                 * Only add "power" once for each
7178                                 * physical package.
7179                                 */
7180                                continue;
7181                        }
7182
7183                        sg_inc_cpu_power(sg, sd->groups->__cpu_power);
7184                }
7185                sg = sg->next;
7186        } while (sg != group_head);
7187}
7188#endif /* CONFIG_NUMA */
7189
7190#ifdef CONFIG_NUMA
7191/* Free memory allocated for various sched_group structures */
7192static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7193{
7194        int cpu, i;
7195
7196        for_each_cpu_mask_nr(cpu, *cpu_map) {
7197                struct sched_group **sched_group_nodes
7198                        = sched_group_nodes_bycpu[cpu];
7199
7200                if (!sched_group_nodes)
7201                        continue;
7202
7203                for (i = 0; i < nr_node_ids; i++) {
7204                        struct sched_group *oldsg, *sg = sched_group_nodes[i];
7205
7206                        *nodemask = node_to_cpumask(i);
7207                        cpus_and(*nodemask, *nodemask, *cpu_map);
7208                        if (cpus_empty(*nodemask))
7209                                continue;
7210
7211                        if (sg == NULL)
7212                                continue;
7213                        sg = sg->next;
7214next_sg:
7215                        oldsg = sg;
7216                        sg = sg->next;
7217                        kfree(oldsg);
7218                        if (oldsg != sched_group_nodes[i])
7219                                goto next_sg;
7220                }
7221                kfree(sched_group_nodes);
7222                sched_group_nodes_bycpu[cpu] = NULL;
7223        }
7224}
7225#else /* !CONFIG_NUMA */
7226static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7227{
7228}
7229#endif /* CONFIG_NUMA */
7230
7231/*
7232 * Initialize sched groups cpu_power.
7233 *
7234 * cpu_power indicates the capacity of sched group, which is used while
7235 * distributing the load between different sched groups in a sched domain.
7236 * Typically cpu_power for all the groups in a sched domain will be same unless
7237 * there are asymmetries in the topology. If there are asymmetries, group
7238 * having more cpu_power will pickup more load compared to the group having
7239 * less cpu_power.
7240 *
7241 * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
7242 * the maximum number of tasks a group can handle in the presence of other idle
7243 * or lightly loaded groups in the same sched domain.
7244 */
7245static void init_sched_groups_power(int cpu, struct sched_domain *sd)
7246{
7247        struct sched_domain *child;
7248        struct sched_group *group;
7249
7250        WARN_ON(!sd || !sd->groups);
7251
7252        if (cpu != first_cpu(sd->groups->cpumask))
7253                return;
7254
7255        child = sd->child;
7256
7257        sd->groups->__cpu_power = 0;
7258
7259        /*
7260         * For perf policy, if the groups in child domain share resources
7261         * (for example cores sharing some portions of the cache hierarchy
7262         * or SMT), then set this domain groups cpu_power such that each group
7263         * can handle only one task, when there are other idle groups in the
7264         * same sched domain.
7265         */
7266        if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
7267                       (child->flags &
7268                        (SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
7269                sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
7270                return;
7271        }
7272
7273        /*
7274         * add cpu_power of each child group to this groups cpu_power
7275         */
7276        group = child->groups;
7277        do {
7278                sg_inc_cpu_power(sd->groups, group->__cpu_power);
7279                group = group->next;
7280        } while (group != child->groups);
7281}
7282
7283/*
7284 * Initializers for schedule domains
7285 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
7286 */
7287
7288#ifdef CONFIG_SCHED_DEBUG
7289# define SD_INIT_NAME(sd, type)                sd->name = #type
7290#else
7291# define SD_INIT_NAME(sd, type)                do { } while (0)
7292#endif
7293
7294#define        SD_INIT(sd, type)        sd_init_##type(sd)
7295
7296#define SD_INIT_FUNC(type)        \
7297static noinline void sd_init_##type(struct sched_domain *sd)        \
7298{                                                                \
7299        memset(sd, 0, sizeof(*sd));                                \
7300        *sd = SD_##type##_INIT;                                        \
7301        sd->level = SD_LV_##type;                                \
7302        SD_INIT_NAME(sd, type);                                        \
7303}
7304
7305SD_INIT_FUNC(CPU)
7306#ifdef CONFIG_NUMA
7307 SD_INIT_FUNC(ALLNODES)
7308 SD_INIT_FUNC(NODE)
7309#endif
7310#ifdef CONFIG_SCHED_SMT
7311 SD_INIT_FUNC(SIBLING)
7312#endif
7313#ifdef CONFIG_SCHED_MC
7314 SD_INIT_FUNC(MC)
7315#endif
7316
7317/*
7318 * To minimize stack usage kmalloc room for cpumasks and share the
7319 * space as the usage in build_sched_domains() dictates.  Used only
7320 * if the amount of space is significant.
7321 */
7322struct allmasks {
7323        cpumask_t tmpmask;                        /* make this one first */
7324        union {
7325                cpumask_t nodemask;
7326                cpumask_t this_sibling_map;
7327                cpumask_t this_core_map;
7328        };
7329        cpumask_t send_covered;
7330
7331#ifdef CONFIG_NUMA
7332        cpumask_t domainspan;
7333        cpumask_t covered;
7334        cpumask_t notcovered;
7335#endif
7336};
7337
7338#if        NR_CPUS > 128
7339#define        SCHED_CPUMASK_ALLOC                1
7340#define        SCHED_CPUMASK_FREE(v)                kfree(v)
7341#define        SCHED_CPUMASK_DECLARE(v)        struct allmasks *v
7342#else
7343#define        SCHED_CPUMASK_ALLOC                0
7344#define        SCHED_CPUMASK_FREE(v)
7345#define        SCHED_CPUMASK_DECLARE(v)        struct allmasks _v, *v = &_v
7346#endif
7347
7348#define        SCHED_CPUMASK_VAR(v, a)         cpumask_t *v = (cpumask_t *) \
7349                        ((unsigned long)(a) + offsetof(struct allmasks, v))
7350
7351static int default_relax_domain_level = -1;
7352
7353static int __init setup_relax_domain_level(char *str)
7354{
7355        unsigned long val;
7356
7357        val = simple_strtoul(str, NULL, 0);
7358        if (val < SD_LV_MAX)
7359                default_relax_domain_level = val;
7360
7361        return 1;
7362}
7363__setup("relax_domain_level=", setup_relax_domain_level);
7364
7365static void set_domain_attribute(struct sched_domain *sd,
7366                                 struct sched_domain_attr *attr)
7367{
7368        int request;
7369
7370        if (!attr || attr->relax_domain_level < 0) {
7371                if (default_relax_domain_level < 0)
7372                        return;
7373                else
7374                        request = default_relax_domain_level;
7375        } else
7376                request = attr->relax_domain_level;
7377        if (request < sd->level) {
7378                /* turn off idle balance on this domain */
7379                sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
7380        } else {
7381                /* turn on idle balance on this domain */
7382                sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
7383        }
7384}
7385
7386/*
7387 * Build sched domains for a given set of cpus and attach the sched domains
7388 * to the individual cpus
7389 */
7390static int __build_sched_domains(const cpumask_t *cpu_map,
7391                                 struct sched_domain_attr *attr)
7392{
7393        int i;
7394        struct root_domain *rd;
7395        SCHED_CPUMASK_DECLARE(allmasks);
7396        cpumask_t *tmpmask;
7397#ifdef CONFIG_NUMA
7398        struct sched_group **sched_group_nodes = NULL;
7399        int sd_allnodes = 0;
7400
7401        /*
7402         * Allocate the per-node list of sched groups
7403         */
7404        sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
7405                                    GFP_KERNEL);
7406        if (!sched_group_nodes) {
7407                printk(KERN_WARNING "Can not alloc sched group node list\n");
7408                return -ENOMEM;
7409        }
7410#endif
7411
7412        rd = alloc_rootdomain();
7413        if (!rd) {
7414                printk(KERN_WARNING "Cannot alloc root domain\n");
7415#ifdef CONFIG_NUMA
7416                kfree(sched_group_nodes);
7417#endif
7418                return -ENOMEM;
7419        }
7420
7421#if SCHED_CPUMASK_ALLOC
7422        /* get space for all scratch cpumask variables */
7423        allmasks = kmalloc(sizeof(*allmasks), GFP_KERNEL);
7424        if (!allmasks) {
7425                printk(KERN_WARNING "Cannot alloc cpumask array\n");
7426                kfree(rd);
7427#ifdef CONFIG_NUMA
7428                kfree(sched_group_nodes);
7429#endif
7430                return -ENOMEM;
7431        }
7432#endif
7433        tmpmask = (cpumask_t *)allmasks;
7434
7435
7436#ifdef CONFIG_NUMA
7437        sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
7438#endif
7439
7440        /*
7441         * Set up domains for cpus specified by the cpu_map.
7442         */
7443        for_each_cpu_mask_nr(i, *cpu_map) {
7444                struct sched_domain *sd = NULL, *p;
7445                SCHED_CPUMASK_VAR(nodemask, allmasks);
7446
7447                *nodemask = node_to_cpumask(cpu_to_node(i));
7448                cpus_and(*nodemask, *nodemask, *cpu_map);
7449
7450#ifdef CONFIG_NUMA
7451                if (cpus_weight(*cpu_map) >
7452                                SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) {
7453                        sd = &per_cpu(allnodes_domains, i);
7454                        SD_INIT(sd, ALLNODES);
7455                        set_domain_attribute(sd, attr);
7456                        sd->span = *cpu_map;
7457                        cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7458                        p = sd;
7459                        sd_allnodes = 1;
7460                } else
7461                        p = NULL;
7462
7463                sd = &per_cpu(node_domains, i);
7464                SD_INIT(sd, NODE);
7465                set_domain_attribute(sd, attr);
7466                sched_domain_node_span(cpu_to_node(i), &sd->span);
7467                sd->parent = p;
7468                if (p)
7469                        p->child = sd;
7470                cpus_and(sd->span, sd->span, *cpu_map);
7471#endif
7472
7473                p = sd;
7474                sd = &per_cpu(phys_domains, i);
7475                SD_INIT(sd, CPU);
7476                set_domain_attribute(sd, attr);
7477                sd->span = *nodemask;
7478                sd->parent = p;
7479                if (p)
7480                        p->child = sd;
7481                cpu_to_phys_group(i, cpu_map, &sd->groups, tmpmask);
7482
7483#ifdef CONFIG_SCHED_MC
7484                p = sd;
7485                sd = &per_cpu(core_domains, i);
7486                SD_INIT(sd, MC);
7487                set_domain_attribute(sd, attr);
7488                sd->span = cpu_coregroup_map(i);
7489                cpus_and(sd->span, sd->span, *cpu_map);
7490                sd->parent = p;
7491                p->child = sd;
7492                cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
7493#endif
7494
7495#ifdef CONFIG_SCHED_SMT
7496                p = sd;
7497                sd = &per_cpu(cpu_domains, i);
7498                SD_INIT(sd, SIBLING);
7499                set_domain_attribute(sd, attr);
7500                sd->span = per_cpu(cpu_sibling_map, i);
7501                cpus_and(sd->span, sd->span, *cpu_map);
7502                sd->parent = p;
7503                p->child = sd;
7504                cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
7505#endif
7506        }
7507
7508#ifdef CONFIG_SCHED_SMT
7509        /* Set up CPU (sibling) groups */
7510        for_each_cpu_mask_nr(i, *cpu_map) {
7511                SCHED_CPUMASK_VAR(this_sibling_map, allmasks);
7512                SCHED_CPUMASK_VAR(send_covered, allmasks);
7513
7514                *this_sibling_map = per_cpu(cpu_sibling_map, i);
7515                cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
7516                if (i != first_cpu(*this_sibling_map))
7517                        continue;
7518
7519                init_sched_build_groups(this_sibling_map, cpu_map,
7520                                        &cpu_to_cpu_group,
7521                                        send_covered, tmpmask);
7522        }
7523#endif
7524
7525#ifdef CONFIG_SCHED_MC
7526        /* Set up multi-core groups */
7527        for_each_cpu_mask_nr(i, *cpu_map) {
7528                SCHED_CPUMASK_VAR(this_core_map, allmasks);
7529                SCHED_CPUMASK_VAR(send_covered, allmasks);
7530
7531                *this_core_map = cpu_coregroup_map(i);
7532                cpus_and(*this_core_map, *this_core_map, *cpu_map);
7533                if (i != first_cpu(*this_core_map))
7534                        continue;
7535
7536                init_sched_build_groups(this_core_map, cpu_map,
7537                                        &cpu_to_core_group,
7538                                        send_covered, tmpmask);
7539        }
7540#endif
7541
7542        /* Set up physical groups */
7543        for (i = 0; i < nr_node_ids; i++) {
7544                SCHED_CPUMASK_VAR(nodemask, allmasks);
7545                SCHED_CPUMASK_VAR(send_covered, allmasks);
7546
7547                *nodemask = node_to_cpumask(i);
7548                cpus_and(*nodemask, *nodemask, *cpu_map);
7549                if (cpus_empty(*nodemask))
7550                        continue;
7551
7552                init_sched_build_groups(nodemask, cpu_map,
7553                                        &cpu_to_phys_group,
7554                                        send_covered, tmpmask);
7555        }
7556
7557#ifdef CONFIG_NUMA
7558        /* Set up node groups */
7559        if (sd_allnodes) {
7560                SCHED_CPUMASK_VAR(send_covered, allmasks);
7561
7562                init_sched_build_groups(cpu_map, cpu_map,
7563                                        &cpu_to_allnodes_group,
7564                                        send_covered, tmpmask);
7565        }
7566
7567        for (i = 0; i < nr_node_ids; i++) {
7568                /* Set up node groups */
7569                struct sched_group *sg, *prev;
7570                SCHED_CPUMASK_VAR(nodemask, allmasks);
7571                SCHED_CPUMASK_VAR(domainspan, allmasks);
7572                SCHED_CPUMASK_VAR(covered, allmasks);
7573                int j;
7574
7575                *nodemask = node_to_cpumask(i);
7576                cpus_clear(*covered);
7577
7578                cpus_and(*nodemask, *nodemask, *cpu_map);
7579                if (cpus_empty(*nodemask)) {
7580                        sched_group_nodes[i] = NULL;
7581                        continue;
7582                }
7583
7584                sched_domain_node_span(i, domainspan);
7585                cpus_and(*domainspan, *domainspan, *cpu_map);
7586
7587                sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i);
7588                if (!sg) {
7589                        printk(KERN_WARNING "Can not alloc domain group for "
7590                                "node %d\n", i);
7591                        goto error;
7592                }
7593                sched_group_nodes[i] = sg;
7594                for_each_cpu_mask_nr(j, *nodemask) {
7595                        struct sched_domain *sd;
7596
7597                        sd = &per_cpu(node_domains, j);
7598                        sd->groups = sg;
7599                }
7600                sg->__cpu_power = 0;
7601                sg->cpumask = *nodemask;
7602                sg->next = sg;
7603                cpus_or(*covered, *covered, *nodemask);
7604                prev = sg;
7605
7606                for (j = 0; j < nr_node_ids; j++) {
7607                        SCHED_CPUMASK_VAR(notcovered, allmasks);
7608                        int n = (i + j) % nr_node_ids;
7609                        node_to_cpumask_ptr(pnodemask, n);
7610
7611                        cpus_complement(*notcovered, *covered);
7612                        cpus_and(*tmpmask, *notcovered, *cpu_map);
7613                        cpus_and(*tmpmask, *tmpmask, *domainspan);
7614                        if (cpus_empty(*tmpmask))
7615                                break;
7616
7617                        cpus_and(*tmpmask, *tmpmask, *pnodemask);
7618                        if (cpus_empty(*tmpmask))
7619                                continue;
7620
7621                        sg = kmalloc_node(sizeof(struct sched_group),
7622                                          GFP_KERNEL, i);
7623                        if (!sg) {
7624                                printk(KERN_WARNING
7625                                "Can not alloc domain group for node %d\n", j);
7626                                goto error;
7627                        }
7628                        sg->__cpu_power = 0;
7629                        sg->cpumask = *tmpmask;
7630                        sg->next = prev->next;
7631                        cpus_or(*covered, *covered, *tmpmask);
7632                        prev->next = sg;
7633                        prev = sg;
7634                }
7635        }
7636#endif
7637
7638        /* Calculate CPU power for physical packages and nodes */
7639#ifdef CONFIG_SCHED_SMT
7640        for_each_cpu_mask_nr(i, *cpu_map) {
7641                struct sched_domain *sd = &per_cpu(cpu_domains, i);
7642
7643                init_sched_groups_power(i, sd);
7644        }
7645#endif
7646#ifdef CONFIG_SCHED_MC
7647        for_each_cpu_mask_nr(i, *cpu_map) {
7648                struct sched_domain *sd = &per_cpu(core_domains, i);
7649
7650                init_sched_groups_power(i, sd);
7651        }
7652#endif
7653
7654        for_each_cpu_mask_nr(i, *cpu_map) {
7655                struct sched_domain *sd = &per_cpu(phys_domains, i);
7656
7657                init_sched_groups_power(i, sd);
7658        }
7659
7660#ifdef CONFIG_NUMA
7661        for (i = 0; i < nr_node_ids; i++)
7662                init_numa_sched_groups_power(sched_group_nodes[i]);
7663
7664        if (sd_allnodes) {
7665                struct sched_group *sg;
7666
7667                cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg,
7668                                                                tmpmask);
7669                init_numa_sched_groups_power(sg);
7670        }
7671#endif
7672
7673        /* Attach the domains */
7674        for_each_cpu_mask_nr(i, *cpu_map) {
7675                struct sched_domain *sd;
7676#ifdef CONFIG_SCHED_SMT
7677                sd = &per_cpu(cpu_domains, i);
7678#elif defined(CONFIG_SCHED_MC)
7679                sd = &per_cpu(core_domains, i);
7680#else
7681                sd = &per_cpu(phys_domains, i);
7682#endif
7683                cpu_attach_domain(sd, rd, i);
7684        }
7685
7686        SCHED_CPUMASK_FREE((void *)allmasks);
7687        return 0;
7688
7689#ifdef CONFIG_NUMA
7690error:
7691        free_sched_groups(cpu_map, tmpmask);
7692        SCHED_CPUMASK_FREE((void *)allmasks);
7693        kfree(rd);
7694        return -ENOMEM;
7695#endif
7696}
7697
7698static int build_sched_domains(const cpumask_t *cpu_map)
7699{
7700        return __build_sched_domains(cpu_map, NULL);
7701}
7702
7703static cpumask_t *doms_cur;        /* current sched domains */
7704static int ndoms_cur;                /* number of sched domains in 'doms_cur' */
7705static struct sched_domain_attr *dattr_cur;
7706                                /* attribues of custom domains in 'doms_cur' */
7707
7708/*
7709 * Special case: If a kmalloc of a doms_cur partition (array of
7710 * cpumask_t) fails, then fallback to a single sched domain,
7711 * as determined by the single cpumask_t fallback_doms.
7712 */
7713static cpumask_t fallback_doms;
7714
7715void __attribute__((weak)) arch_update_cpu_topology(void)
7716{
7717}
7718
7719/*
7720 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
7721 * For now this just excludes isolated cpus, but could be used to
7722 * exclude other special cases in the future.
7723 */
7724static int arch_init_sched_domains(const cpumask_t *cpu_map)
7725{
7726        int err;
7727
7728        arch_update_cpu_topology();
7729        ndoms_cur = 1;
7730        doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
7731        if (!doms_cur)
7732                doms_cur = &fallback_doms;
7733        cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map);
7734        dattr_cur = NULL;
7735        err = build_sched_domains(doms_cur);
7736        register_sched_domain_sysctl();
7737
7738        return err;
7739}
7740
7741static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
7742                                       cpumask_t *tmpmask)
7743{
7744        free_sched_groups(cpu_map, tmpmask);
7745}
7746
7747/*
7748 * Detach sched domains from a group of cpus specified in cpu_map
7749 * These cpus will now be attached to the NULL domain
7750 */
7751static void detach_destroy_domains(const cpumask_t *cpu_map)
7752{
7753        cpumask_t tmpmask;
7754        int i;
7755
7756        unregister_sched_domain_sysctl();
7757
7758        for_each_cpu_mask_nr(i, *cpu_map)
7759                cpu_attach_domain(NULL, &def_root_domain, i);
7760        synchronize_sched();
7761        arch_destroy_sched_domains(cpu_map, &tmpmask);
7762}
7763
7764/* handle null as "default" */
7765static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7766                        struct sched_domain_attr *new, int idx_new)
7767{
7768        struct sched_domain_attr tmp;
7769
7770        /* fast path */
7771        if (!new && !cur)
7772                return 1;
7773
7774        tmp = SD_ATTR_INIT;
7775        return !memcmp(cur ? (cur + idx_cur) : &tmp,
7776                        new ? (new + idx_new) : &tmp,
7777                        sizeof(struct sched_domain_attr));
7778}
7779
7780/*
7781 * Partition sched domains as specified by the 'ndoms_new'
7782 * cpumasks in the array doms_new[] of cpumasks. This compares
7783 * doms_new[] to the current sched domain partitioning, doms_cur[].
7784 * It destroys each deleted domain and builds each new domain.
7785 *
7786 * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'.
7787 * The masks don't intersect (don't overlap.) We should setup one
7788 * sched domain for each mask. CPUs not in any of the cpumasks will
7789 * not be load balanced. If the same cpumask appears both in the
7790 * current 'doms_cur' domains and in the new 'doms_new', we can leave
7791 * it as it is.
7792 *
7793 * The passed in 'doms_new' should be kmalloc'd. This routine takes
7794 * ownership of it and will kfree it when done with it. If the caller
7795 * failed the kmalloc call, then it can pass in doms_new == NULL &&
7796 * ndoms_new == 1, and partition_sched_domains() will fallback to
7797 * the single partition 'fallback_doms', it also forces the domains
7798 * to be rebuilt.
7799 *
7800 * If doms_new == NULL it will be replaced with cpu_online_map.
7801 * ndoms_new == 0 is a special case for destroying existing domains,
7802 * and it will not create the default domain.
7803 *
7804 * Call with hotplug lock held
7805 */
7806void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
7807                             struct sched_domain_attr *dattr_new)
7808{
7809        int i, j, n;
7810
7811        mutex_lock(&sched_domains_mutex);
7812
7813        /* always unregister in case we don't destroy any domains */
7814        unregister_sched_domain_sysctl();
7815
7816        n = doms_new ? ndoms_new : 0;
7817
7818        /* Destroy deleted domains */
7819        for (i = 0; i < ndoms_cur; i++) {
7820                for (j = 0; j < n; j++) {
7821                        if (cpus_equal(doms_cur[i], doms_new[j])
7822                            && dattrs_equal(dattr_cur, i, dattr_new, j))
7823                                goto match1;
7824                }
7825                /* no match - a current sched domain not in new doms_new[] */
7826                detach_destroy_domains(doms_cur + i);
7827match1:
7828                ;
7829        }
7830
7831        if (doms_new == NULL) {
7832                ndoms_cur = 0;
7833                doms_new = &fallback_doms;
7834                cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
7835                dattr_new = NULL;
7836        }
7837
7838        /* Build new domains */
7839        for (i = 0; i < ndoms_new; i++) {
7840                for (j = 0; j < ndoms_cur; j++) {
7841                        if (cpus_equal(doms_new[i], doms_cur[j])
7842                            && dattrs_equal(dattr_new, i, dattr_cur, j))
7843                                goto match2;
7844                }
7845                /* no match - add a new doms_new */
7846                __build_sched_domains(doms_new + i,
7847                                        dattr_new ? dattr_new + i : NULL);
7848match2:
7849                ;
7850        }
7851
7852        /* Remember the new sched domains */
7853        if (doms_cur != &fallback_doms)
7854                kfree(doms_cur);
7855        kfree(dattr_cur);        /* kfree(NULL) is safe */
7856        doms_cur = doms_new;
7857        dattr_cur = dattr_new;
7858        ndoms_cur = ndoms_new;
7859
7860        register_sched_domain_sysctl();
7861
7862        mutex_unlock(&sched_domains_mutex);
7863}
7864
7865#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
7866int arch_reinit_sched_domains(void)
7867{
7868        get_online_cpus();
7869
7870        /* Destroy domains first to force the rebuild */
7871        partition_sched_domains(0, NULL, NULL);
7872
7873        rebuild_sched_domains();
7874        put_online_cpus();
7875
7876        return 0;
7877}
7878
7879static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
7880{
7881        int ret;
7882
7883        if (buf[0] != '0' && buf[0] != '1')
7884                return -EINVAL;
7885
7886        if (smt)
7887                sched_smt_power_savings = (buf[0] == '1');
7888        else
7889                sched_mc_power_savings = (buf[0] == '1');
7890
7891        ret = arch_reinit_sched_domains();
7892
7893        return ret ? ret : count;
7894}
7895
7896#ifdef CONFIG_SCHED_MC
7897static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
7898                                           char *page)
7899{
7900        return sprintf(page, "%u\n", sched_mc_power_savings);
7901}
7902static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
7903                                            const char *buf, size_t count)
7904{
7905        return sched_power_savings_store(buf, count, 0);
7906}
7907static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
7908                         sched_mc_power_savings_show,
7909                         sched_mc_power_savings_store);
7910#endif
7911
7912#ifdef CONFIG_SCHED_SMT
7913static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
7914                                            char *page)
7915{
7916        return sprintf(page, "%u\n", sched_smt_power_savings);
7917}
7918static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
7919                                             const char *buf, size_t count)
7920{
7921        return sched_power_savings_store(buf, count, 1);
7922}
7923static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
7924                   sched_smt_power_savings_show,
7925                   sched_smt_power_savings_store);
7926#endif
7927
7928int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
7929{
7930        int err = 0;
7931
7932#ifdef CONFIG_SCHED_SMT
7933        if (smt_capable())
7934                err = sysfs_create_file(&cls->kset.kobj,
7935                                        &attr_sched_smt_power_savings.attr);
7936#endif
7937#ifdef CONFIG_SCHED_MC
7938        if (!err && mc_capable())
7939                err = sysfs_create_file(&cls->kset.kobj,
7940                                        &attr_sched_mc_power_savings.attr);
7941#endif
7942        return err;
7943}
7944#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
7945
7946#ifndef CONFIG_CPUSETS
7947/*
7948 * Add online and remove offline CPUs from the scheduler domains.
7949 * When cpusets are enabled they take over this function.
7950 */
7951static int update_sched_domains(struct notifier_block *nfb,
7952                                unsigned long action, void *hcpu)
7953{
7954        switch (action) {
7955        case CPU_ONLINE:
7956        case CPU_ONLINE_FROZEN:
7957        case CPU_DEAD:
7958        case CPU_DEAD_FROZEN:
7959                partition_sched_domains(1, NULL, NULL);
7960                return NOTIFY_OK;
7961
7962        default:
7963                return NOTIFY_DONE;
7964        }
7965}
7966#endif
7967
7968static int update_runtime(struct notifier_block *nfb,
7969                                unsigned long action, void *hcpu)
7970{
7971        int cpu = (int)(long)hcpu;
7972
7973        switch (action) {
7974        case CPU_DOWN_PREPARE:
7975        case CPU_DOWN_PREPARE_FROZEN:
7976                disable_runtime(cpu_rq(cpu));
7977                return NOTIFY_OK;
7978
7979        case CPU_DOWN_FAILED:
7980        case CPU_DOWN_FAILED_FROZEN:
7981        case CPU_ONLINE:
7982        case CPU_ONLINE_FROZEN:
7983                enable_runtime(cpu_rq(cpu));
7984                return NOTIFY_OK;
7985
7986        default:
7987                return NOTIFY_DONE;
7988        }
7989}
7990
7991void __init sched_init_smp(void)
7992{
7993        cpumask_t non_isolated_cpus;
7994
7995#if defined(CONFIG_NUMA)
7996        sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
7997                                                                GFP_KERNEL);
7998        BUG_ON(sched_group_nodes_bycpu == NULL);
7999#endif
8000        get_online_cpus();
8001        mutex_lock(&sched_domains_mutex);
8002        arch_init_sched_domains(&cpu_online_map);
8003        cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map);
8004        if (cpus_empty(non_isolated_cpus))
8005                cpu_set(smp_processor_id(), non_isolated_cpus);
8006        mutex_unlock(&sched_domains_mutex);
8007        put_online_cpus();
8008
8009#ifndef CONFIG_CPUSETS
8010        /* XXX: Theoretical race here - CPU may be hotplugged now */
8011        hotcpu_notifier(update_sched_domains, 0);
8012#endif
8013
8014        /* RT runtime code needs to handle some hotplug events */
8015        hotcpu_notifier(update_runtime, 0);
8016
8017        init_hrtick();
8018
8019        /* Move init over to a non-isolated CPU */
8020        if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0)
8021                BUG();
8022        sched_init_granularity();
8023}
8024#else
8025void __init sched_init_smp(void)
8026{
8027        sched_init_granularity();
8028}
8029#endif /* CONFIG_SMP */
8030
8031int in_sched_functions(unsigned long addr)
8032{
8033        return in_lock_functions(addr) ||
8034                (addr >= (unsigned long)__sched_text_start
8035                && addr < (unsigned long)__sched_text_end);
8036}
8037
8038static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
8039{
8040        cfs_rq->tasks_timeline = RB_ROOT;
8041        INIT_LIST_HEAD(&cfs_rq->tasks);
8042#ifdef CONFIG_FAIR_GROUP_SCHED
8043        cfs_rq->rq = rq;
8044#endif
8045        cfs_rq->min_vruntime = (u64)(-(1LL << 20));
8046}
8047
8048static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
8049{
8050        struct rt_prio_array *array;
8051        int i;
8052
8053        array = &rt_rq->active;
8054        for (i = 0; i < MAX_RT_PRIO; i++) {
8055                INIT_LIST_HEAD(array->queue + i);
8056                __clear_bit(i, array->bitmap);
8057        }
8058        /* delimiter for bitsearch: */
8059        __set_bit(MAX_RT_PRIO, array->bitmap);
8060
8061#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
8062        rt_rq->highest_prio = MAX_RT_PRIO;
8063#endif
8064#ifdef CONFIG_SMP
8065        rt_rq->rt_nr_migratory = 0;
8066        rt_rq->overloaded = 0;
8067#endif
8068
8069        rt_rq->rt_time = 0;
8070        rt_rq->rt_throttled = 0;
8071        rt_rq->rt_runtime = 0;