diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index af7e74d7491..6799d57a1ab 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -30,6 +30,7 @@ config ARM select HAVE_GENERIC_HARDIRQS select HAVE_SPARSE_IRQ select GENERIC_IRQ_SHOW + select CPU_PM if (SUSPEND || CPU_IDLE) help The ARM series is a line of low-power-consumption RISC chip designs licensed by ARM Ltd and targeted at embedded applications and diff --git a/arch/arm/common/gic.c b/arch/arm/common/gic.c index 7d3e6117456..51a88a48e71 100644 --- a/arch/arm/common/gic.c +++ b/arch/arm/common/gic.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -301,6 +302,8 @@ static void __init gic_dist_init(struct gic_chip_data *gic, if (gic_irqs > 1020) gic_irqs = 1020; + gic->gic_irqs = gic_irqs; + /* * Set all global interrupts to be level triggered, active low. */ @@ -368,6 +371,189 @@ static void __cpuinit gic_cpu_init(struct gic_chip_data *gic) writel_relaxed(1, base + GIC_CPU_CTRL); } +#ifdef CONFIG_CPU_PM +/* + * Saves the GIC distributor registers during suspend or idle. Must be called + * with interrupts disabled but before powering down the GIC. After calling + * this function, no interrupts will be delivered by the GIC, and another + * platform-specific wakeup source must be enabled. + */ +static void gic_dist_save(unsigned int gic_nr) +{ + unsigned int gic_irqs; + void __iomem *dist_base; + int i; + + if (gic_nr >= MAX_GIC_NR) + BUG(); + + gic_irqs = gic_data[gic_nr].gic_irqs; + dist_base = gic_data[gic_nr].dist_base; + + if (!dist_base) + return; + + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++) + gic_data[gic_nr].saved_spi_conf[i] = + readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4); + + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) + gic_data[gic_nr].saved_spi_target[i] = + readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4); + + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) + gic_data[gic_nr].saved_spi_enable[i] = + readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4); +} + +/* + * Restores the GIC distributor registers during resume or when coming out of + * idle. Must be called before enabling interrupts. If a level interrupt + * that occured while the GIC was suspended is still present, it will be + * handled normally, but any edge interrupts that occured will not be seen by + * the GIC and need to be handled by the platform-specific wakeup source. + */ +static void gic_dist_restore(unsigned int gic_nr) +{ + unsigned int gic_irqs; + unsigned int i; + void __iomem *dist_base; + + if (gic_nr >= MAX_GIC_NR) + BUG(); + + gic_irqs = gic_data[gic_nr].gic_irqs; + dist_base = gic_data[gic_nr].dist_base; + + if (!dist_base) + return; + + writel_relaxed(0, dist_base + GIC_DIST_CTRL); + + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++) + writel_relaxed(gic_data[gic_nr].saved_spi_conf[i], + dist_base + GIC_DIST_CONFIG + i * 4); + + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) + writel_relaxed(0xa0a0a0a0, + dist_base + GIC_DIST_PRI + i * 4); + + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) + writel_relaxed(gic_data[gic_nr].saved_spi_target[i], + dist_base + GIC_DIST_TARGET + i * 4); + + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) + writel_relaxed(gic_data[gic_nr].saved_spi_enable[i], + dist_base + GIC_DIST_ENABLE_SET + i * 4); + + writel_relaxed(1, dist_base + GIC_DIST_CTRL); +} + +static void gic_cpu_save(unsigned int gic_nr) +{ + int i; + u32 *ptr; + void __iomem *dist_base; + void __iomem *cpu_base; + + if (gic_nr >= MAX_GIC_NR) + BUG(); + + dist_base = gic_data[gic_nr].dist_base; + cpu_base = gic_data[gic_nr].cpu_base; + + if (!dist_base || !cpu_base) + return; + + ptr = __this_cpu_ptr(gic_data[gic_nr].saved_ppi_enable); + for (i = 0; i < DIV_ROUND_UP(32, 32); i++) + ptr[i] = readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4); + + ptr = __this_cpu_ptr(gic_data[gic_nr].saved_ppi_conf); + for (i = 0; i < DIV_ROUND_UP(32, 16); i++) + ptr[i] = readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4); + +} + +static void gic_cpu_restore(unsigned int gic_nr) +{ + int i; + u32 *ptr; + void __iomem *dist_base; + void __iomem *cpu_base; + + if (gic_nr >= MAX_GIC_NR) + BUG(); + + dist_base = gic_data[gic_nr].dist_base; + cpu_base = gic_data[gic_nr].cpu_base; + + if (!dist_base || !cpu_base) + return; + + ptr = __this_cpu_ptr(gic_data[gic_nr].saved_ppi_enable); + for (i = 0; i < DIV_ROUND_UP(32, 32); i++) + writel_relaxed(ptr[i], dist_base + GIC_DIST_ENABLE_SET + i * 4); + + ptr = __this_cpu_ptr(gic_data[gic_nr].saved_ppi_conf); + for (i = 0; i < DIV_ROUND_UP(32, 16); i++) + writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4); + + for (i = 0; i < DIV_ROUND_UP(32, 4); i++) + writel_relaxed(0xa0a0a0a0, dist_base + GIC_DIST_PRI + i * 4); + + writel_relaxed(0xf0, cpu_base + GIC_CPU_PRIMASK); + writel_relaxed(1, cpu_base + GIC_CPU_CTRL); +} + +static int gic_notifier(struct notifier_block *self, unsigned long cmd, void *v) +{ + int i; + + for (i = 0; i < MAX_GIC_NR; i++) { + switch (cmd) { + case CPU_PM_ENTER: + gic_cpu_save(i); + break; + case CPU_PM_ENTER_FAILED: + case CPU_PM_EXIT: + gic_cpu_restore(i); + break; + case CPU_CLUSTER_PM_ENTER: + gic_dist_save(i); + break; + case CPU_CLUSTER_PM_ENTER_FAILED: + case CPU_CLUSTER_PM_EXIT: + gic_dist_restore(i); + break; + } + } + + return NOTIFY_OK; +} + +static struct notifier_block gic_notifier_block = { + .notifier_call = gic_notifier, +}; + +static void __init gic_pm_init(struct gic_chip_data *gic) +{ + gic->saved_ppi_enable = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4, + sizeof(u32)); + BUG_ON(!gic->saved_ppi_enable); + + gic->saved_ppi_conf = __alloc_percpu(DIV_ROUND_UP(32, 16) * 4, + sizeof(u32)); + BUG_ON(!gic->saved_ppi_conf); + + cpu_pm_register_notifier(&gic_notifier_block); +} +#else +static void __init gic_pm_init(struct gic_chip_data *gic) +{ +} +#endif + void __init gic_init(unsigned int gic_nr, unsigned int irq_start, void __iomem *dist_base, void __iomem *cpu_base) { @@ -397,6 +583,7 @@ void __init gic_init(unsigned int gic_nr, unsigned int irq_start, gic_dist_init(gic, irq_start); gic_cpu_init(gic); + gic_pm_init(gic); } void __cpuinit gic_secondary_init_base(unsigned int gic_nr, diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 7b5cc8dae06..c9becf9aab6 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -118,6 +118,13 @@ extern void iwmmxt_task_switch(struct thread_info *); extern void vfp_sync_hwstate(struct thread_info *); extern void vfp_flush_hwstate(struct thread_info *); +struct user_vfp; +struct user_vfp_exc; + +extern int vfp_preserve_user_clear_hwstate(struct user_vfp __user *, + struct user_vfp_exc __user *); +extern int vfp_restore_user_hwstate(struct user_vfp __user *, + struct user_vfp_exc __user *); #endif /* diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 927522cfc12..16baba2e436 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -59,6 +59,9 @@ int main(void) DEFINE(TI_TP_VALUE, offsetof(struct thread_info, tp_value)); DEFINE(TI_FPSTATE, offsetof(struct thread_info, fpstate)); DEFINE(TI_VFPSTATE, offsetof(struct thread_info, vfpstate)); +#ifdef CONFIG_SMP + DEFINE(VFP_CPU, offsetof(union vfp_state, hard.cpu)); +#endif #ifdef CONFIG_ARM_THUMBEE DEFINE(TI_THUMBEE_STATE, offsetof(struct thread_info, thumbee_state)); #endif diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index 9e617bd4a14..b02ce1da310 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -179,44 +179,23 @@ static int restore_iwmmxt_context(struct iwmmxt_sigframe *frame) static int preserve_vfp_context(struct vfp_sigframe __user *frame) { - struct thread_info *thread = current_thread_info(); - struct vfp_hard_struct *h = &thread->vfpstate.hard; const unsigned long magic = VFP_MAGIC; const unsigned long size = VFP_STORAGE_SIZE; int err = 0; - vfp_sync_hwstate(thread); __put_user_error(magic, &frame->magic, err); __put_user_error(size, &frame->size, err); - /* - * Copy the floating point registers. There can be unused - * registers see asm/hwcap.h for details. - */ - err |= __copy_to_user(&frame->ufp.fpregs, &h->fpregs, - sizeof(h->fpregs)); - /* - * Copy the status and control register. - */ - __put_user_error(h->fpscr, &frame->ufp.fpscr, err); - - /* - * Copy the exception registers. - */ - __put_user_error(h->fpexc, &frame->ufp_exc.fpexc, err); - __put_user_error(h->fpinst, &frame->ufp_exc.fpinst, err); - __put_user_error(h->fpinst2, &frame->ufp_exc.fpinst2, err); + if (err) + return -EFAULT; - return err ? -EFAULT : 0; + return vfp_preserve_user_clear_hwstate(&frame->ufp, &frame->ufp_exc); } static int restore_vfp_context(struct vfp_sigframe __user *frame) { - struct thread_info *thread = current_thread_info(); - struct vfp_hard_struct *h = &thread->vfpstate.hard; unsigned long magic; unsigned long size; - unsigned long fpexc; int err = 0; __get_user_error(magic, &frame->magic, err); @@ -227,33 +206,7 @@ static int restore_vfp_context(struct vfp_sigframe __user *frame) if (magic != VFP_MAGIC || size != VFP_STORAGE_SIZE) return -EINVAL; - vfp_flush_hwstate(thread); - - /* - * Copy the floating point registers. There can be unused - * registers see asm/hwcap.h for details. - */ - err |= __copy_from_user(&h->fpregs, &frame->ufp.fpregs, - sizeof(h->fpregs)); - /* - * Copy the status and control register. - */ - __get_user_error(h->fpscr, &frame->ufp.fpscr, err); - - /* - * Sanitise and restore the exception registers. - */ - __get_user_error(fpexc, &frame->ufp_exc.fpexc, err); - /* Ensure the VFP is enabled. */ - fpexc |= FPEXC_EN; - /* Ensure FPINST2 is invalid and the exception flag is cleared. */ - fpexc &= ~(FPEXC_EX | FPEXC_FP2V); - h->fpexc = fpexc; - - __get_user_error(h->fpinst, &frame->ufp_exc.fpinst, err); - __get_user_error(h->fpinst2, &frame->ufp_exc.fpinst2, err); - - return err ? -EFAULT : 0; + return vfp_restore_user_hwstate(&frame->ufp, &frame->ufp_exc); } #endif diff --git a/arch/arm/mach-exynos/cpuidle-exynos4.c b/arch/arm/mach-exynos/cpuidle-exynos4.c index 2fd79538a78..88b941f6ca7 100644 --- a/arch/arm/mach-exynos/cpuidle-exynos4.c +++ b/arch/arm/mach-exynos/cpuidle-exynos4.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -568,6 +569,8 @@ static int exynos4_enter_core0_aftr(struct cpuidle_device *dev, local_irq_disable(); + cpu_pm_enter(); + if (log_en) pr_info("+++aftr\n"); @@ -630,6 +633,8 @@ static int exynos4_enter_core0_aftr(struct cpuidle_device *dev, if (log_en) pr_info("---aftr\n"); + cpu_pm_exit(); + local_irq_enable(); idle_time = (after.tv_sec - before.tv_sec) * USEC_PER_SEC + (after.tv_usec - before.tv_usec); @@ -663,6 +668,8 @@ static int exynos4_enter_core0_lpa(struct cpuidle_device *dev, #endif local_irq_disable(); + cpu_pm_enter(); + #if defined(CONFIG_INTERNAL_MODEM_IF) || defined(CONFIG_SAMSUNG_PHONE_TTY) gpio_set_value(GPIO_PDA_ACTIVE, 0); #endif @@ -763,6 +770,8 @@ static int exynos4_enter_core0_lpa(struct cpuidle_device *dev, gpio_set_value(GPIO_PDA_ACTIVE, 1); #endif + cpu_pm_exit(); + local_irq_enable(); idle_time = (after.tv_sec - before.tv_sec) * USEC_PER_SEC + (after.tv_usec - before.tv_usec); diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S index 404538ae591..2d30c7f6edd 100644 --- a/arch/arm/vfp/vfphw.S +++ b/arch/arm/vfp/vfphw.S @@ -82,19 +82,22 @@ ENTRY(vfp_support_entry) ldr r4, [r3, r11, lsl #2] @ vfp_current_hw_state pointer bic r5, r1, #FPEXC_EX @ make sure exceptions are disabled cmp r4, r10 @ this thread owns the hw context? +#ifndef CONFIG_SMP + @ For UP, checking that this thread owns the hw context is + @ sufficient to determine that the hardware state is valid. beq vfp_hw_state_valid + @ On UP, we lazily save the VFP context. As a different + @ thread wants ownership of the VFP hardware, save the old + @ state if there was a previous (valid) owner. + VFPFMXR FPEXC, r5 @ enable VFP, disable any pending @ exceptions, so we can get at the @ rest of it -#ifndef CONFIG_SMP - @ Save out the current registers to the old thread state - @ No need for SMP since this is not done lazily - DBGSTR1 "save old state %p", r4 - cmp r4, #0 - beq no_old_VFP_process + cmp r4, #0 @ if the vfp_current_hw_state is NULL + beq vfp_reload_hw @ then the hw state needs reloading VFPFSTMIA r4, r5 @ save the working registers VFPFMRX r5, FPSCR @ current status #ifndef CONFIG_CPU_FEROCEON @@ -107,11 +110,33 @@ ENTRY(vfp_support_entry) 1: #endif stmia r4, {r1, r5, r6, r8} @ save FPEXC, FPSCR, FPINST, FPINST2 - @ and point r4 at the word at the - @ start of the register dump +vfp_reload_hw: + +#else + @ For SMP, if this thread does not own the hw context, then we + @ need to reload it. No need to save the old state as on SMP, + @ we always save the state when we switch away from a thread. + bne vfp_reload_hw + + @ This thread has ownership of the current hardware context. + @ However, it may have been migrated to another CPU, in which + @ case the saved state is newer than the hardware context. + @ Check this by looking at the CPU number which the state was + @ last loaded onto. + ldr ip, [r10, #VFP_CPU] + teq ip, r11 + beq vfp_hw_state_valid + +vfp_reload_hw: + @ We're loading this threads state into the VFP hardware. Update + @ the CPU number which contains the most up to date VFP context. + str r11, [r10, #VFP_CPU] + + VFPFMXR FPEXC, r5 @ enable VFP, disable any pending + @ exceptions, so we can get at the + @ rest of it #endif -no_old_VFP_process: DBGSTR1 "load state %p", r10 str r10, [r3, r11, lsl #2] @ update the vfp_current_hw_state pointer @ Load the saved state back into the VFP diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 0e46a47ac3c..2541b9b5ab9 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,8 @@ #include #include #include +#include +#include #include #include @@ -35,19 +38,52 @@ void vfp_null_entry(void); void (*vfp_vector)(void) = vfp_null_entry; +/* + * Dual-use variable. + * Used in startup: set to non-zero if VFP checks fail + * After startup, holds VFP architecture + */ +unsigned int VFP_arch; + /* * The pointer to the vfpstate structure of the thread which currently * owns the context held in the VFP hardware, or NULL if the hardware * context is invalid. + * + * For UP, this is sufficient to tell which thread owns the VFP context. + * However, for SMP, we also need to check the CPU number stored in the + * saved state too to catch migrations. */ union vfp_state *vfp_current_hw_state[NR_CPUS]; /* - * Dual-use variable. - * Used in startup: set to non-zero if VFP checks fail - * After startup, holds VFP architecture + * Is 'thread's most up to date state stored in this CPUs hardware? + * Must be called from non-preemptible context. */ -unsigned int VFP_arch; +static bool vfp_state_in_hw(unsigned int cpu, struct thread_info *thread) +{ +#ifdef CONFIG_SMP + if (thread->vfpstate.hard.cpu != cpu) + return false; +#endif + return vfp_current_hw_state[cpu] == &thread->vfpstate; +} + +/* + * Force a reload of the VFP context from the thread structure. We do + * this by ensuring that access to the VFP hardware is disabled, and + * clear last_VFP_context. Must be called from non-preemptible context. + */ +static void vfp_force_reload(unsigned int cpu, struct thread_info *thread) +{ + if (vfp_state_in_hw(cpu, thread)) { + fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); + vfp_current_hw_state[cpu] = NULL; + } +#ifdef CONFIG_SMP + thread->vfpstate.hard.cpu = NR_CPUS; +#endif +} /* * Per-thread VFP initialization. @@ -57,21 +93,27 @@ static void vfp_thread_flush(struct thread_info *thread) union vfp_state *vfp = &thread->vfpstate; unsigned int cpu; - memset(vfp, 0, sizeof(union vfp_state)); - - vfp->hard.fpexc = FPEXC_EN; - vfp->hard.fpscr = FPSCR_ROUND_NEAREST; - /* * Disable VFP to ensure we initialize it first. We must ensure - * that the modification of vfp_current_hw_state[] and hardware disable - * are done for the same CPU and without preemption. + * that the modification of vfp_current_hw_state[] and hardware + * disable are done for the same CPU and without preemption. + * + * Do this first to ensure that preemption won't overwrite our + * state saving should access to the VFP be enabled at this point. */ cpu = get_cpu(); if (vfp_current_hw_state[cpu] == vfp) vfp_current_hw_state[cpu] = NULL; fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); put_cpu(); + + memset(vfp, 0, sizeof(union vfp_state)); + + vfp->hard.fpexc = FPEXC_EN; + vfp->hard.fpscr = FPSCR_ROUND_NEAREST; +#ifdef CONFIG_SMP + vfp->hard.cpu = NR_CPUS; +#endif } static void vfp_thread_exit(struct thread_info *thread) @@ -91,6 +133,9 @@ static void vfp_thread_copy(struct thread_info *thread) vfp_sync_hwstate(parent); thread->vfpstate = parent->vfpstate; +#ifdef CONFIG_SMP + thread->vfpstate.hard.cpu = NR_CPUS; +#endif } /* @@ -136,17 +181,8 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) * case the thread migrates to a different CPU. The * restoring is done lazily. */ - if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) { + if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu]) vfp_save_state(vfp_current_hw_state[cpu], fpexc); - vfp_current_hw_state[cpu]->hard.cpu = cpu; - } - /* - * Thread migration, just force the reloading of the - * state on the new CPU in case the VFP registers - * contain stale data. - */ - if (thread->vfpstate.hard.cpu != cpu) - vfp_current_hw_state[cpu] = NULL; #endif /* @@ -406,9 +442,7 @@ static void vfp_enable(void *unused) set_copro_access(access | CPACC_FULL(10) | CPACC_FULL(11)); } -#ifdef CONFIG_PM -#include - +#ifdef CONFIG_CPU_PM static int vfp_pm_suspend(void) { struct thread_info *ti = current_thread_info(); @@ -444,29 +478,43 @@ static void vfp_pm_resume(void) fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); } -static struct syscore_ops vfp_pm_syscore_ops = { - .suspend = vfp_pm_suspend, - .resume = vfp_pm_resume, +static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, + void *v) +{ + switch (cmd) { + case CPU_PM_ENTER: + vfp_pm_suspend(); + break; + case CPU_PM_ENTER_FAILED: + case CPU_PM_EXIT: + vfp_pm_resume(); + break; + } + return NOTIFY_OK; +} + +static struct notifier_block vfp_cpu_pm_notifier_block = { + .notifier_call = vfp_cpu_pm_notifier, }; static void vfp_pm_init(void) { - register_syscore_ops(&vfp_pm_syscore_ops); + cpu_pm_register_notifier(&vfp_cpu_pm_notifier_block); } #else static inline void vfp_pm_init(void) { } -#endif /* CONFIG_PM */ +#endif /* CONFIG_CPU_PM */ +/* + * Ensure that the VFP state stored in 'thread->vfpstate' is up to date + * with the hardware state. + */ void vfp_sync_hwstate(struct thread_info *thread) { unsigned int cpu = get_cpu(); - /* - * If the thread we're interested in is the current owner of the - * hardware VFP state, then we need to save its state. - */ - if (vfp_current_hw_state[cpu] == &thread->vfpstate) { + if (vfp_state_in_hw(cpu, thread)) { u32 fpexc = fmrx(FPEXC); /* @@ -480,37 +528,101 @@ void vfp_sync_hwstate(struct thread_info *thread) put_cpu(); } +/* Ensure that the thread reloads the hardware VFP state on the next use. */ void vfp_flush_hwstate(struct thread_info *thread) { unsigned int cpu = get_cpu(); + vfp_force_reload(cpu, thread); + + put_cpu(); +} + +/* + * Save the current VFP state into the provided structures and prepare + * for entry into a new function (signal handler). + */ +int vfp_preserve_user_clear_hwstate(struct user_vfp __user *ufp, + struct user_vfp_exc __user *ufp_exc) +{ + struct thread_info *thread = current_thread_info(); + struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; + int err = 0; + + /* Ensure that the saved hwstate is up-to-date. */ + vfp_sync_hwstate(thread); + /* - * If the thread we're interested in is the current owner of the - * hardware VFP state, then we need to save its state. + * Copy the floating point registers. There can be unused + * registers see asm/hwcap.h for details. */ - if (vfp_current_hw_state[cpu] == &thread->vfpstate) { - u32 fpexc = fmrx(FPEXC); + err |= __copy_to_user(&ufp->fpregs, &hwstate->fpregs, + sizeof(hwstate->fpregs)); + /* + * Copy the status and control register. + */ + __put_user_error(hwstate->fpscr, &ufp->fpscr, err); - fmxr(FPEXC, fpexc & ~FPEXC_EN); + /* + * Copy the exception registers. + */ + __put_user_error(hwstate->fpexc, &ufp_exc->fpexc, err); + __put_user_error(hwstate->fpinst, &ufp_exc->fpinst, err); + __put_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err); - /* - * Set the context to NULL to force a reload the next time - * the thread uses the VFP. - */ - vfp_current_hw_state[cpu] = NULL; - } + if (err) + return -EFAULT; + + /* Ensure that VFP is disabled. */ + vfp_flush_hwstate(thread); -#ifdef CONFIG_SMP /* - * For SMP we still have to take care of the case where the thread - * migrates to another CPU and then back to the original CPU on which - * the last VFP user is still the same thread. Mark the thread VFP - * state as belonging to a non-existent CPU so that the saved one will - * be reloaded in the above case. + * As per the PCS, clear the length and stride bits for function + * entry. */ - thread->vfpstate.hard.cpu = NR_CPUS; -#endif - put_cpu(); + hwstate->fpscr &= ~(FPSCR_LENGTH_MASK | FPSCR_STRIDE_MASK); + return 0; +} + +/* Sanitise and restore the current VFP state from the provided structures. */ +int vfp_restore_user_hwstate(struct user_vfp __user *ufp, + struct user_vfp_exc __user *ufp_exc) +{ + struct thread_info *thread = current_thread_info(); + struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; + unsigned long fpexc; + int err = 0; + + /* Disable VFP to avoid corrupting the new thread state. */ + vfp_flush_hwstate(thread); + + /* + * Copy the floating point registers. There can be unused + * registers see asm/hwcap.h for details. + */ + err |= __copy_from_user(&hwstate->fpregs, &ufp->fpregs, + sizeof(hwstate->fpregs)); + /* + * Copy the status and control register. + */ + __get_user_error(hwstate->fpscr, &ufp->fpscr, err); + + /* + * Sanitise and restore the exception registers. + */ + __get_user_error(fpexc, &ufp_exc->fpexc, err); + + /* Ensure the VFP is enabled. */ + fpexc |= FPEXC_EN; + + /* Ensure FPINST2 is invalid and the exception flag is cleared. */ + fpexc &= ~(FPEXC_EX | FPEXC_FP2V); + hwstate->fpexc = fpexc; + + __get_user_error(hwstate->fpinst, &ufp_exc->fpinst, err); + __get_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err); + + return err ? -EFAULT : 0; } /* @@ -527,10 +639,9 @@ void vfp_flush_hwstate(struct thread_info *thread) static int vfp_hotplug(struct notifier_block *b, unsigned long action, void *hcpu) { - if (action == CPU_DYING || action == CPU_DYING_FROZEN) { - unsigned int cpu = (long)hcpu; - vfp_current_hw_state[cpu] = NULL; - } else if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) + if (action == CPU_DYING || action == CPU_DYING_FROZEN) + vfp_current_hw_state[(long)hcpu] = NULL; + else if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) vfp_enable(NULL); return NOTIFY_OK; } diff --git a/include/linux/cpu_pm.h b/include/linux/cpu_pm.h new file mode 100644 index 00000000000..455b233dd3b --- /dev/null +++ b/include/linux/cpu_pm.h @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2011 Google, Inc. + * + * Author: + * Colin Cross + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_CPU_PM_H +#define _LINUX_CPU_PM_H + +#include +#include + +/* + * When a CPU goes to a low power state that turns off power to the CPU's + * power domain, the contents of some blocks (floating point coprocessors, + * interrupt controllers, caches, timers) in the same power domain can + * be lost. The cpm_pm notifiers provide a method for platform idle, suspend, + * and hotplug implementations to notify the drivers for these blocks that + * they may be reset. + * + * All cpu_pm notifications must be called with interrupts disabled. + * + * The notifications are split into two classes: CPU notifications and CPU + * cluster notifications. + * + * CPU notifications apply to a single CPU and must be called on the affected + * CPU. They are used to save per-cpu context for affected blocks. + * + * CPU cluster notifications apply to all CPUs in a single power domain. They + * are used to save any global context for affected blocks, and must be called + * after all the CPUs in the power domain have been notified of the low power + * state. + */ + +/* + * Event codes passed as unsigned long val to notifier calls + */ +enum cpu_pm_event { + /* A single cpu is entering a low power state */ + CPU_PM_ENTER, + + /* A single cpu failed to enter a low power state */ + CPU_PM_ENTER_FAILED, + + /* A single cpu is exiting a low power state */ + CPU_PM_EXIT, + + /* A cpu power domain is entering a low power state */ + CPU_CLUSTER_PM_ENTER, + + /* A cpu power domain failed to enter a low power state */ + CPU_CLUSTER_PM_ENTER_FAILED, + + /* A cpu power domain is exiting a low power state */ + CPU_CLUSTER_PM_EXIT, +}; + +#ifdef CONFIG_CPU_PM +int cpu_pm_register_notifier(struct notifier_block *nb); +int cpu_pm_unregister_notifier(struct notifier_block *nb); +int cpu_pm_enter(void); +int cpu_pm_exit(void); +int cpu_cluster_pm_enter(void); +int cpu_cluster_pm_exit(void); + +#else + +static inline int cpu_pm_register_notifier(struct notifier_block *nb) +{ + return 0; +} + +static inline int cpu_pm_unregister_notifier(struct notifier_block *nb) +{ + return 0; +} + +static inline int cpu_pm_enter(void) +{ + return 0; +} + +static inline int cpu_pm_exit(void) +{ + return 0; +} + +static inline int cpu_cluster_pm_enter(void) +{ + return 0; +} + +static inline int cpu_cluster_pm_exit(void) +{ + return 0; +} +#endif +#endif diff --git a/kernel/Makefile b/kernel/Makefile index 9f665662afe..a1096deef63 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -101,6 +101,7 @@ obj-$(CONFIG_RING_BUFFER) += trace/ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_SMP) += sched_cpupri.o obj-$(CONFIG_IRQ_WORK) += irq_work.o +obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_PERF_EVENTS) += events/ diff --git a/kernel/cpu_pm.c b/kernel/cpu_pm.c new file mode 100644 index 00000000000..9656a3c3650 --- /dev/null +++ b/kernel/cpu_pm.c @@ -0,0 +1,233 @@ +/* + * Copyright (C) 2011 Google, Inc. + * + * Author: + * Colin Cross + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include + +static DEFINE_RWLOCK(cpu_pm_notifier_lock); +static RAW_NOTIFIER_HEAD(cpu_pm_notifier_chain); + +static int cpu_pm_notify(enum cpu_pm_event event, int nr_to_call, int *nr_calls) +{ + int ret; + + ret = __raw_notifier_call_chain(&cpu_pm_notifier_chain, event, NULL, + nr_to_call, nr_calls); + + return notifier_to_errno(ret); +} + +/** + * cpu_pm_register_notifier - register a driver with cpu_pm + * @nb: notifier block to register + * + * Add a driver to a list of drivers that are notified about + * CPU and CPU cluster low power entry and exit. + * + * This function may sleep, and has the same return conditions as + * raw_notifier_chain_register. + */ +int cpu_pm_register_notifier(struct notifier_block *nb) +{ + unsigned long flags; + int ret; + + write_lock_irqsave(&cpu_pm_notifier_lock, flags); + ret = raw_notifier_chain_register(&cpu_pm_notifier_chain, nb); + write_unlock_irqrestore(&cpu_pm_notifier_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(cpu_pm_register_notifier); + +/** + * cpu_pm_unregister_notifier - unregister a driver with cpu_pm + * @nb: notifier block to be unregistered + * + * Remove a driver from the CPU PM notifier list. + * + * This function may sleep, and has the same return conditions as + * raw_notifier_chain_unregister. + */ +int cpu_pm_unregister_notifier(struct notifier_block *nb) +{ + unsigned long flags; + int ret; + + write_lock_irqsave(&cpu_pm_notifier_lock, flags); + ret = raw_notifier_chain_unregister(&cpu_pm_notifier_chain, nb); + write_unlock_irqrestore(&cpu_pm_notifier_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(cpu_pm_unregister_notifier); + +/** + * cpu_pm_enter - CPU low power entry notifier + * + * Notifies listeners that a single CPU is entering a low power state that may + * cause some blocks in the same power domain as the cpu to reset. + * + * Must be called on the affected CPU with interrupts disabled. Platform is + * responsible for ensuring that cpu_pm_enter is not called twice on the same + * CPU before cpu_pm_exit is called. Notified drivers can include VFP + * co-processor, interrupt controller and its PM extensions, local CPU + * timers context save/restore which shouldn't be interrupted. Hence it + * must be called with interrupts disabled. + * + * Return conditions are same as __raw_notifier_call_chain. + */ +int cpu_pm_enter(void) +{ + int nr_calls; + int ret = 0; + + read_lock(&cpu_pm_notifier_lock); + ret = cpu_pm_notify(CPU_PM_ENTER, -1, &nr_calls); + if (ret) + /* + * Inform listeners (nr_calls - 1) about failure of CPU PM + * PM entry who are notified earlier to prepare for it. + */ + cpu_pm_notify(CPU_PM_ENTER_FAILED, nr_calls - 1, NULL); + read_unlock(&cpu_pm_notifier_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(cpu_pm_enter); + +/** + * cpu_pm_exit - CPU low power exit notifier + * + * Notifies listeners that a single CPU is exiting a low power state that may + * have caused some blocks in the same power domain as the cpu to reset. + * + * Notified drivers can include VFP co-processor, interrupt controller + * and its PM extensions, local CPU timers context save/restore which + * shouldn't be interrupted. Hence it must be called with interrupts disabled. + * + * Return conditions are same as __raw_notifier_call_chain. + */ +int cpu_pm_exit(void) +{ + int ret; + + read_lock(&cpu_pm_notifier_lock); + ret = cpu_pm_notify(CPU_PM_EXIT, -1, NULL); + read_unlock(&cpu_pm_notifier_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(cpu_pm_exit); + +/** + * cpu_cluster_pm_enter - CPU cluster low power entry notifier + * + * Notifies listeners that all cpus in a power domain are entering a low power + * state that may cause some blocks in the same power domain to reset. + * + * Must be called after cpu_pm_enter has been called on all cpus in the power + * domain, and before cpu_pm_exit has been called on any cpu in the power + * domain. Notified drivers can include VFP co-processor, interrupt controller + * and its PM extensions, local CPU timers context save/restore which + * shouldn't be interrupted. Hence it must be called with interrupts disabled. + * + * Must be called with interrupts disabled. + * + * Return conditions are same as __raw_notifier_call_chain. + */ +int cpu_cluster_pm_enter(void) +{ + int nr_calls; + int ret = 0; + + read_lock(&cpu_pm_notifier_lock); + ret = cpu_pm_notify(CPU_CLUSTER_PM_ENTER, -1, &nr_calls); + if (ret) + /* + * Inform listeners (nr_calls - 1) about failure of CPU cluster + * PM entry who are notified earlier to prepare for it. + */ + cpu_pm_notify(CPU_CLUSTER_PM_ENTER_FAILED, nr_calls - 1, NULL); + read_unlock(&cpu_pm_notifier_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(cpu_cluster_pm_enter); + +/** + * cpu_cluster_pm_exit - CPU cluster low power exit notifier + * + * Notifies listeners that all cpus in a power domain are exiting form a + * low power state that may have caused some blocks in the same power domain + * to reset. + * + * Must be called after cpu_pm_exit has been called on all cpus in the power + * domain, and before cpu_pm_exit has been called on any cpu in the power + * domain. Notified drivers can include VFP co-processor, interrupt controller + * and its PM extensions, local CPU timers context save/restore which + * shouldn't be interrupted. Hence it must be called with interrupts disabled. + * + * Return conditions are same as __raw_notifier_call_chain. + */ +int cpu_cluster_pm_exit(void) +{ + int ret; + + read_lock(&cpu_pm_notifier_lock); + ret = cpu_pm_notify(CPU_CLUSTER_PM_EXIT, -1, NULL); + read_unlock(&cpu_pm_notifier_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(cpu_cluster_pm_exit); + +#ifdef CONFIG_PM +static int cpu_pm_suspend(void) +{ + int ret; + + ret = cpu_pm_enter(); + if (ret) + return ret; + + ret = cpu_cluster_pm_enter(); + return ret; +} + +static void cpu_pm_resume(void) +{ + cpu_cluster_pm_exit(); + cpu_pm_exit(); +} + +static struct syscore_ops cpu_pm_syscore_ops = { + .suspend = cpu_pm_suspend, + .resume = cpu_pm_resume, +}; + +static int cpu_pm_init(void) +{ + register_syscore_ops(&cpu_pm_syscore_ops); + return 0; +} +core_initcall(cpu_pm_init); +#endif diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 9fda331d400..ee9375768c6 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -338,3 +338,7 @@ config SUSPEND_TIME Prints the time spent in suspend in the kernel log, and keeps statistics on the time spent in suspend in /sys/kernel/debug/suspend_time + +config CPU_PM + bool + depends on SUSPEND || CPU_IDLE