/* * linux/arch/m32r/kernel/smp.c * * M32R SMP support routines. * * Copyright (c) 2001, 2002 Hitoshi Yamamoto * * Taken from i386 version. * (c) 1995 Alan Cox, Building #3 * (c) 1998-99, 2000 Ingo Molnar * * This code is released under the GNU General Public License version 2 or * later. */ #undef DEBUG_SMP #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* Data structures and variables */ /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* * For flush_cache_all() */ static DEFINE_SPINLOCK(flushcache_lock); static volatile unsigned long flushcache_cpumask = 0; /* * For flush_tlb_others() */ static volatile cpumask_t flush_cpumask; static struct mm_struct *flush_mm; static struct vm_area_struct *flush_vma; static volatile unsigned long flush_va; static DEFINE_SPINLOCK(tlbstate_lock); #define FLUSH_ALL 0xffffffff DECLARE_PER_CPU(int, prof_multiplier); DECLARE_PER_CPU(int, prof_old_multiplier); DECLARE_PER_CPU(int, prof_counter); extern spinlock_t ipi_lock[]; /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* Function Prototypes */ /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ void smp_reschedule_interrupt(void); void smp_flush_cache_all_interrupt(void); static void flush_tlb_all_ipi(void *); static void flush_tlb_others(cpumask_t, struct mm_struct *, struct vm_area_struct *, unsigned long); void smp_invalidate_interrupt(void); static void stop_this_cpu(void *); void smp_ipi_timer_interrupt(struct pt_regs *); void smp_local_timer_interrupt(void); static void send_IPI_allbutself(int, int); static void send_IPI_mask(const struct cpumask *, int, int); /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* Rescheduling request Routines */ /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /*==========================================================================* * Name: smp_send_reschedule * * Description: This routine requests other CPU to execute rescheduling. * 1.Send 'RESCHEDULE_IPI' to other CPU. * Request other CPU to execute 'smp_reschedule_interrupt()'. * * Born on Date: 2002.02.05 * * Arguments: cpu_id - Target CPU ID * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_send_reschedule(int cpu_id) { WARN_ON(cpu_is_offline(cpu_id)); send_IPI_mask(cpumask_of(cpu_id), RESCHEDULE_IPI, 1); } /*==========================================================================* * Name: smp_reschedule_interrupt * * Description: This routine executes on CPU which received * 'RESCHEDULE_IPI'. * * Born on Date: 2002.02.05 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_reschedule_interrupt(void) { scheduler_ipi(); } /*==========================================================================* * Name: smp_flush_cache_all * * Description: This routine sends a 'INVALIDATE_CACHE_IPI' to all other * CPUs in the system. * * Born on Date: 2003-05-28 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_flush_cache_all(void) { cpumask_t cpumask; unsigned long *mask; preempt_disable(); cpumask_copy(&cpumask, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), &cpumask); spin_lock(&flushcache_lock); mask=cpumask_bits(&cpumask); atomic_set_mask(*mask, (atomic_t *)&flushcache_cpumask); send_IPI_mask(&cpumask, INVALIDATE_CACHE_IPI, 0); _flush_cache_copyback_all(); while (flushcache_cpumask) mb(); spin_unlock(&flushcache_lock); preempt_enable(); } void smp_flush_cache_all_interrupt(void) { _flush_cache_copyback_all(); clear_bit(smp_processor_id(), &flushcache_cpumask); } /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* TLB flush request Routines */ /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /*==========================================================================* * Name: smp_flush_tlb_all * * Description: This routine flushes all processes TLBs. * 1.Request other CPU to execute 'flush_tlb_all_ipi()'. * 2.Execute 'do_flush_tlb_all_local()'. * * Born on Date: 2002.02.05 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_flush_tlb_all(void) { unsigned long flags; preempt_disable(); local_irq_save(flags); __flush_tlb_all(); local_irq_restore(flags); smp_call_function(flush_tlb_all_ipi, NULL, 1); preempt_enable(); } /*==========================================================================* * Name: flush_tlb_all_ipi * * Description: This routine flushes all local TLBs. * 1.Execute 'do_flush_tlb_all_local()'. * * Born on Date: 2002.02.05 * * Arguments: *info - not used * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ static void flush_tlb_all_ipi(void *info) { __flush_tlb_all(); } /*==========================================================================* * Name: smp_flush_tlb_mm * * Description: This routine flushes the specified mm context TLB's. * * Born on Date: 2002.02.05 * * Arguments: *mm - a pointer to the mm struct for flush TLB * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_flush_tlb_mm(struct mm_struct *mm) { int cpu_id; cpumask_t cpu_mask; unsigned long *mmc; unsigned long flags; preempt_disable(); cpu_id = smp_processor_id(); mmc = &mm->context[cpu_id]; cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(cpu_id, &cpu_mask); if (*mmc != NO_CONTEXT) { local_irq_save(flags); *mmc = NO_CONTEXT; if (mm == current->mm) activate_context(mm); else cpumask_clear_cpu(cpu_id, mm_cpumask(mm)); local_irq_restore(flags); } if (!cpumask_empty(&cpu_mask)) flush_tlb_others(cpu_mask, mm, NULL, FLUSH_ALL); preempt_enable(); } /*==========================================================================* * Name: smp_flush_tlb_range * * Description: This routine flushes a range of pages. * * Born on Date: 2002.02.05 * * Arguments: *mm - a pointer to the mm struct for flush TLB * start - not used * end - not used * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { smp_flush_tlb_mm(vma->vm_mm); } /*==========================================================================* * Name: smp_flush_tlb_page * * Description: This routine flushes one page. * * Born on Date: 2002.02.05 * * Arguments: *vma - a pointer to the vma struct include va * va - virtual address for flush TLB * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long va) { struct mm_struct *mm = vma->vm_mm; int cpu_id; cpumask_t cpu_mask; unsigned long *mmc; unsigned long flags; preempt_disable(); cpu_id = smp_processor_id(); mmc = &mm->context[cpu_id]; cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(cpu_id, &cpu_mask); #ifdef DEBUG_SMP if (!mm) BUG(); #endif if (*mmc != NO_CONTEXT) { local_irq_save(flags); va &= PAGE_MASK; va |= (*mmc & MMU_CONTEXT_ASID_MASK); __flush_tlb_page(va); local_irq_restore(flags); } if (!cpumask_empty(&cpu_mask)) flush_tlb_others(cpu_mask, mm, vma, va); preempt_enable(); } /*==========================================================================* * Name: flush_tlb_others * * Description: This routine requests other CPU to execute flush TLB. * 1.Setup parameters. * 2.Send 'INVALIDATE_TLB_IPI' to other CPU. * Request other CPU to execute 'smp_invalidate_interrupt()'. * 3.Wait for other CPUs operation finished. * * Born on Date: 2002.02.05 * * Arguments: cpumask - bitmap of target CPUs * *mm - a pointer to the mm struct for flush TLB * *vma - a pointer to the vma struct include va * va - virtual address for flush TLB * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long va) { unsigned long *mask; #ifdef DEBUG_SMP unsigned long flags; __save_flags(flags); if (!(flags & 0x0040)) /* Interrupt Disable NONONO */ BUG(); #endif /* DEBUG_SMP */ /* * A couple of (to be removed) sanity checks: * * - we do not send IPIs to not-yet booted CPUs. * - current CPU must not be in mask * - mask must exist :) */ BUG_ON(cpumask_empty(&cpumask)); BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask)); BUG_ON(!mm); /* If a CPU which we ran on has gone down, OK. */ cpumask_and(&cpumask, &cpumask, cpu_online_mask); if (cpumask_empty(&cpumask)) return; /* * i'm not happy about this global shared spinlock in the * MM hot path, but we'll see how contended it is. * Temporarily this turns IRQs off, so that lockups are * detected by the NMI watchdog. */ spin_lock(&tlbstate_lock); flush_mm = mm; flush_vma = vma; flush_va = va; mask=cpumask_bits(&cpumask); atomic_set_mask(*mask, (atomic_t *)&flush_cpumask); /* * We have to send the IPI only to * CPUs affected. */ send_IPI_mask(&cpumask, INVALIDATE_TLB_IPI, 0); while (!cpumask_empty((cpumask_t*)&flush_cpumask)) { /* nothing. lockup detection does not belong here */ mb(); } flush_mm = NULL; flush_vma = NULL; flush_va = 0; spin_unlock(&tlbstate_lock); } /*==========================================================================* * Name: smp_invalidate_interrupt * * Description: This routine executes on CPU which received * 'INVALIDATE_TLB_IPI'. * 1.Flush local TLB. * 2.Report flush TLB process was finished. * * Born on Date: 2002.02.05 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_invalidate_interrupt(void) { int cpu_id = smp_processor_id(); unsigned long *mmc = &flush_mm->context[cpu_id]; if (!cpumask_test_cpu(cpu_id, &flush_cpumask)) return; if (flush_va == FLUSH_ALL) { *mmc = NO_CONTEXT; if (flush_mm == current->active_mm) activate_context(flush_mm); else cpumask_clear_cpu(cpu_id, mm_cpumask(flush_mm)); } else { unsigned long va = flush_va; if (*mmc != NO_CONTEXT) { va &= PAGE_MASK; va |= (*mmc & MMU_CONTEXT_ASID_MASK); __flush_tlb_page(va); } } cpumask_clear_cpu(cpu_id, (cpumask_t*)&flush_cpumask); } /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* Stop CPU request Routines */ /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /*==========================================================================* * Name: smp_send_stop * * Description: This routine requests stop all CPUs. * 1.Request other CPU to execute 'stop_this_cpu()'. * * Born on Date: 2002.02.05 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_send_stop(void) { smp_call_function(stop_this_cpu, NULL, 0); } /*==========================================================================* * Name: stop_this_cpu * * Description: This routine halt CPU. * * Born on Date: 2002.02.05 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ static void stop_this_cpu(void *dummy) { int cpu_id = smp_processor_id(); /* * Remove this CPU: */ set_cpu_online(cpu_id, false); /* * PSW IE = 1; * IMASK = 0; * goto SLEEP */ local_irq_disable(); outl(0, M32R_ICU_IMASK_PORTL); inl(M32R_ICU_IMASK_PORTL); /* dummy read */ local_irq_enable(); for ( ; ; ); } void arch_send_call_function_ipi_mask(const struct cpumask *mask) { send_IPI_mask(mask, CALL_FUNCTION_IPI, 0); } void arch_send_call_function_single_ipi(int cpu) { send_IPI_mask(cpumask_of(cpu), CALL_FUNC_SINGLE_IPI, 0); } /*==========================================================================* * Name: smp_call_function_interrupt * * Description: This routine executes on CPU which received * 'CALL_FUNCTION_IPI'. * * Born on Date: 2002.02.05 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_call_function_interrupt(void) { irq_enter(); generic_smp_call_function_interrupt(); irq_exit(); } void smp_call_function_single_interrupt(void) { irq_enter(); generic_smp_call_function_single_interrupt(); irq_exit(); } /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* Timer Routines */ /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /*==========================================================================* * Name: smp_send_timer * * Description: This routine sends a 'LOCAL_TIMER_IPI' to all other CPUs * in the system. * * Born on Date: 2002.02.05 * * Arguments: NONE * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_send_timer(void) { send_IPI_allbutself(LOCAL_TIMER_IPI, 1); } /*==========================================================================* * Name: smp_send_timer * * Description: This routine executes on CPU which received * 'LOCAL_TIMER_IPI'. * * Born on Date: 2002.02.05 * * Arguments: *regs - a pointer to the saved regster info * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ void smp_ipi_timer_interrupt(struct pt_regs *regs) { struct pt_regs *old_regs; old_regs = set_irq_regs(regs); irq_enter(); smp_local_timer_interrupt(); irq_exit(); set_irq_regs(old_regs); } /*==========================================================================* * Name: smp_local_timer_interrupt * * Description: Local timer interrupt handler. It does both profiling and * process statistics/rescheduling. * We do profiling in every local tick, statistics/rescheduling * happen only every 'profiling multiplier' ticks. The default * multiplier is 1 and it can be changed by writing the new * multiplier value into /proc/profile. * * Born on Date: 2002.02.05 * * Arguments: *regs - a pointer to the saved regster info * * Returns: void (cannot fail) * * Original: arch/i386/kernel/apic.c * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * 2003-06-24 hy use per_cpu structure. *==========================================================================*/ void smp_local_timer_interrupt(void) { int user = user_mode(get_irq_regs()); int cpu_id = smp_processor_id(); /* * The profiling function is SMP safe. (nothing can mess * around with "current", and the profiling counters are * updated with atomic operations). This is especially * useful with a profiling multiplier != 1 */ profile_tick(CPU_PROFILING); if (--per_cpu(prof_counter, cpu_id) <= 0) { /* * The multiplier may have changed since the last time we got * to this point as a result of the user writing to * /proc/profile. In this case we need to adjust the APIC * timer accordingly. * * Interrupts are already masked off at this point. */ per_cpu(prof_counter, cpu_id) = per_cpu(prof_multiplier, cpu_id); if (per_cpu(prof_counter, cpu_id) != per_cpu(prof_old_multiplier, cpu_id)) { per_cpu(prof_old_multiplier, cpu_id) = per_cpu(prof_counter, cpu_id); } update_process_times(user); } } /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* Send IPI Routines */ /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /*==========================================================================* * Name: send_IPI_allbutself * * Description: This routine sends a IPI to all other CPUs in the system. * * Born on Date: 2002.02.05 * * Arguments: ipi_num - Number of IPI * try - 0 : Send IPI certainly. * !0 : The following IPI is not sent when Target CPU * has not received the before IPI. * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ static void send_IPI_allbutself(int ipi_num, int try) { cpumask_t cpumask; cpumask_copy(&cpumask, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), &cpumask); send_IPI_mask(&cpumask, ipi_num, try); } /*==========================================================================* * Name: send_IPI_mask * * Description: This routine sends a IPI to CPUs in the system. * * Born on Date: 2002.02.05 * * Arguments: cpu_mask - Bitmap of target CPUs logical ID * ipi_num - Number of IPI * try - 0 : Send IPI certainly. * !0 : The following IPI is not sent when Target CPU * has not received the before IPI. * * Returns: void (cannot fail) * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ static void send_IPI_mask(const struct cpumask *cpumask, int ipi_num, int try) { cpumask_t physid_mask, tmp; int cpu_id, phys_id; int num_cpus = num_online_cpus(); if (num_cpus <= 1) /* NO MP */ return; cpumask_and(&tmp, cpumask, cpu_online_mask); BUG_ON(!cpumask_equal(cpumask, &tmp)); cpumask_clear(&physid_mask); for_each_cpu(cpu_id, cpumask) { if ((phys_id = cpu_to_physid(cpu_id)) != -1) cpumask_set_cpu(phys_id, &physid_mask); } send_IPI_mask_phys(&physid_mask, ipi_num, try); } /*==========================================================================* * Name: send_IPI_mask_phys * * Description: This routine sends a IPI to other CPUs in the system. * * Born on Date: 2002.02.05 * * Arguments: cpu_mask - Bitmap of target CPUs physical ID * ipi_num - Number of IPI * try - 0 : Send IPI certainly. * !0 : The following IPI is not sent when Target CPU * has not received the before IPI. * * Returns: IPICRi regster value. * * Modification log: * Date Who Description * ---------- --- -------------------------------------------------------- * *==========================================================================*/ unsigned long send_IPI_mask_phys(const cpumask_t *physid_mask, int ipi_num, int try) { spinlock_t *ipilock; volatile unsigned long *ipicr_addr; unsigned long ipicr_val; unsigned long my_physid_mask; unsigned long mask = cpumask_bits(physid_mask)[0]; if (mask & ~physids_coerce(phys_cpu_present_map)) BUG(); if (ipi_num >= NR_IPIS || ipi_num < 0) BUG(); mask <<= IPI_SHIFT; ipilock = &ipi_lock[ipi_num]; ipicr_addr = (volatile unsigned long *)(M32R_ICU_IPICR_ADDR + (ipi_num << 2)); my_physid_mask = ~(1 << smp_processor_id()); /* * lock ipi_lock[i] * check IPICRi == 0 * write IPICRi (send IPIi) * unlock ipi_lock[i] */ spin_lock(ipilock); __asm__ __volatile__ ( ";; CHECK IPICRi == 0 \n\t" ".fillinsn \n" "1: \n\t" "ld %0, @%1 \n\t" "and %0, %4 \n\t" "beqz %0, 2f \n\t" "bnez %3, 3f \n\t" "bra 1b \n\t" ";; WRITE IPICRi (send IPIi) \n\t" ".fillinsn \n" "2: \n\t" "st %2, @%1 \n\t" ".fillinsn \n" "3: \n\t" : "=&r"(ipicr_val) : "r"(ipicr_addr), "r"(mask), "r"(try), "r"(my_physid_mask) : "memory" ); spin_unlock(ipilock); return ipicr_val; }