RISCV: Support cpu hotplug.

This patch enable support for cpu hotplug in RISC-V. In absensece of generic cpu stop functions, WFI is used to put the cpu in low power state during offline. An IPI is sent to bring it out of WFI during online operation. Tested both on QEMU and HighFive Unleashed board with 4 cpus. Test result follows. $ echo 0 > /sys/devices/system/cpu/cpu2/online [ 31.828562] CPU2: shutdown $ cat /proc/cpuinfo hart : 1 isa : rv64imafdc mmu : sv39 uarch : sifive,rocket0 hart : 3 isa : rv64imafdc mmu : sv39 uarch : sifive,rocket0 hart : 4 isa : rv64imafdc mmu : sv39 uarch : sifive,rocket0 $ echo 0 > /sys/devices/system/cpu/cpu4/online [ 52.968495] CPU4: shutdown $ cat /proc/cpuinfo hart : 1 isa : rv64imafdc mmu : sv39 uarch : sifive,rocket0 hart : 3 isa : rv64imafdc mmu : sv39 uarch : sifive,rocket0 $ echo 1 > /sys/devices/system/cpu/cpu4/online [ 64.298250] CPU4: online $ cat /proc/cpuinfo hart : 1 isa : rv64imafdc mmu : sv39 uarch : sifive,rocket0 hart : 3 isa : rv64imafdc mmu : sv39 uarch : sifive,rocket0 hart : 4 isa : rv64imafdc mmu : sv39 uarch : sifive,rocket0 Signed-off-by: Atish Patra <[email protected]> Signed-off-by: Palmer Dabbelt <[email protected]>
riscvarchive · Jun 22, 2018 · cf0183b · cf0183b
1 parent bce521e
commit cf0183b
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 10 deletions.
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
@@ -22,7 +22,6 @@ config RISCV
 	select COMMON_CLK
 	select DMA_DIRECT_OPS
 	select GENERIC_CLOCKEVENTS
-	select GENERIC_CPU_DEVICES
 	select GENERIC_IRQ_SHOW
 	select GENERIC_PCI_IOMAP
 	select GENERIC_STRNCPY_FROM_USER
@@ -174,6 +173,16 @@ config NR_CPUS
 	depends on SMP
 	default "8"
 
+config HOTPLUG_CPU
+	bool "Support for hot-pluggable CPUs"
+	select GENERIC_IRQ_MIGRATION
+	help
+
+	  Say Y here to experiment with turning CPUs off and on.  CPUs
+	  can be controlled through /sys/devices/system/cpu.
+
+	  Say N if you want to disable CPU hotplug.
+
 choice
 	prompt "CPU Tuning"
 	default TUNE_GENERIC

diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
@@ -54,6 +54,7 @@
 /* Interrupt Enable and Interrupt Pending flags */
 #define SIE_SSIE _AC(0x00000002, UL) /* Software Interrupt Enable */
 #define SIE_STIE _AC(0x00000020, UL) /* Timer Interrupt Enable */
+#define SIE_SEIE _AC(0x000000200, UL) /* External Interrupt Enable */
 
 #define EXC_INST_MISALIGNED     0
 #define EXC_INST_ACCESS         1

diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
@@ -24,9 +24,6 @@
 
 #ifdef CONFIG_SMP
 
-/* SMP initialization hook for setup_arch */
-void init_clockevent(void);
-
 /* SMP initialization hook for setup_arch */
 void __init setup_smp(void);
 
@@ -47,6 +44,12 @@ void arch_send_call_function_single_ipi(int cpu);
 /* Interprocessor interrupt handler */
 irqreturn_t handle_ipi(void);
 
+#ifdef CONFIG_HOTPLUG_CPU
+extern int __cpu_disable(void);
+extern void __cpu_die(unsigned int cpu);
+extern void cpu_play_dead(void);
+extern void boot_sec_cpu(void);
+#endif
 #endif /* CONFIG_SMP */
 
 #endif /* _ASM_RISCV_SMP_H */
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
@@ -149,6 +149,18 @@ relocate:
 	j .Lsecondary_park
 END(_start)
 
+.section .text
+.global boot_sec_cpu
+
+boot_sec_cpu:
+	/* clear all pending flags */
+	csrw sip, zero
+	/* Mask all interrupts */
+	csrw sie, zero
+	fence
+
+	tail smp_callin
+
 __PAGE_ALIGNED_BSS
 	/* Empty zero page */
 	.balign PAGE_SIZE
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
@@ -42,6 +42,13 @@ void arch_cpu_idle(void)
 	local_irq_enable();
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+void arch_cpu_idle_dead(void)
+{
+	cpu_play_dead();
+}
+#endif
+
 void show_regs(struct pt_regs *regs)
 {
 	show_regs_print_info(KERN_DEFAULT);

diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
@@ -60,6 +60,7 @@ EXPORT_SYMBOL(empty_zero_page);
 
 /* The lucky hart to first increment this variable will boot the other cores */
 atomic_t hart_lottery;
+static DEFINE_PER_CPU(struct cpu, cpu_devices);
 
 #ifdef CONFIG_BLK_DEV_INITRD
 static void __init setup_initrd(void)
@@ -220,6 +221,22 @@ void __init setup_arch(char **cmdline_p)
 	riscv_fill_hwcap();
 }
 
+static int __init topology_init(void)
+{
+	int i;
+
+	for_each_possible_cpu(i) {
+		struct cpu *cpu = &per_cpu(cpu_devices, i);
+#ifdef CONFIG_HOTPLUG_CPU
+		cpu->hotpluggable = 1;
+#endif
+		register_cpu(cpu, i);
+	}
+
+	return 0;
+}
+subsys_initcall(topology_init);
+
 static int __init riscv_device_init(void)
 {
 	return of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);

diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
@@ -30,6 +30,7 @@
 #include <linux/irq.h>
 #include <linux/of.h>
 #include <linux/sched/task_stack.h>
+#include <linux/sched/hotplug.h>
 #include <asm/irq.h>
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
@@ -82,20 +83,84 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 	__cpu_up_stack_pointer[cpu] = task_stack_page(tidle) + THREAD_SIZE;
 	__cpu_up_task_pointer[cpu] = tidle;
 
+	arch_send_call_function_single_ipi(cpu);
 	while (!cpu_online(cpu))
 		cpu_relax();
 
+	pr_notice("CPU%u: online\n", cpu);
 	return 0;
 }
 
 void __init smp_cpus_done(unsigned int max_cpus)
 {
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * __cpu_disable runs on the processor to be shutdown.
+ */
+int __cpu_disable(void)
+{
+	unsigned int cpu = smp_processor_id();
+	int ret;
+
+	set_cpu_online(cpu, false);
+	irq_migrate_all_off_this_cpu();
+
+	return 0;
+}
+/*
+ * called on the thread which is asking for a CPU to be shutdown -
+ * waits until shutdown has completed, or it is timed out.
+ */
+void __cpu_die(unsigned int cpu)
+{
+	int err = 0;
+
+	if (!cpu_wait_death(cpu, 5)) {
+		pr_err("CPU %u: didn't die\n", cpu);
+		return;
+	}
+	pr_notice("CPU%u: shutdown\n", cpu);
+}
+/*
+ * Called from the idle thread for the CPU which has been shutdown.
+ */
+void cpu_play_dead(void)
+{
+	int sipval, sieval, scauseval;
+	int cpu = smp_processor_id();
+
+	idle_task_exit();
+
+	(void)cpu_report_death();
+
+	/* Do not disable software interrupt to restart cpu after WFI */
+	csr_clear(sie, SIE_STIE | SIE_SEIE);
+
+	/* clear all pending flags */
+	csr_write(sip, 0);
+	/* clear any previous scause data */
+	csr_write(scause, 0);
+
+	do {
+		wait_for_interrupt();
+		sipval = csr_read(sip);
+		sieval = csr_read(sie);
+		scauseval = csr_read(scause);
+	/* only break if wfi returns for an enabled interrupt */
+	} while ((sipval & sieval) == 0 &&
+		 scauseval != INTERRUPT_CAUSE_SOFTWARE);
+
+	boot_sec_cpu();
+}
+
+
+#endif
 /*
  * C entry point for a secondary processor.
  */
-asmlinkage void __init smp_callin(void)
+asmlinkage void smp_callin(void)
 {
 	struct mm_struct *mm = &init_mm;
 
@@ -104,9 +169,8 @@ asmlinkage void __init smp_callin(void)
 	current->active_mm = mm;
 
 	trap_init();
-	init_clockevent();
 	notify_cpu_starting(smp_processor_id());
-	set_cpu_online(smp_processor_id(), 1);
+	set_cpu_online(smp_processor_id(), true);
 	local_flush_tlb_all();
 	local_irq_enable();
 	preempt_disable();

diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
@@ -154,7 +154,7 @@ int is_valid_bugaddr(unsigned long pc)
 }
 #endif /* CONFIG_GENERIC_BUG */
 
-void __init trap_init(void)
+void trap_init(void)
 {
 	/*
 	 * Set sup0 scratch register to 0, indicating to exception vector
@@ -163,6 +163,6 @@ void __init trap_init(void)
 	csr_write(sscratch, 0);
 	/* Set the exception vector address */
 	csr_write(stvec, &handle_exception);
-	/* Enable all interrupts */
-	csr_write(sie, -1);
+	/* Enable all interrupts but timer interrupt*/
+	csr_set(sie, SIE_SSIE | SIE_SEIE);
 }