// SPDX-License-Identifier: GPL-2.0 #include #include #include #include #include #include #include #include #include static int has_steal_clock; struct static_key paravirt_steal_enabled; struct static_key paravirt_steal_rq_enabled; static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); static u64 native_steal_clock(int cpu) { return 0; } DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); static bool steal_acc = true; static int __init parse_no_stealacc(char *arg) { steal_acc = false; return 0; } early_param("no-steal-acc", parse_no_stealacc); static u64 paravt_steal_clock(int cpu) { int version; u64 steal; struct kvm_steal_time *src; src = &per_cpu(steal_time, cpu); do { version = src->version; virt_rmb(); /* Make sure that the version is read before the steal */ steal = src->steal; virt_rmb(); /* Make sure that the steal is read before the next version */ } while ((version & 1) || (version != src->version)); return steal; } #ifdef CONFIG_SMP static void pv_send_ipi_single(int cpu, unsigned int action) { int min, old; irq_cpustat_t *info = &per_cpu(irq_stat, cpu); old = atomic_fetch_or(BIT(action), &info->message); if (old) return; min = cpu_logical_map(cpu); kvm_hypercall3(KVM_HCALL_FUNC_IPI, 1, 0, min); } #define KVM_IPI_CLUSTER_SIZE (2 * BITS_PER_LONG) static void pv_send_ipi_mask(const struct cpumask *mask, unsigned int action) { int i, cpu, min = 0, max = 0, old; __uint128_t bitmap = 0; irq_cpustat_t *info; if (cpumask_empty(mask)) return; action = BIT(action); for_each_cpu(i, mask) { info = &per_cpu(irq_stat, i); old = atomic_fetch_or(action, &info->message); if (old) continue; cpu = cpu_logical_map(i); if (!bitmap) { min = max = cpu; } else if (cpu < min && cpu > (max - KVM_IPI_CLUSTER_SIZE)) { /* cpu < min, and bitmap still enough */ bitmap <<= min - cpu; min = cpu; } else if (cpu > min && cpu < (min + KVM_IPI_CLUSTER_SIZE)) { /* cpu > min, and bitmap still enough */ max = cpu > max ? cpu : max; } else { /* * With cpu, bitmap will exceed KVM_IPI_CLUSTER_SIZE, * send IPI here directly and skip the remaining CPUs. */ kvm_hypercall3(KVM_HCALL_FUNC_IPI, (unsigned long)bitmap, (unsigned long)(bitmap >> BITS_PER_LONG), min); min = max = cpu; bitmap = 0; } __set_bit(cpu - min, (unsigned long *)&bitmap); } if (bitmap) kvm_hypercall3(KVM_HCALL_FUNC_IPI, (unsigned long)bitmap, (unsigned long)(bitmap >> BITS_PER_LONG), min); } static irqreturn_t pv_ipi_interrupt(int irq, void *dev) { u32 action; irq_cpustat_t *info; /* Clear SWI interrupt */ clear_csr_estat(1 << INT_SWI0); info = this_cpu_ptr(&irq_stat); action = atomic_xchg(&info->message, 0); if (action & SMP_RESCHEDULE) { scheduler_ipi(); info->ipi_irqs[IPI_RESCHEDULE]++; } if (action & SMP_CALL_FUNCTION) { generic_smp_call_function_interrupt(); info->ipi_irqs[IPI_CALL_FUNCTION]++; } if (action & SMP_IRQ_WORK) { irq_work_run(); info->ipi_irqs[IPI_IRQ_WORK]++; } if (action & SMP_CLEAR_VECTOR) { complete_irq_moving(); info->ipi_irqs[IPI_CLEAR_VECTOR]++; } return IRQ_HANDLED; } static void pv_init_ipi(void) { int r, swi; swi = get_percpu_irq(INT_SWI0); if (swi < 0) panic("SWI0 IRQ mapping failed\n"); irq_set_percpu_devid(swi); r = request_percpu_irq(swi, pv_ipi_interrupt, "SWI0-IPI", &irq_stat); if (r < 0) panic("SWI0 IRQ request failed\n"); } #endif bool kvm_para_available(void) { int config; static int hypervisor_type; if (!cpu_has_hypervisor) return false; if (!hypervisor_type) { config = read_cpucfg(CPUCFG_KVM_SIG); if (!memcmp(&config, KVM_SIGNATURE, 4)) hypervisor_type = HYPERVISOR_KVM; } return hypervisor_type == HYPERVISOR_KVM; } unsigned int kvm_arch_para_features(void) { static unsigned int feature; if (!kvm_para_available()) return 0; if (!feature) feature = read_cpucfg(CPUCFG_KVM_FEATURE); return feature; } int __init pv_ipi_init(void) { if (!kvm_para_has_feature(KVM_FEATURE_IPI)) return 0; #ifdef CONFIG_SMP mp_ops.init_ipi = pv_init_ipi; mp_ops.send_ipi_single = pv_send_ipi_single; mp_ops.send_ipi_mask = pv_send_ipi_mask; #endif return 0; } static int pv_enable_steal_time(void) { int cpu = smp_processor_id(); unsigned long addr; struct kvm_steal_time *st; if (!has_steal_clock) return -EPERM; st = &per_cpu(steal_time, cpu); addr = per_cpu_ptr_to_phys(st); /* The whole structure kvm_steal_time should be in one page */ if (PFN_DOWN(addr) != PFN_DOWN(addr + sizeof(*st))) { pr_warn("Illegal PV steal time addr %lx\n", addr); return -EFAULT; } addr |= KVM_STEAL_PHYS_VALID; kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), addr); return 0; } static void pv_disable_steal_time(void) { if (has_steal_clock) kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), 0); } #ifdef CONFIG_SMP static int pv_time_cpu_online(unsigned int cpu) { unsigned long flags; local_irq_save(flags); pv_enable_steal_time(); local_irq_restore(flags); return 0; } static int pv_time_cpu_down_prepare(unsigned int cpu) { unsigned long flags; local_irq_save(flags); pv_disable_steal_time(); local_irq_restore(flags); return 0; } #endif static void pv_cpu_reboot(void *unused) { pv_disable_steal_time(); } static int pv_reboot_notify(struct notifier_block *nb, unsigned long code, void *unused) { on_each_cpu(pv_cpu_reboot, NULL, 1); return NOTIFY_DONE; } static struct notifier_block pv_reboot_nb = { .notifier_call = pv_reboot_notify, }; int __init pv_time_init(void) { int r; if (!kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) return 0; has_steal_clock = 1; r = pv_enable_steal_time(); if (r < 0) { has_steal_clock = 0; return 0; } register_reboot_notifier(&pv_reboot_nb); #ifdef CONFIG_SMP r = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "loongarch/pv_time:online", pv_time_cpu_online, pv_time_cpu_down_prepare); if (r < 0) { has_steal_clock = 0; pr_err("Failed to install cpu hotplug callbacks\n"); return r; } #endif static_call_update(pv_steal_clock, paravt_steal_clock); static_key_slow_inc(¶virt_steal_enabled); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (steal_acc) static_key_slow_inc(¶virt_steal_rq_enabled); #endif pr_info("Using paravirt steal-time\n"); return 0; } int __init pv_spinlock_init(void) { if (!cpu_has_hypervisor) return 0; static_branch_enable(&virt_spin_lock_key); return 0; }