x86: Enhanced copy capabilities for Hygon processor

hygon inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/IAQQDF
CVE: NA

 ---------------------------

The following methods are used to improve the large memory copy
performance of the Hygon processor between kernel and user mode.

Prefetch is a technique for reading blocks of data from main memory at
very high data rates, then operating on them within the cache. Results
are then written out to memory, all with high efficiency.

The code can employ a very special instruction: NT. This is a streaming
store instruction for writing data to memory. This instruction bypasses
the on-chip cache and sends data directly into a write-combining buffer.
Because NT allows the CPU to avoid reading the old data from the memory
destination address, NT can effectively improve the total write bandwidth.
There are similar optimizations for reading data from memory.

Interruptions may occur when copying large memory, which may trigger
thread switching. You need to save the current MMX register context and
continue copying when switching back to the thread next time.

Signed-off-by: zhuchao <zhuchao@hygon.cn>
Signed-off-by: qiuzhiteng <qiuzhiteng@hygon.cn>

Author: zhitengqiu

arch/x86/Kconfig

@@ -899,6 +899,7 @@ config INTEL_TDX_GUEST
 endif # HYPERVISOR_GUEST
 
 source "arch/x86/Kconfig.cpu"
+source "arch/x86/Kconfig.fpu"
 
 config HPET_TIMER
 	def_bool X86_64

arch/x86/Kconfig.fpu

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: GPL-2.0
+
+menuconfig USING_FPU_IN_KERNEL_NONATOMIC
+	bool "Hygon large memory copy support"
+	depends on X86_64 && CPU_SUP_HYGON
+	default y
+	help
+	  This option enables support for optimized large memory copy operations
+	  on Hygon processors in the kernel space using SSE2 or AVX2 non-temporal (NT)
+	  copy instructions. NT instructions are streaming store instructions that bypass
+	  the on-chip cache and send data directly to a write-combining buffer.
+
+	  When this option is enabled, you can choose the specific instruction set to use
+	  for large memory copy: SSE2 or AVX2. Using these instruction sets can improve data
+	  throughput and reduce the number of cache misses during memory copy operations.
+
+if USING_FPU_IN_KERNEL_NONATOMIC
+
+choice
+	prompt "X86_HYGON_LMC"
+	depends on X86_64 && CPU_SUP_HYGON
+	default X86_HYGON_LMC_AVX2_ON
+	help
+	  Select the type of non-temporal (NT) copy instructions to use for
+	  large memory copy operations between kernel and user mode. You can
+	  choose between SSE2 or AVX2 instructions based on the processor
+	  capabilities and the size of the memory being copied.
+
+	  To use this feature, you also need to configure the data copy size.
+	  The file is in `/sys/c86_features/hygon_c86/nt_cpy_mini_len`. Please
+	  refer to configuration 4096 and above.
+
+config X86_HYGON_LMC_SSE2_ON
+	bool "Using sse2 nt copy for large memory copy"
+	help
+	  When this feature is enabled, the kernel will use the
+	  copy_user_sse2_opt_string function for large memory copy operations.
+
+	  SSE2 (Streaming SIMD Extensions 2) instructions support non-temporal
+	  (NT) stores that bypass the CPU cache and write data directly to
+	  memory. This can improve performance for large memory copies by reducing
+	  cache pollution and taking advantage of the write-combining buffer.
+
+	  However, using SSE2 NT copy may require saving and restoring MMX and
+	  SSE2 register contexts during thread switching if an interruption occurs.
+
+config X86_HYGON_LMC_AVX2_ON
+	bool "Using avx2 nt copy for large memory copy"
+	help
+	  When this feature is enabled, the kernel will use the
+	  copy_user_avx2_pf64_nt_string function for large memory copy operations.
+
+	  AVX2 (Advanced Vector Extensions 2) instructions provide enhanced
+	  vector processing capabilities and support for non-temporal (NT) stores,
+	  which can significantly improve memory copy performance for large blocks
+	  of data. By bypassing the cache and writing data directly to memory,
+	  AVX2 NT copy can achieve higher throughput than SSE2 NT copy.
+
+	  Similar to SSE2, using AVX2 NT copy may require saving and restoring
+	  AVX2 register contexts if an interruption occurs during large memory
+	  copying, to ensure the process continues smoothly after resuming.
+
+endchoice
+endif

arch/x86/configs/deepin_x86_desktop_defconfig

@@ -5897,3 +5897,4 @@ CONFIG_IO_DELAY_0XED=y
 # CONFIG_X86_DEBUG_FPU is not set
 CONFIG_UNWINDER_FRAME_POINTER=y
 # CONFIG_RUNTIME_TESTING_MENU is not set
+

arch/x86/include/asm/fpu/api.h

@@ -49,6 +49,43 @@ static inline void kernel_fpu_begin(void)
 #endif
 }
 
+#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \
+	defined(CONFIG_X86_HYGON_LMC_AVX2_ON)
+extern int kernel_fpu_begin_nonatomic_mask(unsigned int kfpu_mask);
+extern void kernel_fpu_end_nonatomic(void);
+
+/* Code that is unaware of kernel_fpu_begin_nonatomic_mask() can use this */
+static inline int kernel_fpu_begin_nonatomic(void)
+{
+#ifdef CONFIG_X86_64
+	/*
+	 * Any 64-bit code that uses 387 instructions must explicitly request
+	 * KFPU_387.
+	 */
+	return kernel_fpu_begin_nonatomic_mask(KFPU_MXCSR);
+#else
+	/*
+	 * 32-bit kernel code may use 387 operations as well as SSE2, etc,
+	 * as long as it checks that the CPU has the required capability.
+	 */
+	return kernel_fpu_begin_nonatomic_mask(KFPU_387 | KFPU_MXCSR);
+#endif
+}
+
+/*
+ * It means we call kernel_fpu_end after kernel_fpu_begin_nonatomic
+ * func, but before kernel_fpu_end_nonatomic
+ */
+static inline void check_using_kernel_fpu(void)
+{
+	WARN_ON_ONCE(test_thread_flag(TIF_USING_FPU_NONATOMIC));
+}
+
+#else
+static inline void check_using_kernel_fpu(void) { }
+
+#endif
+
 /*
  * Use fpregs_lock() while editing CPU's FPU registers or fpu->fpstate.
  * A context switch will (and softirq might) save CPU's FPU registers to

arch/x86/include/asm/fpu/sched.h

@@ -66,4 +66,62 @@ static inline void switch_fpu_finish(void)
 		set_thread_flag(TIF_NEED_FPU_LOAD);
 }
 
+/*
+ * Kernel FPU state switching for scheduling.
+ *
+ * This is a two-stage process:
+ *
+ *  - switch_kernel_fpu_prepare() saves the old kernel fpu state.
+ *    This is done within the context of the old process.
+ *
+ *  - switch_kernel_fpu_finish() restore new kernel fpu state.
+ *
+ * The kernel FPU context is only stored/restored for a user task in kernel
+ * mode and PF_KTHREAD is used to distinguish between kernel and user threads.
+ */
+#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \
+	defined(CONFIG_X86_HYGON_LMC_AVX2_ON)
+extern void save_fpregs_to_fpkernelstate(struct fpu *kfpu);
+extern unsigned long get_fpu_registers_pos(struct fpu *fpu, unsigned int off);
+static inline void switch_kernel_fpu_prepare(struct task_struct *prev, int cpu)
+{
+	struct fpu *old_fpu = &prev->thread.fpu;
+
+	if (!test_thread_flag(TIF_USING_FPU_NONATOMIC))
+		return;
+
+	if (static_cpu_has(X86_FEATURE_FPU) && !(prev->flags & PF_KTHREAD))
+		save_fpregs_to_fpkernelstate(old_fpu);
+}
+
+/* Internal helper for switch_kernel_fpu_finish() and signal frame setup */
+static inline void fpregs_restore_kernelregs(struct fpu *kfpu)
+{
+	kernel_fpu_states_restore(NULL, (void *)get_fpu_registers_pos(kfpu, MAX_FPU_CTX_SIZE),
+						MAX_FPU_CTX_SIZE);
+}
+
+/* Loading of the complete FPU state immediately. */
+static inline void switch_kernel_fpu_finish(struct task_struct *next)
+{
+	struct fpu *new_fpu = &next->thread.fpu;
+
+	if (next->flags & PF_KTHREAD)
+		return;
+
+	if (cpu_feature_enabled(X86_FEATURE_FPU) &&
+	    test_ti_thread_flag((struct thread_info *)next,
+				TIF_USING_FPU_NONATOMIC))
+		fpregs_restore_kernelregs(new_fpu);
+}
+#else
+static inline void switch_kernel_fpu_prepare(struct task_struct *prev, int cpu)
+{
+}
+static inline void switch_kernel_fpu_finish(struct task_struct *next)
+{
+}
+
+#endif
+
 #endif /* _ASM_X86_FPU_SCHED_H */

arch/x86/include/asm/thread_info.h

@@ -103,6 +103,7 @@ struct thread_info {
 #define TIF_BLOCKSTEP		25	/* set when we want DEBUGCTLMSR_BTF */
 #define TIF_LAZY_MMU_UPDATES	27	/* task is updating the mmu lazily */
 #define TIF_ADDR32		29	/* 32-bit address space on 64 bits */
+#define TIF_USING_FPU_NONATOMIC 30      /* using fpu in kernel non-atomic context */
 
 #define _TIF_NOTIFY_RESUME	(1 << TIF_NOTIFY_RESUME)
 #define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)

arch/x86/include/asm/uaccess_64.h

@@ -11,6 +11,12 @@
 #include <asm/alternative.h>
 #include <asm/cpufeatures.h>
 #include <asm/page.h>
+#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \
+	defined(CONFIG_X86_HYGON_LMC_AVX2_ON)
+#include <asm/fpu/api.h>
+#endif
+
+extern struct static_key_false hygon_lmc_key;
 
 #ifdef CONFIG_ADDRESS_MASKING
 /*
@@ -97,13 +103,91 @@ static inline bool __access_ok(const void __user *ptr, unsigned long size)
  * Copy To/From Userspace
  */
 
+#ifdef CONFIG_X86_HYGON_LMC_SSE2_ON
+void fpu_save_xmm0_3(void *to, const void *from, unsigned long len);
+void fpu_restore_xmm0_3(void *to, const void *from, unsigned long len);
+
+#define kernel_fpu_states_save fpu_save_xmm0_3
+#define kernel_fpu_states_restore fpu_restore_xmm0_3
+
+__must_check unsigned long copy_user_sse2_opt_string(void *to, const void *from,
+						     unsigned long len);
+
+#define MAX_FPU_CTX_SIZE 64
+#define KERNEL_FPU_NONATOMIC_SIZE (2 * (MAX_FPU_CTX_SIZE))
+
+#define copy_user_large_memory_generic_string copy_user_sse2_opt_string
+
+#endif
+
+#ifdef CONFIG_X86_HYGON_LMC_AVX2_ON
+void fpu_save_ymm0_7(void *to, const void *from, unsigned long len);
+void fpu_restore_ymm0_7(void *to, const void *from, unsigned long len);
+
+#define kernel_fpu_states_save fpu_save_ymm0_7
+#define kernel_fpu_states_restore fpu_restore_ymm0_7
+
+__must_check unsigned long
+copy_user_avx2_pf64_nt_string(void *to, const void *from, unsigned long len);
+
+#define MAX_FPU_CTX_SIZE 256
+#define KERNEL_FPU_NONATOMIC_SIZE (2 * (MAX_FPU_CTX_SIZE))
+
+#define copy_user_large_memory_generic_string copy_user_avx2_pf64_nt_string
+#endif
+
+#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \
+	defined(CONFIG_X86_HYGON_LMC_AVX2_ON)
+unsigned int get_nt_block_copy_mini_len(void);
+static inline bool Hygon_LMC_check(unsigned long len)
+{
+	unsigned int nt_blk_cpy_mini_len = get_nt_block_copy_mini_len();
+
+	if (((nt_blk_cpy_mini_len) && (nt_blk_cpy_mini_len <= len) &&
+	     (system_state == SYSTEM_RUNNING) &&
+	     (!kernel_fpu_begin_nonatomic())))
+		return true;
+	else
+		return false;
+}
+static inline unsigned long
+copy_large_memory_generic_string(void *to, const void *from, unsigned long len)
+{
+	unsigned long ret;
+
+	ret = copy_user_large_memory_generic_string(to, from, len);
+	kernel_fpu_end_nonatomic();
+	return ret;
+}
+#else
+static inline bool Hygon_LMC_check(unsigned long len)
+{
+	return false;
+}
+static inline unsigned long
+copy_large_memory_generic_string(void *to, const void *from, unsigned long len)
+{
+	return 0;
+}
+#endif
+
 /* Handles exceptions in both to and from, but doesn't do access_ok */
 __must_check unsigned long
 rep_movs_alternative(void *to, const void *from, unsigned len);
 
 static __always_inline __must_check unsigned long
 copy_user_generic(void *to, const void *from, unsigned long len)
 {
+	/* Check if Hygon large memory copy support enabled. */
+	if (static_branch_unlikely(&hygon_lmc_key)) {
+		if (Hygon_LMC_check(len)) {
+			unsigned long ret;
+
+			ret = copy_large_memory_generic_string(to, from, len);
+			return ret;
+		}
+	}
+
 	stac();
 	/*
 	 * If CPU has FSRM feature, use 'rep movs'.

arch/x86/kernel/cpu/common.c

@@ -92,6 +92,9 @@ EXPORT_SYMBOL_GPL(get_llc_id);
 /* L2 cache ID of each logical CPU */
 DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id) = BAD_APICID;
 
+DEFINE_STATIC_KEY_FALSE(hygon_lmc_key);
+EXPORT_SYMBOL_GPL(hygon_lmc_key);
+
 static struct ppin_info {
 	int	feature;
 	int	msr_ppin_ctl;
@@ -2511,6 +2514,17 @@ void arch_smt_update(void)
 	apic_smt_update();
 }
 
+#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \
+	defined(CONFIG_X86_HYGON_LMC_AVX2_ON)
+static inline void update_lmc_branch_cond(void)
+{
+	if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
+		static_branch_enable(&hygon_lmc_key);
+}
+#else
+static inline void update_lmc_branch_cond(void) { }
+#endif
+
 void __init arch_cpu_finalize_init(void)
 {
 	identify_boot_cpu();
@@ -2530,6 +2544,8 @@ void __init arch_cpu_finalize_init(void)
 
 	arch_smt_update();
 
+	update_lmc_branch_cond();
+
 	if (IS_ENABLED(CONFIG_X86_32)) {
 		/*
 		 * Check whether this is a real i386 which is not longer