diff --git a/Makefile b/Makefile index 2ac6cfb..1dca17a 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 5 PATCHLEVEL = 4 -SUBLEVEL = 58 -EXTRAVERSION = -2.3 +SUBLEVEL = 91 +EXTRAVERSION = -2.9 NAME = Kleptomaniac Octopus # *DOCUMENTATION* @@ -394,8 +394,13 @@ HOST_LFS_CFLAGS := $(shell getconf LFS_CFLAGS 2>/dev/null) HOST_LFS_LDFLAGS := $(shell getconf LFS_LDFLAGS 2>/dev/null) HOST_LFS_LIBS := $(shell getconf LFS_LIBS 2>/dev/null) -HOSTCC = gcc -HOSTCXX = g++ +ifneq ($(LLVM),) +HOSTCC = clang +HOSTCXX = clang++ +else +HOSTCC = gcc +HOSTCXX = g++ +endif KBUILD_HOSTCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 \ -fomit-frame-pointer -std=gnu89 $(HOST_LFS_CFLAGS) \ $(HOSTCFLAGS) @@ -404,39 +409,56 @@ KBUILD_HOSTLDFLAGS := $(HOST_LFS_LDFLAGS) $(HOSTLDFLAGS) KBUILD_HOSTLDLIBS := $(HOST_LFS_LIBS) $(HOSTLDLIBS) # Make variables (CC, etc...) -CC = $(CROSS_COMPILE)gcc CPP = $(CC) -E +ifneq ($(LLVM),) +CC = clang +LD = ld.lld +AR = llvm-ar +NM = llvm-nm +OBJCOPY = llvm-objcopy +OBJDUMP = llvm-objdump +READELF = llvm-readelf +OBJSIZE = llvm-size +STRIP = llvm-strip +else +CC = $(CROSS_COMPILE)gcc ifeq ($(call cc-lcc-yn),y) -AS := $(shell $(CC) -print-prog-name=as) -LD := $(shell $(CC) -print-prog-name=ld) +LD := $(shell $(CC) -print-prog-name=ld) AR := $(shell $(CC) -print-prog-name=ar) NM := $(shell $(CC) -print-prog-name=nm) -STRIP := $(shell $(CC) -print-prog-name=strip) OBJCOPY := $(shell $(CC) -print-prog-name=objcopy) OBJDUMP := $(shell $(CC) -print-prog-name=objdump) +READELF := $(shell $(CC) -print-prog-name=readelf) OBJSIZE := $(shell $(CC) -print-prog-name=size) +STRIP := $(shell $(CC) -print-prog-name=strip) else -AS = $(CROSS_COMPILE)as LD = $(CROSS_COMPILE)ld AR = $(CROSS_COMPILE)ar NM = $(CROSS_COMPILE)nm -STRIP = $(CROSS_COMPILE)strip OBJCOPY = $(CROSS_COMPILE)objcopy OBJDUMP = $(CROSS_COMPILE)objdump +READELF = $(CROSS_COMPILE)readelf OBJSIZE = $(CROSS_COMPILE)size +STRIP = $(CROSS_COMPILE)strip +endif endif PAHOLE = pahole LEX = flex YACC = bison AWK = awk INSTALLKERNEL := installkernel -DEPMOD = /sbin/depmod +DEPMOD = depmod PERL = perl PYTHON = python -PYTHON2 = python2 PYTHON3 = python3 CHECK = sparse BASH = bash +KGZIP = gzip +KBZIP2 = bzip2 +KLZOP = lzop +LZMA = lzma +LZ4 = lz4c +XZ = xz CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \ -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF) @@ -475,7 +497,7 @@ ifeq ($(call cc-lcc-yn),y) KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -fno-common -fno-PIE \ -Werror=implicit-function-declaration -Werror=implicit-int \ - -Wno-format-security \ + -Werror=return-type -Wno-format-security \ -std=gnu89 else KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \ @@ -498,9 +520,10 @@ KBUILD_LDFLAGS := GCC_PLUGINS_CFLAGS := CLANG_FLAGS := -export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE AS LD CC -export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE PAHOLE LEX YACC AWK INSTALLKERNEL -export PERL PYTHON PYTHON2 PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX +export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC +export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL +export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX +export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS @@ -561,7 +584,7 @@ endif ifneq ($(GCC_TOOLCHAIN),) CLANG_FLAGS += --gcc-toolchain=$(GCC_TOOLCHAIN) endif -ifeq ($(shell $(AS) --version 2>&1 | head -n 1 | grep clang),) +ifneq ($(LLVM_IAS),1) CLANG_FLAGS += -no-integrated-as endif CLANG_FLAGS += -Werror=unknown-warning-option @@ -811,8 +834,11 @@ DEBUG_CFLAGS += -gsplit-dwarf else DEBUG_CFLAGS += -g endif +ifneq ($(LLVM_IAS),1) KBUILD_AFLAGS += -Wa,-gdwarf-2 endif +endif + ifdef CONFIG_DEBUG_INFO_DWARF4 DEBUG_CFLAGS += -gdwarf-4 endif @@ -1018,10 +1044,10 @@ export mod_strip_cmd mod_compress_cmd = true ifdef CONFIG_MODULE_COMPRESS ifdef CONFIG_MODULE_COMPRESS_GZIP - mod_compress_cmd = gzip -n -f + mod_compress_cmd = $(KGZIP) -n -f endif # CONFIG_MODULE_COMPRESS_GZIP ifdef CONFIG_MODULE_COMPRESS_XZ - mod_compress_cmd = xz -f + mod_compress_cmd = $(XZ) -f endif # CONFIG_MODULE_COMPRESS_XZ endif # CONFIG_MODULE_COMPRESS export mod_compress_cmd diff --git a/arch/e2k/Makefile b/arch/e2k/Makefile index 8c2470d..50d58cb 100644 --- a/arch/e2k/Makefile +++ b/arch/e2k/Makefile @@ -17,7 +17,7 @@ LD = $(shell $(CC) -print-prog-name=ld) OBJCOPY = $(shell $(CC) -print-prog-name=objcopy) KBUILD_CFLAGS += -fkernel -gline -masm-inline $(call cc-option,-fforbid-fp) \ - $(call cc-option,-fmax-errors=5) + $(call cc-option,-fmax-errors=5) $(call cc-option,-fno-loop-apb) ifeq ($(PROFILE_GENERATE), 1) KBUILD_CFLAGS += -fprofile-generate-kernel @@ -146,7 +146,6 @@ drivers-$(CONFIG_PM) += arch/e2k/power/ #KVM hypervisor and guest support core-$(CONFIG_KVM) += arch/e2k/kvm/ -core-$(CONFIG_KVM_GUEST) += arch/e2k/kvm/guest/ # Elbrus common modules core-y += arch/l/ diff --git a/arch/e2k/include/asm-l/i2c-spi.h b/arch/e2k/include/asm-l/i2c-spi.h index e3e6cba..105c257 100644 --- a/arch/e2k/include/asm-l/i2c-spi.h +++ b/arch/e2k/include/asm-l/i2c-spi.h @@ -30,9 +30,9 @@ #define I2C_SPI_DATA_AREA_SIZE 0x40 #define I2C_SPI_DEFAULT_IRQ 23 -#define I2C_MAX_ADAPTERS_PER_CONTROLLER 5 -#define I2C_MAX_BUSSES I2C_MAX_ADAPTERS_PER_CONTROLLER +#define I2C_MAX_BUSSES 5 +#define I2C_DST_BUSSES 4 #ifdef CONFIG_E2K extern int iohub_i2c_line_id; diff --git a/arch/e2k/include/asm-l/pci.h b/arch/e2k/include/asm-l/pci.h index 2af6d42..a1eb4fb 100644 --- a/arch/e2k/include/asm-l/pci.h +++ b/arch/e2k/include/asm-l/pci.h @@ -60,22 +60,33 @@ typedef struct iohub_sysdata { int node; /* NUMA node */ int link; /* local number of IO link on the node */ #endif /* CONFIG_IOHUB_DOMAINS */ - u32 pci_msi_addr_lo; /* MSI transaction address */ - u32 pci_msi_addr_hi; /* MSI transaction upper address */ - u8 revision; /* IOHUB revision */ - u8 generation; /* IOHUB generation */ + u32 pci_msi_addr_lo; /* MSI transaction address */ + u32 pci_msi_addr_hi;/* MSI transaction upper address */ + /*IOHUB can be connected to EIOHUB and vice versa */ + bool has_iohub; + u8 iohub_revision; /* IOHUB revision */ + u8 iohub_generation; /* IOHUB generation */ + bool has_eioh; + u8 eioh_generation; /* EIOHUB generation */ + u8 eioh_revision; /* EIOHUB revision */ - struct resource mem_space; + struct resource mem_space; /* pci registers memory */ void *l_iommu; } iohub_sysdata_t; -#define iohub_revision(pdev) ({ \ - struct iohub_sysdata *sd = pdev->bus->sysdata; \ - (sd->revision >> 1); \ +bool l_eioh_device(struct pci_dev *pdev); + +#define iohub_revision(pdev) ({ \ + struct iohub_sysdata *sd = pdev->bus->sysdata; \ + u8 rev = l_eioh_device(pdev) ? sd->eioh_revision : \ + sd->iohub_revision; \ + (rev >> 1); \ }) + #define iohub_generation(pdev) ({ \ struct iohub_sysdata *sd = pdev->bus->sysdata; \ - sd->generation; \ + (l_eioh_device(pdev) ? sd->eioh_generation : \ + sd->iohub_generation); \ }) #ifdef CONFIG_IOHUB_DOMAINS diff --git a/arch/e2k/include/asm/aau_context.h b/arch/e2k/include/asm/aau_context.h index f30e0f5..42714d0 100644 --- a/arch/e2k/include/asm/aau_context.h +++ b/arch/e2k/include/asm/aau_context.h @@ -233,6 +233,13 @@ native_set_aau_context(e2k_aau_t *context) ({ \ regs = native_read_aafstr_reg_value(); \ }) + +static __always_inline void +set_aau_context(e2k_aau_t *context) +{ + native_set_aau_context(context); +} + #endif /* CONFIG_KVM_GUEST_KERNEL */ /* diff --git a/arch/e2k/include/asm/aau_regs_access.h b/arch/e2k/include/asm/aau_regs_access.h index ecd399a..e2b3b5f 100644 --- a/arch/e2k/include/asm/aau_regs_access.h +++ b/arch/e2k/include/asm/aau_regs_access.h @@ -651,6 +651,8 @@ static inline void write_aaldv_reg(e2k_aaldv_t *aaldv) native_write_aaldv_reg(aaldv); } +#define clear_apb() native_clear_apb() + #ifdef CONFIG_USE_AAU # define SAVE_AAU_REGS_FOR_PTRACE(__regs, ti) \ NATIVE_SAVE_AAU_REGS_FOR_PTRACE(__regs, ti) diff --git a/arch/e2k/include/asm/cacheflush.h b/arch/e2k/include/asm/cacheflush.h index 6b8fddc..2b272d6 100644 --- a/arch/e2k/include/asm/cacheflush.h +++ b/arch/e2k/include/asm/cacheflush.h @@ -50,11 +50,12 @@ typedef struct icache_range_array { struct mm_struct *mm; } icache_range_array_t; -extern void __flush_icache_all(void); -extern void native_flush_icache_range(e2k_addr_t start, e2k_addr_t end); -extern void __flush_icache_range_array( - icache_range_array_t *icache_range_arr); -extern void __flush_icache_page(struct vm_area_struct *vma, struct page *page); +extern void native_flush_icache_all(void); +extern void native_flush_icache_range(e2k_addr_t start, e2k_addr_t end); +extern void native_flush_icache_range_array( + icache_range_array_t *icache_range_arr); +extern void native_flush_icache_page(struct vm_area_struct *vma, + struct page *page); #ifndef CONFIG_SMP #define flush_icache_all() __flush_icache_all() @@ -62,18 +63,18 @@ extern void __flush_icache_page(struct vm_area_struct *vma, struct page *page); #define flush_icache_range_array __flush_icache_range_array #define flush_icache_page(vma, page) __flush_icache_page(vma, page) +#define smp_flush_icache_all() #define native_smp_flush_icache_range(start, end) #define native_smp_flush_icache_range_array(icache_range_arr) #define native_smp_flush_icache_page(vma, page) -#define native_smp_flush_icache_all() #define native_smp_flush_icache_kernel_line(addr) #else /* CONFIG_SMP */ +extern void smp_flush_icache_all(void); extern void native_smp_flush_icache_range(e2k_addr_t start, e2k_addr_t end); extern void native_smp_flush_icache_range_array( icache_range_array_t *icache_range_arr); extern void native_smp_flush_icache_page(struct vm_area_struct *vma, struct page *page); -extern void native_smp_flush_icache_all(void); extern void native_smp_flush_icache_kernel_line(e2k_addr_t addr); #define flush_icache_all() smp_flush_icache_all() @@ -176,20 +177,32 @@ smp_flush_icache_page(struct vm_area_struct *vma, struct page *page) native_smp_flush_icache_page(vma, page); } static inline void -smp_flush_icache_all(void) -{ - native_smp_flush_icache_all(); -} -static inline void smp_flush_icache_kernel_line(e2k_addr_t addr) { native_smp_flush_icache_kernel_line(addr); } + +static inline void +__flush_icache_all(void) +{ + native_flush_icache_all(); +} static inline void __flush_icache_range(e2k_addr_t start, e2k_addr_t end) { native_flush_icache_range(start, end); } +static inline void +__flush_icache_range_array(icache_range_array_t *icache_range_arr) +{ + native_flush_icache_range_array(icache_range_arr); +} +static inline void +__flush_icache_page(struct vm_area_struct *vma, struct page *page) +{ + native_flush_icache_page(vma, page); +} + static inline void flush_DCACHE_range(void *addr, size_t len) { diff --git a/arch/e2k/include/asm/compat.h b/arch/e2k/include/asm/compat.h index d48d19a..9bf8d52 100644 --- a/arch/e2k/include/asm/compat.h +++ b/arch/e2k/include/asm/compat.h @@ -208,7 +208,8 @@ struct compat_shmid64_ds { static inline int is_compat_task(void) { - return current->thread.flags & E2K_FLAG_32BIT; + return (TASK_IS_PROTECTED(current)) ? 0 : + (current->thread.flags & E2K_FLAG_32BIT); } #endif /* _ASM_E2K_COMPAT_H */ diff --git a/arch/e2k/include/asm/convert_array.h b/arch/e2k/include/asm/convert_array.h index 2f434f0..0889b1c 100644 --- a/arch/e2k/include/asm/convert_array.h +++ b/arch/e2k/include/asm/convert_array.h @@ -13,44 +13,56 @@ #ifdef CONFIG_PROTECTED_MODE -#define convert_array(prot_array, new_array, max_prot_array_size, fields, \ - items, mask_type, mask_align) \ - convert_array_3(prot_array, new_array, max_prot_array_size, fields, \ - items, mask_type, mask_align, 0, 0) +/* New mask format: 4 bits per structure field */ +#define get_pm_struct_simple(struct128, struct64, \ + max_prot_array_size, fields, \ + items, mask_type, mask_align) \ + get_pm_struct(struct128, struct64, \ + max_prot_array_size, fields, \ + items, mask_type, mask_align, 0, 0) -extern int convert_array_3(long __user *prot_array, long *new_array, - const int max_prot_array_size, const int fields, + +extern int get_pm_struct(long __user *struct128, + long *struct64, + const int max_prot_array_size, const int fieldnum, const int items, const long mask_type, const long mask_align, const long mask_rw, const int rval_mode); /* - * Converts the given array of structures, which can contain + * Converts protected structure (array of structures), which can contain * protected user pointers to memory, function descriptors, and int values. - * prot_array - pointer to the original (user-space) array - * new_array - pointer to area where to put converted array - * max_prot_array_size - the maximum size, which prot_array can occupy - * fileds - number of enries in each element - * items - number of identical elements in the array to convert - * mask_type - mask for encoding of field type in each element: - * 2 bits per each entry: - * --- 00 (0x0) - int - * --- 01 (0x1) - long - * --- 10 (0x2) - pointer to function - * --- 11 (0x3) - pointer to memory (descriptor) - * mask_align - mask for encoding of alignment of the NEXT! field - * 2 bits per each entry: - * --- 00 (0x0) - next field aligned as int (to 4 bytes) - * --- 01 (0x1) - next field aligned as long (to 8 bytes) - * --- 10 (0x2) - not used yet - * --- 11 (0x3) - next field aligned as pointer (to 16 bytes) - * mask_rw - mask for encoding access type of the structure elements - * 2 bits per each entry: - * --- 01 (0x1) - the field's content gets read by syscall (READ-able) - * --- 02 (0x2) - the field's content gets updated by syscall (WRITE-able) - * --- 11 (0x3) - the field is both READ-able and WRITE-able - * --- 00 (0x0) - default type; the same as (READ-able) + * struct128 - pointer to the protected (user-space) structure (128 bit). + * struct64 - pointer to allocated area where to put converted structure. + * max_prot_array_size - estimated maximum size, which struct128 occupies + * filednum - number of fields in the given structure. + * items - number of elements (structures) in array (items == array size) + * if 'struct128' is array of structures to be converted. + * mask_type - mask for encoding structure field types: + * (4 bits per each entry): + * --- 0000 (0x0) - int + * --- 0001 (0x1) - long + * --- 0010 (0x2) - Fptr (pointer to function) + * --- 0011 (0x3) - descriptor (pointer to memory) + * --- 0100 (0x4) - descriptor or int + * --- 0101 (0x5) - descriptor or long + * --- 0110 (0x6) - descriptor or Fptr + * --- 0111 (0x7) - everything is possible (i/P/F) + * --- 1*** (0x8) - may be uninitialized (empty tag allowed) + * mask_align - mask for encoding alignment of the NEXT (!!!) structure field; + * for example, bits #0-3 code alignment of the 2nd structure field + * (4 bits per each entry): + * --- 00 (0x0) - next field aligned as int (to 4 bytes) + * --- 01 (0x1) - next field aligned as long (to 8 bytes) + * --- 10 (0x2) - not used yet + * --- 11 (0x3) - next field aligned as pointer (to 16 bytes) + * mask_rw - mask for encoding access type of structure fields + * (4 bits per each entry): + * --- 01 (0x1) - the field's content gets read by syscall (READ-able) + * --- 02 (0x2) - the field's content gets updated by syscall (WRITE-able) + * --- 11 (0x3) - the field is both READ-able and WRITE-able + * --- 00 (0x0) - default type; the same as (READ-able) * rval_mode - error (return value) reporting mode mask: - * 0 - report only critical problems in prot_array structure; + * 0 - report only critical problems in struct128 structure; * 1 - return with -EFAULT if wrong tag in 'int' field; * 2 - --'-- --'-- 'long' field; * 4 - --'-- --'-- 'func' field; @@ -63,6 +75,7 @@ extern int convert_array_3(long __user *prot_array, long *new_array, * error number - otherwise. */ + #define CONV_ARR_WRONG_INT_FLD 1 #define CONV_ARR_WRONG_LONG_FLD 2 #define CONV_ARR_WRONG_FUNC_FLD 4 @@ -74,13 +87,11 @@ extern int convert_array_3(long __user *prot_array, long *new_array, #define CONV_ARR_IGNORE_DSCR_FLD_ERR 128 -extern int check_args_array(const long __user *args_array, - const long tags, - const int arg_num, - const long mask_type, - const int rval_mode, - const char *ErrMsgHeader); - +extern int check_args_array4(const long __user *args_array, + const long tags, + const int arg_num, + const long mask_type, + const int rval_mode, const char *ErrMsgHeader); /* * This function checks protected syscall arguments on correspondence with * the given mask: @@ -88,11 +99,14 @@ extern int check_args_array(const long __user *args_array, * tags - argument tags (4 bits per arg; lower to higher bits ordered) * arg_num - number of arguments * mask_type - mask for encoding of field type in each element - * 2 bits per each entry: - * --- 00 (0x0) - int - * --- 01 (0x1) - long - * --- 10 (0x2) - pointer to function - * --- 11 (0x3) - pointer to memory. + * 4 bits per each entry: + * --- 0000 (0x0) - int + * --- 0001 (0x1) - long + * --- 0010 (0x2) - pointer to function + * --- 0011 (0x3) - pointer to memory + * --- 0100 (0x4) - descriptor or int + * --- 0101 (0x5) - descriptor or long + * --- 1*** (0x8) - may be uninitialized (empty tag allowed) * rval_mode - error (return value) reporting mode mask: * 0 - report only critical problems; * 1 - return with -EFAULT if wrong tag in 'int' field; @@ -107,6 +121,31 @@ extern int check_args_array(const long __user *args_array, * error number - otherwise. */ + +/* This function realizes compact mask format: 2 bits per structure field */ +extern int convert_array_3(long __user *prot_array, long *new_array, + const int max_prot_array_size, const int fields, + const int items, unsigned long mask_type, + unsigned long mask_align, unsigned long mask_rw, + const int rval_mode); + +/* This function realizes compact mask format: 2 bits per structure field */ +extern int check_args_array(const long __user *args_array, + const long tags, + const int arg_num, + unsigned long mask_type, + const int rval_mode, + const char *ErrMsgHeader); + + +/* This is deprecated. Not reconnemded to use. + * Old mask format: 2 bits per structure field + */ +#define convert_array(prot_array, new_array, max_prot_array_size, fields, \ + items, mask_type, mask_align) \ + convert_array_3(prot_array, new_array, max_prot_array_size, fields, \ + items, mask_type, mask_align, 0, 0) + #else # define convert_array(...) 0 #endif /* CONFIG_PROTECTED_MODE */ diff --git a/arch/e2k/include/asm/cpu_regs_access.h b/arch/e2k/include/asm/cpu_regs_access.h index 81e49e1..83f627c 100644 --- a/arch/e2k/include/asm/cpu_regs_access.h +++ b/arch/e2k/include/asm/cpu_regs_access.h @@ -274,10 +274,14 @@ NATIVE_NV_WRITE_USBR_REG_VALUE(USBR_value) #define WRITE_SBR_REG_VALUE(SBR_value) \ NATIVE_NV_WRITE_SBR_REG_VALUE(SBR_value) +#define NV_WRITE_USBR_USD_REG_VALUE(usbr, usd_hi, usd_lo) \ + NATIVE_NV_WRITE_USBR_USD_REG_VALUE(usbr, usd_hi, usd_lo) #define BOOT_WRITE_USBR_REG_VALUE(USBR_value) \ NATIVE_NV_WRITE_USBR_REG_VALUE(USBR_value) #define BOOT_WRITE_SBR_REG_VALUE(SBR_value) \ NATIVE_NV_WRITE_SBR_REG_VALUE(SBR_value) +#define BOOT_NV_WRITE_USBR_USD_REG_VALUE(usbr, usd_hi, usd_lo) \ + NATIVE_NV_WRITE_USBR_USD_REG_VALUE(usbr, usd_hi, usd_lo) /* * Read/write double-word Window Descriptor Register (WD) diff --git a/arch/e2k/include/asm/cpu_regs_types.h b/arch/e2k/include/asm/cpu_regs_types.h index 398a05e..b287fc0 100644 --- a/arch/e2k/include/asm/cpu_regs_types.h +++ b/arch/e2k/include/asm/cpu_regs_types.h @@ -9,11 +9,11 @@ #ifndef __ASSEMBLY__ /* E2K physical address definitions */ -#define MAX_PA_SIZE 40 /* E2K physical address size */ - /* (bits number) */ -#define MAX_PA_MSB (MAX_PA_SIZE - 1) /* The number of the most */ - /* significant bit of E2K */ - /* physical address */ + +/* E2K physical address size (bits number) */ +#define MAX_PA_SIZE CONFIG_E2K_PA_BITS +/* The number of the most significant bit of E2K physical address */ +#define MAX_PA_MSB (MAX_PA_SIZE - 1) #define MAX_PA_MASK ((1UL << MAX_PA_SIZE) - 1) #define MAX_PM_SIZE (1UL << MAX_PA_SIZE) @@ -2356,23 +2356,23 @@ typedef struct e2k_svd_gregs_struct { } e2k_svd_gregs_t; /* CU_HW0 register */ -#define _CU_HW0_TRWM_ITAG_MASK 0x00000007 /* IB tag */ -#define _CU_HW0_TRWM_IDATA_MASK 0x00000038 /* IB data */ -#define _CU_HW0_TRWM_CF_MASK 0x000001c0 /* Chain File */ -/* Disable IB snooping */ -#define _CU_HW0_IB_SNOOP_DISABLE_MASK 0x00000200 -#define _CU_HW0_BIST_CF_MASK 0x00000400 /* Chain File */ -#define _CU_HW0_BIST_TU_MASK 0x00000800 /* Trap Unit */ -#define _CU_HW0_BIST_ITAG_MASK 0x00001000 /* IB tag */ -#define _CU_HW0_BIST_ITLB_TAG_MASK 0x00002000 /* ITLB tag */ -#define _CU_HW0_BIST_ITLB_DATA_MASK 0x00004000 /* ITLB data */ -#define _CU_HW0_BIST_IDATA_NM_MASK 0x00078000 /* IB data */ -#define _CU_HW0_BIST_IDATA_CNT_MASK 0x1ff80000 /* IB tag */ -#define _CU_HW0_PIPE_FROST_DISABLE_MASK 0x20000000 /* Instruction pipe */ -#define _CU_HW0_RF_CLEAN_DISABLE_MASK 0x40000000 /* Register File */ -/* Disable hardware virtualization support */ -#define _CU_HW0_VIRT_DISABLE_MASK 0x80000000 - +#define _CU_HW0_TRWM_ITAG_MASK 0x000000007 /* IB tag */ +#define _CU_HW0_TRWM_IDATA_MASK 0x000000038 /* IB data */ +#define _CU_HW0_TRWM_CF_MASK 0x0000001c0 /* Chain File */ +#define _CU_HW0_IB_SNOOP_DISABLE_MASK 0x000000200 /* Disable IB snooping */ +#define _CU_HW0_BIST_CF_MASK 0x000000400 /* Chain File */ +#define _CU_HW0_BIST_TU_MASK 0x000000800 /* Trap Unit */ +#define _CU_HW0_BIST_ITAG_MASK 0x000001000 /* IB tag */ +#define _CU_HW0_BIST_ITLB_TAG_MASK 0x000002000 /* ITLB tag */ +#define _CU_HW0_BIST_ITLB_DATA_MASK 0x000004000 /* ITLB data */ +#define _CU_HW0_BIST_IDATA_NM_MASK 0x000078000 /* IB data */ +#define _CU_HW0_BIST_IDATA_CNT_MASK 0x01ff80000 /* IB tag */ +#define _CU_HW0_PIPE_FROST_DISABLE_MASK 0x020000000 /* Instruction pipe */ +#define _CU_HW0_RF_CLEAN_DISABLE_MASK 0x040000000 /* Register File */ +#define _CU_HW0_VIRT_DISABLE_MASK 0x080000000 /* Disable hardware */ + /* virtualization support */ +#define _CU_HW0_UPT_SEC_AD_SHIFT_DSBL_MASK 0x100000000 /* Disable address shift in */ + /* MMU_CR.upt mode */ struct hw_stacks { e2k_psp_lo_t psp_lo; diff --git a/arch/e2k/include/asm/device.h b/arch/e2k/include/asm/device.h index 710ad02..1827e0b 100644 --- a/arch/e2k/include/asm/device.h +++ b/arch/e2k/include/asm/device.h @@ -8,7 +8,12 @@ struct dev_archdata { unsigned int link; - struct e2k_iommu_dev_data iommu; +#ifdef CONFIG_IOMMU_API + void *iommu; /* private IOMMU data */ + struct e2k_iommu_domain *domain; /* Domain the device is bound to */ + struct kvm *kvm; /* Virtual machine, to which device is + * passed */ +#endif }; struct pdev_archdata { diff --git a/arch/e2k/include/asm/e12c.h b/arch/e2k/include/asm/e12c.h index 918c746..d16f7e1 100644 --- a/arch/e2k/include/asm/e12c.h +++ b/arch/e2k/include/asm/e12c.h @@ -47,8 +47,8 @@ extern void setup_APIC_vector_handler(int vector, #define E12C_TLB_ADDR_SET_NUM E2S_TLB_ADDR_SET_NUM #define E12C_TLB_ADDR_SET_NUM_SHIFT E2S_TLB_ADDR_SET_NUM_SHIFT -#define E12C_SIC_MC_COUNT E8C_SIC_MC_COUNT -#define E12C_SIC_MC1_ECC E2S_SIC_MC1_ECC +#define E12C_SIC_MC_SIZE E16C_SIC_MC_SIZE +#define E12C_SIC_MC_COUNT 2 #define E12C_CLOCK_TICK_RATE ES2_CLOCK_TICK_RATE diff --git a/arch/e2k/include/asm/e16c.h b/arch/e2k/include/asm/e16c.h index f27e6eb..a83362f 100644 --- a/arch/e2k/include/asm/e16c.h +++ b/arch/e2k/include/asm/e16c.h @@ -47,8 +47,8 @@ extern void setup_APIC_vector_handler(int vector, #define E16C_TLB_ADDR_SET_NUM E2S_TLB_ADDR_SET_NUM #define E16C_TLB_ADDR_SET_NUM_SHIFT E2S_TLB_ADDR_SET_NUM_SHIFT -#define E16C_SIC_MC_COUNT E8C_SIC_MC_COUNT -#define E16C_SIC_MC1_ECC E2S_SIC_MC1_ECC +#define E16C_SIC_MC_SIZE 0x60 +#define E16C_SIC_MC_COUNT 8 #define E16C_CLOCK_TICK_RATE ES2_CLOCK_TICK_RATE diff --git a/arch/e2k/include/asm/e2c3.h b/arch/e2k/include/asm/e2c3.h index d75a717..d0b1dd1 100644 --- a/arch/e2k/include/asm/e2c3.h +++ b/arch/e2k/include/asm/e2c3.h @@ -47,8 +47,8 @@ extern void setup_APIC_vector_handler(int vector, #define E2C3_TLB_ADDR_SET_NUM E2S_TLB_ADDR_SET_NUM #define E2C3_TLB_ADDR_SET_NUM_SHIFT E2S_TLB_ADDR_SET_NUM_SHIFT -#define E2C3_SIC_MC_COUNT E8C_SIC_MC_COUNT -#define E2C3_SIC_MC1_ECC E2S_SIC_MC1_ECC +#define E2C3_SIC_MC_SIZE E16C_SIC_MC_SIZE +#define E2C3_SIC_MC_COUNT E12C_SIC_MC_COUNT #define E2C3_CLOCK_TICK_RATE ES2_CLOCK_TICK_RATE diff --git a/arch/e2k/include/asm/e2k-iommu.h b/arch/e2k/include/asm/e2k-iommu.h index c54bb90..5d619ec 100644 --- a/arch/e2k/include/asm/e2k-iommu.h +++ b/arch/e2k/include/asm/e2k-iommu.h @@ -1,14 +1,6 @@ #ifndef __ASM_E2K_IOMMU_H #define __ASM_E2K_IOMMU_H -/* - * This struct contains device specific data for the IOMMU - */ -struct e2k_iommu_dev_data { - struct e2k_iommu_domain *domain; /* Domain the device is bound to */ - struct kvm *kvm; /* Virtual machine, to which device is - * passed */ -}; extern int iommu_panic_off; extern void e2k_iommu_error_interrupt(void); diff --git a/arch/e2k/include/asm/e2k_api.h b/arch/e2k/include/asm/e2k_api.h index 0e8e5f6..2b60ca1 100644 --- a/arch/e2k/include/asm/e2k_api.h +++ b/arch/e2k/include/asm/e2k_api.h @@ -1476,43 +1476,6 @@ do { \ #define NATIVE_WRITE_MAS_D(addr, val, mas) \ NATIVE_DO_WRITE_MAS(addr, val, mas, __e2k_u64_t, d, 2) - -/* - * Relaxed IO read/write - * - * bug #81369: put every UC access into a separate - * wide instruction to avoid reorderings possible if - * one access hits in DTLB and another one misses. - */ -#define IO_READ(_addr, type, size_letter) \ -({ \ - type __ior_val; \ - asm ("{ld" #size_letter " %[addr], %[val]}" \ - : [val] "=r" (__ior_val) \ - : [addr] "m" (*((volatile type *) (_addr))) \ - : "memory"); \ - __ior_val; \ -}) - -#define IO_WRITE(_addr, _val, type, size_letter) \ -do { \ - asm ("{st" #size_letter " %[addr], %[val]}" \ - : [addr] "=m" (*((volatile type *) (_addr))) \ - : [val] "r" ((type) (_val)) \ - : "memory"); \ -} while (0) - -#define IO_READ_B(addr) IO_READ((addr), u8, b) -#define IO_READ_H(addr) IO_READ((addr), u16, h) -#define IO_READ_W(addr) IO_READ((addr), u32, w) -#define IO_READ_D(addr) IO_READ((addr), u64, d) - -#define IO_WRITE_B(addr, val) IO_WRITE((addr), (val), u8, b) -#define IO_WRITE_H(addr, val) IO_WRITE((addr), (val), u16, h) -#define IO_WRITE_W(addr, val) IO_WRITE((addr), (val), u32, w) -#define IO_WRITE_D(addr, val) IO_WRITE((addr), (val), u64, d) - - /* * Read from and write to system configuration registers SIC * Now SIC is the same as NBSRs registers @@ -1719,7 +1682,7 @@ do { \ u64 val, val_8; \ u32 __chan = (u32) (_chan); \ u32 __quadro = (u32) (_quadro); \ - u32 __chan_q = (__quadro) ? __chan : -1; \ + u32 __chan_q = (__quadro) ? __chan : 4; /* Not existent channel - skip */ \ u64 __opc = (_opc); \ asm volatile ( \ "{disp %%ctpr1, qpswitchd_sm\n" \ @@ -2103,7 +2066,7 @@ do { \ ({ \ u64 tmp, tmp_ext; \ u32 __chan = (u32) (_chan); \ - u32 __chan_q = (_quadro) ? __chan : -1; \ + u32 __chan_q = (_quadro) ? __chan : 4; /* Not existent channel - skip */ \ asm ( "{nop 1\n" \ " puttagd,2 %[val], %[tag], %[tmp]\n" \ " puttagd,5,sm %[val_ext], %[tag_ext], %[tmp_ext]\n" \ @@ -2120,7 +2083,7 @@ do { \ [val] "r" ((u64) (_val)), [val_ext] "r" ((u64) (_val_ext)), \ [tag] "r" ((u32) (_tag)), [tag_ext] "r" ((u32) (_tag_ext)), \ [opc] "ir" (_opc), [opc_ext] "ir" (_opc_ext), \ - [chan] "r" ((u32) (__chan)), [chan_q] "r" ((u32) (__chan_q)) \ + [chan] "ir" ((u32) (__chan)), [chan_q] "ir" ((u32) (__chan_q)) \ : "memory", "pred20", "pred21", "pred22", "pred23"); \ }) @@ -2250,17 +2213,23 @@ do { \ * quadro: set if this is a non-atomic quadro operation to move 16 bytes * vr: set to 0 if we want to preserve the lower 4-byte word * (same as vr in cellar) + * not_single_byte: set to "false" if we want to write only 1 byte at target + * address (i.e. do not clear the whole register we are + * writing into). This makes sense when we manually split + * the faulting load into a series of 1-byte loads - only + * the first one should clear the register then. */ #define NATIVE_MOVE_TAGGED_DWORD_WITH_OPC_CH_VR(_from, _to, _to_hi, _vr, _opc, \ - _chan, _quadro) \ + _chan, _quadro, _not_single_byte) \ do { \ u64 prev, val, val_8; \ u32 __chan = (u32) (_chan); \ u32 __quadro = (u32) (_quadro); \ - u32 __chan_q = (__quadro) ? __chan : -1; \ + u32 __chan_q = (__quadro) ? __chan : 4 /* Not existent channel - skip */; \ u64 __opc = (_opc); \ asm ( "{cmpesb %[quadro], 0, %%pred18\n" \ - " cmpesb %[vr], 0, %%pred19}\n" \ + " cmpesb %[vr], 0, %%pred19\n" \ + " cmpesb %[not_single_byte], 0, %%pred28}\n" \ "{cmpesb,0 %[chan], 0, %%pred20\n" \ " cmpesb,1 %[chan], 1, %%pred21\n" \ " cmpesb,3 %[chan], 2, %%pred22\n" \ @@ -2280,7 +2249,8 @@ do { \ " ldrd,3 [ %[from] + %[opc_8] ], %[val_8] ? %%pred26\n" \ " ldrd,5 [ %[from] + %[opc_8] ], %[val_8] ? %%pred27}\n" \ "{movts,1 %[prev], %[val] ? %%pred19}\n" \ - "{strd,2 [ %[to] + %[opc_st] ], %[val]\n" \ + "{strd,2 [ %[to] + %[opc_st_byte] ], %[val] ? %%pred28}\n" \ + "{strd,2 [ %[to] + %[opc_st] ], %[val] ? ~%%pred28\n" \ " strd,5 [ %[to_hi] + %[opc_st] ], %[val_8] ? ~ %%pred18}\n" \ : [prev] "=&r" (prev), [val] "=&r" (val), \ [val_8] "=&r" (val_8) \ @@ -2288,10 +2258,13 @@ do { \ [vr] "ir" ((u32) (_vr)), [quadro] "r" (__quadro), \ [chan] "ir" (__chan), [chan_q] "ir" (__chan_q), \ [opc] "r" (__opc), [opc_8] "r" (__opc | 8ull), \ + [not_single_byte] "ir" (_not_single_byte), \ [opc_ld] "i" (TAGGED_MEM_LOAD_REC_OPC), \ + [opc_st_byte] "i" (MEM_STORE_REC_OPC_B), \ [opc_st] "i" (TAGGED_MEM_STORE_REC_OPC) \ - : "memory", "pred18", "pred19", "pred20", "pred21", "pred22", \ - "pred23", "pred24", "pred25", "pred26", "pred27"); \ + : "memory", "pred18", "pred19", "pred20", "pred21", \ + "pred22", "pred23", "pred24", "pred25", "pred26", \ + "pred27", "pred28"); \ } while (false) /* @@ -4938,6 +4911,20 @@ do { \ DO_ATOMIC_WRITE_PSR_REG_VALUE(greg_no, psr_off, psr_value, \ under_upsr_off, under_upsr_bool) \ +#define DO_ATOMIC_WRITE_UPSR_REG_VALUE(greg_no, upsr_off, upsr_value) \ +({ \ + asm volatile ( \ + "{\n\t" \ + " rws %1, %%upsr\n\t" \ + " stw %%dg" #greg_no ", [%0], %1\n\t" \ + "}" \ + : \ + : "ri" ((__e2k_u64_t)(upsr_off)), \ + "r" ((__e2k_u32_t)(upsr_value))); \ +}) +#define KVM_DO_ATOMIC_WRITE_UPSR_REG_VALUE(greg_no, upsr_off, upsr_value) \ + DO_ATOMIC_WRITE_UPSR_REG_VALUE(greg_no, upsr_off, upsr_value) + #define NATIVE_GET_TCD() \ ({ \ register __e2k_u64_t res; \ @@ -4965,14 +4952,26 @@ do { \ /* Add ctpr3 to clobbers to explain to lcc that this * GNU asm does a return. */ -#define E2K_DONE \ +#define E2K_DONE() \ do { \ /* #80747: must repeat interrupted barriers */ \ asm volatile ("{nop 3; wait st_c=1} {done}" ::: "ctpr3"); \ } while (0) -#define E2K_SYSCALL_RETURN E2K_RETURN -#define E2K_RETURN(rval) \ +#define NATIVE_RETURN() \ +do { \ + asm volatile( "{\n" \ + "return %%ctpr3\n" \ + "}\n" \ + "{\n" \ + "ct %%ctpr3\n" \ + "}\n" \ + : \ + : \ + : "ctpr3"); \ +} while (0) + +#define NATIVE_RETURN_VALUE(rval) \ do { \ asm volatile( "{\n" \ "return %%ctpr3\n" \ @@ -4981,10 +4980,13 @@ do { \ "{\n" \ "ct %%ctpr3\n" \ "}\n" \ - :: [r0] "ir" (rval) \ + : \ + : [r0] "ir" (rval) \ : "ctpr3"); \ } while (0) +#define E2K_SYSCALL_RETURN NATIVE_RETURN_VALUE + #define E2K_EMPTY_CMD(input...) \ do { \ asm volatile ("{nop}" :: input); \ @@ -5517,6 +5519,24 @@ _Pragma("no_asm_inline") \ "ri" ((__e2k_u64_t) (arg3)) \ ); \ } while (false) +#define E2K_GOTO_ARG4(label, arg1, arg2, arg3, arg4) \ +do { \ +_Pragma("no_asm_inline") \ + asm volatile ("\n" \ + "{\n" \ + "addd \t 0, %0, %%dr0\n" \ + "addd \t 0, %1, %%dr1\n" \ + "addd \t 0, %2, %%dr2\n" \ + "addd \t 0, %3, %%dr3\n" \ + "ibranch \t" #label "\n" \ + "}\n" \ + : \ + : "ri" ((__e2k_u64_t) (arg1)), \ + "ri" ((__e2k_u64_t) (arg2)), \ + "ri" ((__e2k_u64_t) (arg3)), \ + "ri" ((__e2k_u64_t) (arg4)) \ + ); \ +} while (false) #define E2K_GOTO_AND_RETURN_ARG6(label, \ arg1, arg2, arg3, arg4, arg5, arg6) \ do { \ @@ -6134,6 +6154,199 @@ do { \ __dres; \ }) +#define SIMPLE_RECOVERY_STORE(_addr, _data, _opc) \ +do { \ + u32 _fmt = ((ldst_rec_op_t *) &_opc)->fmt; \ + u32 _ind = ((ldst_rec_op_t *) &_opc)->index; \ + asm ( \ + "{nop 1\n" \ + " cmpesb,0 %[fmt], 1, %%pred20\n" \ + " cmpesb,1 %[fmt], 2, %%pred21\n" \ + " cmpesb,3 %[fmt], 3, %%pred22\n" \ + " cmpesb,4 %[fmt], 4, %%pred23}\n" \ + "{stb,2 %[addr], %[ind], %[data] ? %%pred20\n" \ + " sth,5 %[addr], %[ind], %[data] ? %%pred21}\n" \ + "{stw,2 %[addr], %[ind], %[data] ? %%pred22\n" \ + " std,5 %[addr], %[ind], %[data] ? %%pred23}\n" \ + : \ + : [addr] "r" (_addr), [data] "r" (_data), \ + [fmt] "r" (_fmt), [ind] "r" (_ind) \ + : "memory", "pred20", "pred21", "pred22", "pred23" \ + ); \ +} while (0) + +#define SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, _greg_no, _sm, _mas) \ +do { \ + u32 _fmt = ((ldst_rec_op_t *) &_opc)->fmt; \ + u32 _ind = ((ldst_rec_op_t *) &_opc)->index; \ + asm ( \ + "{nop 1\n" \ + " cmpesb,0 %[fmt], 1, %%pred20\n" \ + " cmpesb,1 %[fmt], 2, %%pred21\n" \ + " cmpesb,3 %[fmt], 3, %%pred22\n" \ + " cmpesb,4 %[fmt], 4, %%pred23}\n" \ + "{nop 4\n" \ + " ldb" _sm ",0 %[addr], %[ind], %%dg" #_greg_no ", " \ + "mas=%[mas] ? %%pred20\n" \ + " ldh" _sm ",2 %[addr], %[ind], %%dg" #_greg_no ", " \ + "mas=%[mas] ? %%pred21\n" \ + " ldw" _sm ",3 %[addr], %[ind], %%dg" #_greg_no ", " \ + "mas=%[mas] ? %%pred22\n" \ + " ldd" _sm ",5 %[addr], %[ind], %%dg" #_greg_no ", " \ + "mas=%[mas] ? %%pred23}\n" \ + : \ + : [addr] "r" (_addr), [fmt] "r" (_fmt), \ + [ind] "r" (_ind), [mas] "r" (_mas) \ + : "memory", "pred20", "pred21", "pred22", "pred23", \ + "g" #_greg_no \ + ); \ +} while (0) + +#define SIMPLE_RECOVERY_LOAD_TO_GREG(_addr, _opc, _greg_num, _sm, _mas) \ +do { \ + switch (_greg_num) { \ + case 0: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 0, _sm, _mas); \ + break; \ + case 1: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 1, _sm, _mas); \ + break; \ + case 2: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 2, _sm, _mas); \ + break; \ + case 3: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 3, _sm, _mas); \ + break; \ + case 4: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 4, _sm, _mas); \ + break; \ + case 5: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 5, _sm, _mas); \ + break; \ + case 6: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 6, _sm, _mas); \ + break; \ + case 7: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 7, _sm, _mas); \ + break; \ + case 8: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 8, _sm, _mas); \ + break; \ + case 9: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 9, _sm, _mas); \ + break; \ + case 10: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 10, _sm, _mas); \ + break; \ + case 11: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 11, _sm, _mas); \ + break; \ + case 12: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 12, _sm, _mas); \ + break; \ + case 13: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 13, _sm, _mas); \ + break; \ + case 14: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 14, _sm, _mas); \ + break; \ + case 15: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 15, _sm, _mas); \ + break; \ + /* Do not load g16-g19 as they are used by kernel */ \ + case 16: \ + case 17: \ + case 18: \ + case 19: \ + break; \ + case 20: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 20, _sm, _mas); \ + break; \ + case 21: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 21, _sm, _mas); \ + break; \ + case 22: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 22, _sm, _mas); \ + break; \ + case 23: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 23, _sm, _mas); \ + break; \ + case 24: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 24, _sm, _mas); \ + break; \ + case 25: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 25, _sm, _mas); \ + break; \ + case 26: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 26, _sm, _mas); \ + break; \ + case 27: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 27, _sm, _mas); \ + break; \ + case 28: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 28, _sm, _mas); \ + break; \ + case 29: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 29, _sm, _mas); \ + break; \ + case 30: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 30, _sm, _mas); \ + break; \ + case 31: \ + SIMPLE_RECOVERY_LOAD_TO_GREG_NO(_addr, _opc, 31, _sm, _mas); \ + break; \ + default: \ + panic("Invalid global register # %d\n", _greg_num); \ + } \ +} while (0) + +#define SIMPLE_RECOVERY_MOVE(_from, _to, _opc, _first_time, _sm, _mas) \ +do { \ + u64 _data; \ + u32 _fmt = ((ldst_rec_op_t *) &_opc)->fmt; \ + u32 _ind = ((ldst_rec_op_t *) &_opc)->index; \ + asm ( \ + "{nop 1\n" \ + " cmpesb,0 %[fmt], 1, %%pred20\n" \ + " cmpesb,1 %[fmt], 2, %%pred21\n" \ + " cmpesb,3 %[fmt], 3, %%pred22\n" \ + " cmpesb,4 %[fmt], 4, %%pred23}\n" \ + "{nop 4\n" \ + " ldb" _sm ",0 %[from], %[ind], %[data], " \ + "mas=%[mas] ? %%pred20\n" \ + " ldh" _sm ",2 %[from], %[ind], %[data], " \ + "mas=%[mas] ? %%pred21\n" \ + " ldw" _sm ",3 %[from], %[ind], %[data], " \ + "mas=%[mas] ? %%pred22\n" \ + " ldd" _sm ",5 %[from], %[ind], %[data], " \ + "mas=%[mas] ? %%pred23}\n" \ + "{cmpesb,0 %[first_time], 0, %%pred19}\n" \ + "{pass %%pred19, @p0\n" \ + " pass %%pred20, @p1\n" \ + " pass %%pred21, @p2\n" \ + " pass %%pred22, @p3\n" \ + " landp @p0, @p1, @p4\n" \ + " pass @p4, %%pred20\n" \ + " landp @p0, @p2, @p5\n" \ + " pass @p5, %%pred21\n" \ + " landp @p0, @p3, @p6\n" \ + " pass @p6, %%pred22}\n" \ + "{pass %%pred19, @p0\n" \ + " pass %%pred23, @p1\n" \ + " landp @p0, ~@p1, @p4\n" \ + " pass @p4, %%pred23}\n" \ + "{stb,sm,2 %[to], 0, %[data] ? %%pred20\n" \ + " sth,sm,5 %[to], 0, %[data] ? %%pred21}\n" \ + "{stw,sm,2 %[to], 0, %[data] ? %%pred22\n" \ + " std,sm,5 %[to], 0, %[data] ? ~%%pred23}\n" \ + : [data] "=&r" (_data) \ + : [from] "r" (_from), [to] "r" (_to), \ + [fmt] "r" (_fmt), [ind] "r" (_ind), \ + [first_time] "r" (_first_time), [mas] "r" (_mas) \ + : "memory", "pred19", "pred20", "pred21", "pred22", "pred23" \ + ); \ +} while (0) + /* Since v6 this got replaced with "wait int=1,mem_mod=1" */ #define C1_WAIT_TRAP_V3() \ do { \ @@ -6204,7 +6417,7 @@ do { \ [addr_flush_icache] "r" ((u64) (_FLUSH_ICACHE_ALL_OP)), \ [val_icache] "r" (0ULL), \ [mas_icache] "i" (MAS_ICACHE_FLUSH), \ - [addr_flush_tlb] "r" ((u64) (_FLUSH_ICACHE_ALL_OP)), \ + [addr_flush_tlb] "r" ((u64) (_FLUSH_TLB_ALL_OP)), \ [val_tlb] "r" (0ULL), \ [mas_tlb] "i" (MAS_TLB_FLUSH), \ [mas_ioaddr] "i" (MAS_IOADDR), \ @@ -6266,7 +6479,7 @@ do { \ [addr_flush_icache] "r" ((u64) (_FLUSH_ICACHE_ALL_OP)), \ [val_icache] "r" (0ULL), \ [mas_icache] "i" (MAS_ICACHE_FLUSH), \ - [addr_flush_tlb] "r" ((u64) (_FLUSH_ICACHE_ALL_OP)), \ + [addr_flush_tlb] "r" ((u64) (_FLUSH_TLB_ALL_OP)), \ [val_tlb] "r" (0ULL), \ [mas_tlb] "i" (MAS_TLB_FLUSH), \ [mas_ioaddr] "i" (MAS_IOADDR) \ diff --git a/arch/e2k/include/asm/e2k_debug.h b/arch/e2k/include/asm/e2k_debug.h index ce10667..fc1bb8c 100644 --- a/arch/e2k/include/asm/e2k_debug.h +++ b/arch/e2k/include/asm/e2k_debug.h @@ -25,6 +25,14 @@ #define CHK_DEBUGGER(trapnr, signr, error_code, address, regs, after) +#define IS_KERNEL_THREAD(task, mm) \ +({ \ + e2k_addr_t ps_base; \ + \ + ps_base = (e2k_addr_t)task_thread_info(task)->u_hw_stack.ps.base; \ + ((mm) == NULL || ps_base >= TASK_SIZE); \ +}) + extern void print_stack_frames(struct task_struct *task, struct pt_regs *pt_regs, int show_reg_window) __cold; extern void print_mmap(struct task_struct *task) __cold; @@ -190,6 +198,18 @@ typedef struct stack_regs { extern void print_chain_stack(struct stack_regs *regs, int show_reg_window); +extern void copy_stack_regs(struct task_struct *task, + const struct pt_regs *limit_regs, struct stack_regs *regs); +extern int parse_chain_stack(int flags, struct task_struct *p, + parse_chain_fn_t func, void *arg); + +extern struct stack_regs stack_regs_cache[NR_CPUS]; +extern int debug_userstack; +extern int print_window_regs; + +#ifdef CONFIG_DATA_STACK_WINDOW +extern int debug_datastack; +#endif #ifndef CONFIG_VIRTUALIZATION /* it is native kernel without any virtualization */ diff --git a/arch/e2k/include/asm/fast_syscalls.h b/arch/e2k/include/asm/fast_syscalls.h index 9dcab84..3995c77 100644 --- a/arch/e2k/include/asm/fast_syscalls.h +++ b/arch/e2k/include/asm/fast_syscalls.h @@ -203,15 +203,23 @@ DO_FAST_CLOCK_GETTIME(const clockid_t which_clock, struct timespec *tp) /* trap table entry is called as function (it is closer to hardware start) */ typedef long (*ttable_entry_args3)(int sys_num, u64 arg1, u64 arg2); +typedef long (*ttable_entry_args4)(int sys_num, u64 arg1, u64 arg2, u64 arg3); #define ttable_entry3_args3(sys_num, arg1, arg2) \ ((ttable_entry_args3)(get_ttable_entry3))(sys_num, arg1, arg2) +#define ttable_entry3_args4(sys_num, arg1, arg2) \ + ((ttable_entry_args4)(get_ttable_entry3))(sys_num, arg1, arg2, arg3) /* trap table entry started by direct branch (it is closer to fast system */ /* call wirthout switch and use user local data stack */ #define goto_ttable_entry_args3(entry_label, sys_num, arg1, arg2) \ E2K_GOTO_ARG3(entry_label, sys_num, arg1, arg2) +#define goto_ttable_entry_args4(entry_label, sys_num, arg1, arg2, arg3) \ + E2K_GOTO_ARG4(entry_label, sys_num, arg1, arg2, arg3) #define goto_ttable_entry3_args3(sys_num, arg1, arg2) \ goto_ttable_entry_args3(ttable_entry3, sys_num, arg1, arg2) +#define goto_ttable_entry3_args4(sys_num, arg1, arg2, arg3) \ + goto_ttable_entry_args4(ttable_entry3, sys_num, arg1, arg2, arg3) + #define ttable_entry_clock_gettime(which, time) \ /* ibranch */ goto_ttable_entry3_args3(__NR_clock_gettime, which, time) @@ -219,6 +227,12 @@ typedef long (*ttable_entry_args3)(int sys_num, u64 arg1, u64 arg2); #define ttable_entry_gettimeofday(tv, tz) \ /* ibranch */ goto_ttable_entry3_args3(__NR_gettimeofday, tv, tz) /* ttable_entry3_args3(__NR_gettimeofday, tv, tz) */ +#define ttable_entry_sigprocmask(how, nset, oset) \ +/* ibranch */ goto_ttable_entry3_args4(__NR_sigprocmask, how, nset, oset) +/* ttable_entry3_args4(__NR_sigprocmask, how, nset, oset) */ +#define ttable_entry_getcpu(cpup, nodep, unused) \ +/* ibranch */ goto_ttable_entry3_args4(__NR_getcpu, cpup, nodep, unused) +/* ttable_entry3_args4(__NR_getcpu, cpup, nodep, unused) */ static inline int FAST_SYS_CLOCK_GETTIME(const clockid_t which_clock, struct timespec __user *tp) @@ -228,14 +242,19 @@ FAST_SYS_CLOCK_GETTIME(const clockid_t which_clock, struct timespec __user *tp) prefetchw(&fsys_data); +#ifdef CONFIG_KVM_HOST_MODE + if (unlikely(test_ti_status_flag(ti, TS_HOST_AT_VCPU_MODE))) + ttable_entry_clock_gettime((u64) which_clock, (u64) tp); +#endif + tp = (typeof(tp)) ((u64) tp & E2K_VA_MASK); if (unlikely((u64) tp + sizeof(struct timespec) > ti->addr_limit.seg)) return -EFAULT; r = do_fast_clock_gettime(which_clock, tp); if (unlikely(r)) -/* ibranch */ ttable_entry_clock_gettime((u64) which_clock, (u64) tp); -/* call r = ttable_entry_clock_gettime((u64) which_clock, (u64) tp); */ + ttable_entry_clock_gettime((u64) which_clock, (u64) tp); + return r; } @@ -259,6 +278,9 @@ FAST_SYS_SIGGETMASK(u64 __user *oset, size_t sigsetsize) { struct thread_info *const ti = READ_CURRENT_REG(); struct task_struct *task = thread_info_task(ti); +#ifdef CONFIG_KVM_HOST_MODE + bool guest = test_ti_status_flag(ti, TS_HOST_AT_VCPU_MODE); +#endif u64 set; set = task->blocked.sig[0]; @@ -266,6 +288,11 @@ FAST_SYS_SIGGETMASK(u64 __user *oset, size_t sigsetsize) if (unlikely(sigsetsize != 8)) return -EINVAL; +#ifdef CONFIG_KVM_HOST_MODE + if (unlikely(guest)) + ttable_entry_sigprocmask((u64) 0, (u64) NULL, (u64) oset); +#endif + oset = (typeof(oset)) ((u64) oset & E2K_VA_MASK); if (unlikely((u64) oset + sizeof(sigset_t) > ti->addr_limit.seg)) return -EFAULT; diff --git a/arch/e2k/include/asm/io.h b/arch/e2k/include/asm/io.h index 0c6de82..bc59709 100644 --- a/arch/e2k/include/asm/io.h +++ b/arch/e2k/include/asm/io.h @@ -38,7 +38,7 @@ extern int __init native_arch_pci_init(void); static inline u8 native_readb_relaxed(const volatile void __iomem *addr) { - u8 res = IO_READ_B(addr); + u8 res = *(const volatile u8 __force *) addr; if (cpu_has(CPU_HWBUG_PIO_READS)) __E2K_WAIT(_ld_c); return res; @@ -46,7 +46,7 @@ static inline u8 native_readb_relaxed(const volatile void __iomem *addr) static inline u16 native_readw_relaxed(const volatile void __iomem *addr) { - u16 res = IO_READ_H(addr); + u16 res = *(const volatile u16 __force *) addr; if (cpu_has(CPU_HWBUG_PIO_READS)) __E2K_WAIT(_ld_c); return res; @@ -54,7 +54,7 @@ static inline u16 native_readw_relaxed(const volatile void __iomem *addr) static inline u32 native_readl_relaxed(const volatile void __iomem *addr) { - u32 res = IO_READ_W(addr); + u32 res = *(const volatile u32 __force *) addr; if (cpu_has(CPU_HWBUG_PIO_READS)) __E2K_WAIT(_ld_c); return res; @@ -62,7 +62,7 @@ static inline u32 native_readl_relaxed(const volatile void __iomem *addr) static inline u64 native_readq_relaxed(const volatile void __iomem *addr) { - u64 res = IO_READ_D(addr); + u64 res = *(const volatile u64 __force *) addr; if (cpu_has(CPU_HWBUG_PIO_READS)) __E2K_WAIT(_ld_c); return res; @@ -70,23 +70,22 @@ static inline u64 native_readq_relaxed(const volatile void __iomem *addr) static inline void native_writeb_relaxed(u8 value, volatile void __iomem *addr) { - IO_WRITE_B(addr, value); + *(volatile u8 __force *) addr = value; } static inline void native_writew_relaxed(u16 value, volatile void __iomem *addr) { - IO_WRITE_H(addr, value); *(volatile u16 __force *) addr = value; } static inline void native_writel_relaxed(u32 value, volatile void __iomem *addr) { - IO_WRITE_W(addr, value); + *(volatile u32 __force *) addr = value; } static inline void native_writeq_relaxed(u64 value, volatile void __iomem *addr) { - IO_WRITE_D(addr, value); + *(volatile u64 __force *) addr = value; } diff --git a/arch/e2k/include/asm/irqflags.h b/arch/e2k/include/asm/irqflags.h index 363b17c..fa21595 100644 --- a/arch/e2k/include/asm/irqflags.h +++ b/arch/e2k/include/asm/irqflags.h @@ -269,8 +269,6 @@ do { \ #define native_psr_irqs_disabled() \ psr_irqs_disabled_flags(NATIVE_NV_READ_PSR_REG_VALUE()) -#define native_trap_irqs_disabled(regs) (regs->irqs_disabled) - #define psr_and_upsr_nm_irqs_disabled() \ ({ \ int ret; \ diff --git a/arch/e2k/include/asm/kvm/aau_regs_access.h b/arch/e2k/include/asm/kvm/aau_regs_access.h index 62ac478..01a3a67 100644 --- a/arch/e2k/include/asm/kvm/aau_regs_access.h +++ b/arch/e2k/include/asm/kvm/aau_regs_access.h @@ -612,6 +612,8 @@ kvm_write_aads_4_reg(int AADs_no, e2k_aadj_t *mem_p) KVM_SET_AAU_4_AADs(AADs_no, mem_p); } +#define kvm_clear_apb() /* AAU context should restore host */ + #ifdef CONFIG_KVM_GUEST_KERNEL /* It is pure kvm kernel without paravirtualization */ @@ -658,6 +660,8 @@ static inline void write_aaldv_reg(e2k_aaldv_t *aaldv) kvm_write_aaldm_reg_value(aaldv->lo, aaldv->hi); } +#define clear_apb() kvm_clear_apb() + #endif /* CONFIG_KVM_GUEST_KERNEL */ #endif /* _KVM_AAU_REGS_ACCESS_H_ */ diff --git a/arch/e2k/include/asm/kvm/cpu_regs_access.h b/arch/e2k/include/asm/kvm/cpu_regs_access.h index 1062baf..2c82ad3 100644 --- a/arch/e2k/include/asm/kvm/cpu_regs_access.h +++ b/arch/e2k/include/asm/kvm/cpu_regs_access.h @@ -53,6 +53,9 @@ (offsetof(kvm_cpu_regs_t, CPU_TIRs)) + \ (sizeof(e2k_tir_t) * TIR_no) + \ (offsetof(e2k_tir_t, TIR_hi))) +#define GUEST_CPU_SBBP(SBBP_no) (GUEST_CPU_SREGS_BASE + \ + (offsetof(kvm_cpu_regs_t, CPU_SBBP)) + \ + (sizeof(u64) * SBBP_no)) #define GUEST_GET_CPU_SREG(reg_name) \ E2K_LOAD_GUEST_VCPU_STATE_W(GUEST_CPU_SREG(reg_name)) #define GUEST_GET_CPU_DSREG(reg_name) \ @@ -65,6 +68,8 @@ E2K_LOAD_GUEST_VCPU_STATE_D(GUEST_CPU_TIR_lo(TIR_no)) #define GUEST_GET_CPU_TIR_hi(TIR_no) \ E2K_LOAD_GUEST_VCPU_STATE_D(GUEST_CPU_TIR_hi(TIR_no)) +#define GUEST_GET_CPU_SBBP(SBBP_no) \ + E2K_LOAD_GUEST_VCPU_STATE_D(GUEST_CPU_SBBP(SBBP_no)) #define GUEST_IRQS_UNDER_UPSR() \ offsetof(kvm_vcpu_state_t, irqs_under_upsr) @@ -750,6 +755,11 @@ #define KVM_WRITE_TIR_HI_REG_VALUE(TIR_hi_value) \ KVM_WRITE_TIRs_num(-1) +/* + * Read double-word Stcak of Base Blocks Pointers (SBBP) + */ +#define KVM_READ_SBBP_REG_VALUE(no) GUEST_GET_CPU_SBBP(no) + /* * Read/write virtual deferred traps register - DTR */ @@ -1034,7 +1044,6 @@ */ #define KVM_READ_RPR_LO_REG_VALUE() NATIVE_GET_DSREG_OPEN(rpr.lo) #define KVM_READ_RPR_HI_REG_VALUE() NATIVE_GET_DSREG_OPEN(rpr.hi) -#define KVM_READ_SBBP_REG_VALUE() NATIVE_GET_DSREG_OPEN(sbbp) #define KVM_WRITE_RPR_LO_REG_VALUE(RPR_lo_value) \ NATIVE_SET_DSREG_OPEN(rpr.lo, RPR_lo_value) @@ -1092,7 +1101,7 @@ #define KVM_READ_PSR_REG_VALUE() \ ({ \ extern void dump_stack(void); \ - unsigned long PSR_value = GUEST_GET_CPU_SREG(PSR); \ + unsigned long PSR_value = GUEST_GET_CPU_SREG(E2K_PSR); \ unsigned long vcpu_base; \ \ KVM_GET_VCPU_STATE_BASE(vcpu_base); \ @@ -1110,14 +1119,24 @@ extern void dump_stack(void); \ if (BOOT_IS_HV_GM()) \ PSR_value = NATIVE_NV_READ_PSR_REG_VALUE(); \ else \ - PSR_value = GUEST_GET_CPU_SREG(PSR); \ + PSR_value = GUEST_GET_CPU_SREG(E2K_PSR); \ PSR_value; \ }) #define KVM_ATOMIC_WRITE_PSR_REG_VALUE(PSR_value, under_upsr) \ - KVM_DO_ATOMIC_WRITE_PSR_REG_VALUE(GUEST_VCPU_STATE_GREG, \ - GUEST_CPU_SREG(PSR), PSR_value, \ - GUEST_IRQS_UNDER_UPSR(), under_upsr) +({ \ + KVM_DO_ATOMIC_WRITE_PSR_REG_VALUE(GUEST_VCPU_STATE_GREG, \ + GUEST_CPU_SREG(E2K_PSR), PSR_value, \ + GUEST_IRQS_UNDER_UPSR(), under_upsr); \ + trace_vcpu_psr_update(PSR_value, under_upsr); \ +}) + +#define BOOT_KVM_ATOMIC_WRITE_PSR_REG_VALUE(PSR_value, under_upsr) \ +({ \ + KVM_DO_ATOMIC_WRITE_PSR_REG_VALUE(GUEST_VCPU_STATE_GREG, \ + GUEST_CPU_SREG(E2K_PSR), PSR_value, \ + GUEST_IRQS_UNDER_UPSR(), under_upsr); \ +}) #define KVM_WRITE_SW_PSR_REG_VALUE(PSR_value) \ ({ \ @@ -1133,6 +1152,21 @@ extern void dump_stack(void); \ under_upsr = false; \ KVM_ATOMIC_WRITE_PSR_REG_VALUE(PSR_value, under_upsr); \ }) + +#define BOOT_KVM_WRITE_SW_PSR_REG_VALUE(PSR_value) \ +({ \ + kvm_vcpu_state_t *vcpu_state; \ + bool under_upsr; \ + \ + KVM_GET_VCPU_STATE_BASE(vcpu_state); \ + under_upsr = vcpu_state->irqs_under_upsr; \ + if (((PSR_value) & (PSR_IE | PSR_NMIE | PSR_UIE | PSR_UNMIE)) == \ + (PSR_IE | PSR_NMIE | PSR_UIE | PSR_UNMIE)) \ + under_upsr = true; \ + if (((PSR_value) & (PSR_IE | PSR_NMIE | PSR_UIE | PSR_UNMIE)) == 0) \ + under_upsr = false; \ + BOOT_KVM_ATOMIC_WRITE_PSR_REG_VALUE(PSR_value, under_upsr); \ +}) #define KVM_WRITE_PSR_REG_VALUE(PSR_value) \ ({ \ KVM_WRITE_SW_PSR_REG_VALUE(PSR_value); \ @@ -1141,7 +1175,7 @@ extern void dump_stack(void); \ }) #define BOOT_KVM_WRITE_PSR_REG_VALUE(PSR_value) \ ({ \ - KVM_WRITE_SW_PSR_REG_VALUE(PSR_value); \ + BOOT_KVM_WRITE_SW_PSR_REG_VALUE(PSR_value); \ if (BOOT_IS_HV_GM()) \ NATIVE_WRITE_PSR_REG_VALUE(PSR_value); \ }) @@ -1172,6 +1206,19 @@ extern void dump_stack(void); \ UPSR_value = GUEST_GET_CPU_SREG(UPSR); \ UPSR_value; \ }) + +#define KVM_ATOMIC_WRITE_UPSR_REG_VALUE(UPSR_value) \ +({ \ + KVM_DO_ATOMIC_WRITE_UPSR_REG_VALUE(GUEST_VCPU_STATE_GREG, \ + GUEST_CPU_SREG(UPSR), UPSR_value); \ +}) + +#define BOOT_KVM_ATOMIC_WRITE_UPSR_REG_VALUE(UPSR_value) \ +({ \ + KVM_DO_ATOMIC_WRITE_UPSR_REG_VALUE(GUEST_VCPU_STATE_GREG, \ + GUEST_CPU_SREG(UPSR), UPSR_value); \ +}) + #if defined(CONFIG_DIRECT_VIRQ_INJECTION) #define KVM_WRITE_UPSR_REG_VALUE(UPSR_value) \ ({ \ @@ -1180,11 +1227,14 @@ extern void dump_stack(void); \ \ KVM_GET_VCPU_STATE_BASE(vcpu_state); \ under_upsr = vcpu_state->irqs_under_upsr; \ - GUEST_SET_CPU_SREG(UPSR, UPSR_value); \ - NATIVE_WRITE_UPSR_REG_VALUE(UPSR_value); \ + KVM_ATOMIC_WRITE_UPSR_REG_VALUE(UPSR_value); \ if (under_upsr && vcpu_state->lapic.virqs_num.counter) { \ - if ((UPSR_value) & UPSR_IE) \ - HYPERVISOR_inject_interrupt(); \ + if ((UPSR_value) & UPSR_IE) { \ + trace_vcpu_upsr_update(UPSR_value, true); \ + kvm_hypervisor_inject_interrupt(); \ + } \ + } else { \ + trace_vcpu_upsr_update(UPSR_value, false); \ } \ }) #define BOOT_KVM_WRITE_UPSR_REG_VALUE(UPSR_value) \ @@ -1194,25 +1244,13 @@ extern void dump_stack(void); \ \ KVM_GET_VCPU_STATE_BASE(vcpu_state); \ under_upsr = vcpu_state->irqs_under_upsr; \ - GUEST_SET_CPU_SREG(UPSR, UPSR_value); \ - NATIVE_WRITE_UPSR_REG_VALUE(UPSR_value); \ + BOOT_KVM_ATOMIC_WRITE_UPSR_REG_VALUE(UPSR_value); \ if (under_upsr && vcpu_state->lapic.virqs_num.counter) { \ if ((UPSR_value) & UPSR_IE) \ HYPERVISOR_inject_interrupt(); \ } \ }) -#elif defined(CONFIG_VIRQ_VCPU_INJECTION) -#define KVM_WRITE_UPSR_REG_VALUE(UPSR_value) \ -({ \ - GUEST_SET_CPU_SREG(UPSR, UPSR_value); \ - NATIVE_WRITE_UPSR_REG_VALUE(UPSR_value); \ -}) -#define BOOT_KVM_WRITE_UPSR_REG_VALUE(UPSR_value) \ -({ \ - GUEST_SET_CPU_SREG(UPSR, UPSR_value); \ - NATIVE_WRITE_UPSR_REG_VALUE(UPSR_value); \ -}) -#else /* ! CONFIG_DIRECT_VIRQ_INJECTION && ! CONFIG_VIRQ_VCPU_INJECTION */ +#else /* ! CONFIG_DIRECT_VIRQ_INJECTION */ #define KVM_WRITE_UPSR_REG_VALUE(UPSR_value) #define BOOT_KVM_WRITE_UPSR_REG_VALUE(UPSR_value) #endif /* CONFIG_DIRECT_VIRQ_INJECTION */ @@ -1488,10 +1526,14 @@ extern void dump_stack(void); \ KVM_WRITE_SBR_REG_VALUE(SBR_value) #define WRITE_USBR_REG_VALUE(USBR_value) \ KVM_WRITE_USBR_REG_VALUE(USBR_value) +#define NV_WRITE_USBR_USD_REG_VALUE(usbr, usd_hi, usd_lo) \ + KVM_NV_WRITE_USBR_USD_REG_VALUE(usbr, usd_hi, usd_lo) #define BOOT_WRITE_USBR_REG_VALUE(USBR_value) \ KVM_WRITE_USBR_REG_VALUE(USBR_value) #define BOOT_WRITE_SBR_REG_VALUE(SBR_value) \ KVM_WRITE_SBR_REG_VALUE(SBR_value) +#define BOOT_NV_WRITE_USBR_USD_REG_VALUE(usbr, usd_hi, usd_lo) \ + KVM_NV_WRITE_USBR_USD_REG_VALUE(usbr, usd_hi, usd_lo) /* * Read/write double-word Window Descriptor Register (WD) diff --git a/arch/e2k/include/asm/kvm/debug.h b/arch/e2k/include/asm/kvm/debug.h index a337d98..d6cce8b 100644 --- a/arch/e2k/include/asm/kvm/debug.h +++ b/arch/e2k/include/asm/kvm/debug.h @@ -9,6 +9,7 @@ /* do not include this header directly, only through asm/e2k_debug.h */ #include +#include /* * Some definitions to print/dump/show stacks diff --git a/arch/e2k/include/asm/kvm/gmmu_context.h b/arch/e2k/include/asm/kvm/gmmu_context.h index d0e28f1..15b4e2f 100644 --- a/arch/e2k/include/asm/kvm/gmmu_context.h +++ b/arch/e2k/include/asm/kvm/gmmu_context.h @@ -232,7 +232,9 @@ switch_guest_mm(gthread_info_t *next_gti, struct gmm_struct *next_gmm) DebugKVMSW("started to switch guest mm from GPID #%d to GPID #%d\n", cur_gti->gpid->nid.nr, next_gti->gpid->nid.nr); active_gmm = pv_vcpu_get_active_gmm(vcpu); - if (next_gmm == NULL || next_gti->gmm == NULL) { + if (next_gmm == NULL || + next_gti->gmm == NULL || + next_gti->gmm_in_release) { #ifdef DO_NOT_USE_ACTIVE_GMM /* switch to guest kernel thread, but optimization */ /* has been turned OFF, so switch to init gmm & PTs */ @@ -258,12 +260,13 @@ switch_guest_mm(gthread_info_t *next_gti, struct gmm_struct *next_gmm) active_gmm, active_gmm->nid.nr); goto out; } - if (likely(!next_gmm->in_release && !next_gti->gmm_in_release && - !pv_vcpu_is_init_gmm(vcpu, next_gmm))) { + if (likely(!pv_vcpu_is_init_gmm(vcpu, next_gmm))) { next_pgd = kvm_mmu_load_gmm_root(current_thread_info(), next_gti); + pv_vcpu_set_gmm(vcpu, next_gmm); } else { next_pgd = kvm_mmu_load_init_root(vcpu); + pv_vcpu_clear_gmm(vcpu); } switch_guest_pgd(next_pgd); pv_vcpu_set_active_gmm(vcpu, next_gmm); diff --git a/arch/e2k/include/asm/kvm/gregs.h b/arch/e2k/include/asm/kvm/gregs.h index 80da247..a32ba9f 100644 --- a/arch/e2k/include/asm/kvm/gregs.h +++ b/arch/e2k/include/asm/kvm/gregs.h @@ -30,6 +30,15 @@ greg_vs; \ }) +#define HOST_GET_SAVED_VCPU_STATE_GREG_AS_LIGHT(__ti) \ +({ \ + unsigned long greg_vs; \ + \ + HOST_ONLY_COPY_FROM_VCPU_STATE_GREG(&(__ti)->k_gregs_light, \ + greg_vs); \ + greg_vs; \ +}) + #define HOST_ONLY_SAVE_VCPU_STATE_GREG(vs__) \ ({ \ (vs__) = NATIVE_GET_UNTEGGED_DGREG(GUEST_VCPU_STATE_GREG); \ @@ -69,7 +78,7 @@ }) #define HOST_SAVE_KERNEL_GREGS_AS_LIGHT(__ti) \ - HOST_SAVE_HOST_GREGS_TO(&(__ti)->k_gregs_light, true) + HOST_SAVE_HOST_GREGS_TO(&(__ti)->k_gregs_light, false) #define HOST_SAVE_KERNEL_GREGS(__ti) \ HOST_SAVE_HOST_GREGS_TO(&(__ti)->k_gregs, true) @@ -77,6 +86,9 @@ #define HOST_SAVE_HOST_GREGS(__ti) \ HOST_SAVE_HOST_GREGS_TO(&(__ti)->k_gregs, false) +#define HOST_SAVE_GUEST_KERNEL_GREGS(__gti) \ + HOST_SAVE_HOST_GREGS_TO(&(__gti)->gk_gregs, false) + #define HOST_RESTORE_HOST_GREGS_FROM(__k_gregs, only_kernel) \ ({ \ kernel_gregs_t *k_gregs = (__k_gregs); \ @@ -97,13 +109,16 @@ }) #define HOST_RESTORE_KERNEL_GREGS_AS_LIGHT(_ti) \ - HOST_RESTORE_HOST_GREGS_FROM(&(_ti)->k_gregs_light, true) + HOST_RESTORE_HOST_GREGS_FROM(&(_ti)->k_gregs_light, false) #define HOST_RESTORE_KERNEL_GREGS(_ti) \ HOST_RESTORE_HOST_GREGS_FROM(&(_ti)->k_gregs, true) #define HOST_RESTORE_HOST_GREGS(_ti) \ HOST_RESTORE_HOST_GREGS_FROM(&(_ti)->k_gregs, false) + +#define HOST_RESTORE_GUEST_KERNEL_GREGS(_gti) \ + HOST_RESTORE_HOST_GREGS_FROM(&(_gti)->gk_gregs, false) #else /* !CONFIG_KVM_HOST_MODE */ #define HOST_SAVE_HOST_GREGS(__ti) #define HOST_RESTORE_HOST_GREGS(_ti) diff --git a/arch/e2k/include/asm/kvm/guest.h b/arch/e2k/include/asm/kvm/guest.h index 40de524..504ba1a 100644 --- a/arch/e2k/include/asm/kvm/guest.h +++ b/arch/e2k/include/asm/kvm/guest.h @@ -19,8 +19,9 @@ #include #include #include - +#include #include +#include typedef struct kvm_cpu_regs { #if defined(CONFIG_KVM_GUEST_KERNEL) && \ @@ -62,6 +63,7 @@ typedef struct kvm_cpu_regs { e2k_ctpr_t CPU_CTPR3; e2k_tir_t CPU_TIRs[MAX_TIRs_NUM]; /* Trap Info Registers */ int CPU_TIRs_num; /* number of occupied TIRs */ + u64 CPU_SBBP[SBBP_ENTRIES_NUM]; e2k_wd_t CPU_WD; /* Window Descriptor Register */ e2k_bgr_t CPU_BGR; /* Base Global Register */ e2k_lsr_t CPU_LSR; /* Loop Status Register */ @@ -72,7 +74,7 @@ typedef struct kvm_cpu_regs { e2k_cuir_t CPU_OSCUIR; /* CUI register of OS */ u64 CPU_OSR0; /* OS register #0 */ u32 CPU_OSEM; /* OS Entries Mask */ - e2k_psr_t CPU_PSR; /* Processor State Register */ + e2k_psr_t CPU_E2K_PSR; /* Processor State Register */ e2k_upsr_t CPU_UPSR; /* User Processor State Register */ e2k_pfpfr_t CPU_PFPFR; /* floating point control registers */ e2k_fpcr_t CPU_FPCR; @@ -297,6 +299,9 @@ typedef struct kvm_vcpu_state { bool irqs_under_upsr; bool do_dump_state; /* dump all stacks */ bool do_dump_stack; /* dump only active stack */ +#ifdef VCPU_REGS_DEBUG + vcpu_regs_trace_t trace; /* VCPU state trace */ +#endif /* VCPU_REGS_DEBUG */ } kvm_vcpu_state_t; #define DEBUG_MODE_ON (vcpu->arch.kmap_vcpu_state->debug_mode_on) diff --git a/arch/e2k/include/asm/kvm/guest/aau_context.h b/arch/e2k/include/asm/kvm/guest/aau_context.h index 5dffb86..ea0e78b 100644 --- a/arch/e2k/include/asm/kvm/guest/aau_context.h +++ b/arch/e2k/include/asm/kvm/guest/aau_context.h @@ -32,6 +32,7 @@ #define KVM_RESTORE_AAU_MASK_REGS(aau_context) \ ({ \ if (IS_HV_GM()) { \ + E2K_CMD_SEPARATOR; \ NATIVE_RESTORE_AAU_MASK_REGS(aau_context); \ } else { \ PREFIX_RESTORE_AAU_MASK_REGS(KVM, kvm, aau_context); \ @@ -253,6 +254,12 @@ kvm_get_aau_context_v5(e2k_aau_t *context) KVM_GET_AAU_CONTEXT_V5(context); } +static __always_inline void +kvm_set_aau_context(e2k_aau_t *context) +{ + /* AAU contesxt should restore host */ +} + #ifdef CONFIG_KVM_GUEST_KERNEL /* It is pure kvm kernel without paravirtualization */ @@ -307,6 +314,12 @@ get_aau_context(e2k_aau_t *context) kvm_get_aau_context(context); } +static __always_inline void +set_aau_context(e2k_aau_t *context) +{ + kvm_set_aau_context(context); +} + #endif /* CONFIG_KVM_GUEST_KERNEL */ #endif /* _ASM_E2K_KVM_GUEST_AAU_CONTEXT_H_ */ diff --git a/arch/e2k/include/asm/kvm/guest/cacheflush.h b/arch/e2k/include/asm/kvm/guest/cacheflush.h index 47e7381..12e80ef 100644 --- a/arch/e2k/include/asm/kvm/guest/cacheflush.h +++ b/arch/e2k/include/asm/kvm/guest/cacheflush.h @@ -3,35 +3,39 @@ #include +#ifndef __ASSEMBLY__ + struct icache_range_array; struct vm_area_struct; struct page; #ifdef CONFIG_SMP /* - * Guest kernel supports pseudo page tables, - * real page tables are managed now by host kernel - * So guest flushes can be empty + * Guest kernel functions can be run on any guest user processes and can have + * arbitrary MMU contexts to track which on host is not possible, therefore + * it is necessary to flush all instruction caches */ +extern void smp_flush_icache_all(void); + static inline void kvm_smp_flush_icache_range(e2k_addr_t start, e2k_addr_t end) { + smp_flush_icache_all(); } static inline void kvm_smp_flush_icache_range_array(struct icache_range_array *icache_range_arr) { + smp_flush_icache_all(); } static inline void kvm_smp_flush_icache_page(struct vm_area_struct *vma, struct page *page) { -} -static inline void -kvm_smp_flush_icache_all(void) -{ + smp_flush_icache_all(); } static inline void kvm_smp_flush_icache_kernel_line(e2k_addr_t addr) { + smp_flush_icache_all(); } #endif /* CONFIG_SMP */ @@ -42,7 +46,12 @@ extern void kvm_clear_dcache_l1_range(void *virt_addr, size_t len); extern void kvm_write_dcache_l2_reg(unsigned long reg_val, int reg_num, int bank_num); extern unsigned long kvm_read_dcache_l2_reg(int reg_num, int bank_num); -extern int kvm_flush_icache_range(e2k_addr_t start, e2k_addr_t end); +extern void kvm_flush_icache_all(void); +extern void kvm_flush_icache_range(e2k_addr_t start, e2k_addr_t end); +extern void kvm_flush_icache_range_array( + struct icache_range_array *icache_range_arr); +extern void kvm_flush_icache_page(struct vm_area_struct *vma, + struct page *page); #ifdef CONFIG_KVM_GUEST_KERNEL /* it is pure guest kernel (not paravirtualized based on pv_ops) */ @@ -63,11 +72,6 @@ smp_flush_icache_page(struct vm_area_struct *vma, struct page *page) kvm_smp_flush_icache_page(vma, page); } static inline void -smp_flush_icache_all(void) -{ - kvm_smp_flush_icache_all(); -} -static inline void smp_flush_icache_kernel_line(e2k_addr_t addr) { kvm_smp_flush_icache_kernel_line(addr); @@ -85,16 +89,27 @@ clear_DCACHE_L1_range(void *virt_addr, size_t len) kvm_clear_dcache_l1_range(virt_addr, len); } static inline void +__flush_icache_all(void) +{ + kvm_flush_icache_all(); +} +static inline void __flush_icache_range(e2k_addr_t start, e2k_addr_t end) { - int ret; - - ret = kvm_flush_icache_range(start, end); - if (ret) { - panic("%s(): could not flush ICACHE, error %d\n", - __func__, ret); - } + kvm_flush_icache_range(start, end); +} +static inline void +__flush_icache_range_array(struct icache_range_array *icache_range_arr) +{ + kvm_flush_icache_range_array(icache_range_arr); +} +static inline void +__flush_icache_page(struct vm_area_struct *vma, struct page *page) +{ + kvm_flush_icache_page(vma, page); } #endif /* CONFIG_KVM_GUEST_KERNEL */ +#endif /* !__ASSEMBLY__ */ + #endif /* __ASM_KVM_GUEST_CACHEFLUSH_H */ diff --git a/arch/e2k/include/asm/kvm/guest/debug.h b/arch/e2k/include/asm/kvm/guest/debug.h index aebb016..fb1f729 100644 --- a/arch/e2k/include/asm/kvm/guest/debug.h +++ b/arch/e2k/include/asm/kvm/guest/debug.h @@ -27,6 +27,8 @@ kvm_modify_instr_on_IP(e2k_addr_t ip, e2k_addr_t phys_ip, #ifdef CONFIG_KVM_GUEST_KERNEL /* it is pure guest kernel (not paravirtualized based on pv_ops) */ +#include + #define GET_PHYS_ADDR(task, addr) GUEST_GET_PHYS_ADDR(task, addr) #define debug_guest_regs(task) false /* none any guests */ diff --git a/arch/e2k/include/asm/kvm/guest/e2k.h b/arch/e2k/include/asm/kvm/guest/e2k.h index 8785b28..dd726bd 100644 --- a/arch/e2k/include/asm/kvm/guest/e2k.h +++ b/arch/e2k/include/asm/kvm/guest/e2k.h @@ -10,26 +10,10 @@ #ifdef CONFIG_VIRTUALIZATION -#if defined(CONFIG_PARAVIRT_GUEST) +#if defined(CONFIG_KVM_GUEST_KERNEL) || defined(CONFIG_PARAVIRT_GUEST) extern unsigned int guest_machine_id; #define boot_guest_machine_id boot_get_vo_value(guest_machine_id) -#endif /* CONFIG_E2K_MACHINE */ - -extern void kvm_set_mach_type_id(void); - -#ifdef CONFIG_KVM_GUEST_KERNEL -/* it is native guest kernel */ -#ifdef CONFIG_E2K_MACHINE - #if defined(CONFIG_E2K_VIRT) - #define guest_machine_id MACHINE_ID_E2K_VIRT - #define boot_guest_machine_id guest_machine_id - #else - #error "E2K VIRTUAL MACHINE type does not defined" - #endif -#else /* ! CONFIG_E2K_MACHINE */ -extern unsigned int guest_machine_id; -#define boot_guest_machine_id boot_get_vo_value(guest_machine_id) -#endif /* CONFIG_E2K_MACHINE */ +#endif /* CONFIG_KVM_GUEST_KERNEL || CONFIG_PARAVIRT_GUEST */ #define machine_id guest_machine_id #define boot_machine_id boot_guest_machine_id @@ -39,13 +23,12 @@ extern unsigned int guest_machine_id; #define set_machine_id(mach_id) (machine_id = (mach_id)) #define boot_set_machine_id(mach_id) (boot_machine_id = (mach_id)) +extern void kvm_set_mach_type_id(void); static inline void set_mach_type_id(void) { kvm_set_mach_type_id(); } -#endif /* CONFIG_KVM_GUEST_KERNEL */ - #endif /* CONFIG_VIRTUALIZATION */ #endif /* _ASM_KVM_GUEST_E2K_H_ */ diff --git a/arch/e2k/include/asm/kvm/guest/host_printk.h b/arch/e2k/include/asm/kvm/guest/host_printk.h index c3cf946..afddb99 100644 --- a/arch/e2k/include/asm/kvm/guest/host_printk.h +++ b/arch/e2k/include/asm/kvm/guest/host_printk.h @@ -15,6 +15,17 @@ extern int kvm_host_printk(const char *fmt, ...); #ifdef CONFIG_KVM_GUEST_KERNEL /* it is native guest */ #define host_printk(fmt, args...) kvm_host_printk(fmt, ##args) + +#define host_pr_alert(fmt, args...) host_printk(fmt, ##args) +#define host_pr_cont(fmt, args...) host_printk(fmt, ##args) +#define host_pr_info(fmt, args...) host_printk(fmt, ##args) + +extern void host_dump_stack(void); +extern u64 host_print_all_TIRs(const e2k_tir_t *TIRs, u64 nr_TIRs); +extern void host_print_tc_record(const trap_cellar_t *tcellar, int num); +extern void host_print_all_TC(const trap_cellar_t *TC, int TC_count); +extern void host_print_pt_regs(const struct pt_regs *regs); + #endif /* ! CONFIG_KVM_GUEST_KERNEL */ #endif /* ! _E2K_KVM_GUEST_HOST_PRINTK_H */ diff --git a/arch/e2k/include/asm/kvm/guest/mmu.h b/arch/e2k/include/asm/kvm/guest/mmu.h index 6473af2..ac57c7b 100644 --- a/arch/e2k/include/asm/kvm/guest/mmu.h +++ b/arch/e2k/include/asm/kvm/guest/mmu.h @@ -8,21 +8,24 @@ #include #include -extern long kvm_recovery_faulted_tagged_store(e2k_addr_t address, u64 wr_data, +extern void kvm_recovery_faulted_tagged_store(e2k_addr_t address, u64 wr_data, u32 data_tag, u64 st_rec_opc, u64 data_ext, u32 data_ext_tag, u64 opc_ext, int chan, int qp_store, int atomic_store); -extern long kvm_recovery_faulted_load(e2k_addr_t address, u64 *ld_val, - u8 *data_tag, u64 ld_rec_opc, int chan); -extern long kvm_recovery_faulted_move(e2k_addr_t addr_from, e2k_addr_t addr_to, +extern void kvm_recovery_faulted_load(e2k_addr_t address, u64 *ld_val, + u8 *data_tag, u64 ld_rec_opc, int chan, + tc_cond_t cond); +extern void kvm_recovery_faulted_move(e2k_addr_t addr_from, e2k_addr_t addr_to, e2k_addr_t addr_to_hi, int vr, u64 ld_rec_opc, int chan, - int qp_load, int atomic_load); -extern long kvm_recovery_faulted_load_to_greg(e2k_addr_t address, + int qp_load, int atomic_load, u32 first_time, + tc_cond_t cond); +extern void kvm_recovery_faulted_load_to_greg(e2k_addr_t address, u32 greg_num_d, int vr, u64 ld_rec_opc, int chan, int qp_load, int atomic_load, - void *saved_greg_lo, void *saved_greg_hi); -extern long kvm_move_tagged_word(e2k_addr_t addr_from, e2k_addr_t addr_to); -extern long kvm_move_tagged_dword(e2k_addr_t addr_from, e2k_addr_t addr_to); -extern long kvm_move_tagged_qword(e2k_addr_t addr_from, e2k_addr_t addr_to); + void *saved_greg_lo, void *saved_greg_hi, + tc_cond_t cond); +extern void kvm_move_tagged_word(e2k_addr_t addr_from, e2k_addr_t addr_to); +extern void kvm_move_tagged_dword(e2k_addr_t addr_from, e2k_addr_t addr_to); +extern void kvm_move_tagged_qword(e2k_addr_t addr_from, e2k_addr_t addr_to); static inline void kvm_handle_mpdma_fault(e2k_addr_t hva) @@ -47,13 +50,13 @@ kvm_is_guest_kernel_gregs(struct thread_info *ti, #ifdef CONFIG_KVM_GUEST_KERNEL static inline int -guest_addr_to_host(void **addr, pt_regs_t *regs) +guest_addr_to_host(void **addr, const pt_regs_t *regs) { return native_guest_addr_to_host(addr); } static inline void * -guest_ptr_to_host(void *ptr, int size, pt_regs_t *regs) +guest_ptr_to_host(void *ptr, int size, const pt_regs_t *regs) { /* there are not any guests, so nothing convertion */ return native_guest_ptr_to_host(ptr, size); @@ -65,82 +68,85 @@ is_guest_kernel_gregs(struct thread_info *ti, { return kvm_is_guest_kernel_gregs(ti, greg_num_d, greg_copy); } -static inline long +static inline void recovery_faulted_tagged_store(e2k_addr_t address, u64 wr_data, u32 data_tag, u64 st_rec_opc, u64 data_ext, u32 data_ext_tag, u64 opc_ext, int chan, int qp_store, int atomic_store) { if (likely(IS_HV_GM())) - return native_recovery_faulted_tagged_store(address, wr_data, + native_recovery_faulted_tagged_store(address, wr_data, data_tag, st_rec_opc, data_ext, data_ext_tag, opc_ext, chan, qp_store, atomic_store); else - return kvm_recovery_faulted_tagged_store(address, wr_data, + kvm_recovery_faulted_tagged_store(address, wr_data, data_tag, st_rec_opc, data_ext, data_ext_tag, opc_ext, chan, qp_store, atomic_store); } -static inline long +static inline void recovery_faulted_load(e2k_addr_t address, u64 *ld_val, u8 *data_tag, - u64 ld_rec_opc, int chan) + u64 ld_rec_opc, int chan, + tc_cond_t cond) { if (likely(IS_HV_GM())) - return native_recovery_faulted_load(address, ld_val, + native_recovery_faulted_load(address, ld_val, data_tag, ld_rec_opc, chan); else - return kvm_recovery_faulted_load(address, ld_val, - data_tag, ld_rec_opc, chan); + kvm_recovery_faulted_load(address, ld_val, + data_tag, ld_rec_opc, chan, cond); } -static inline long +static inline void recovery_faulted_move(e2k_addr_t addr_from, e2k_addr_t addr_to, e2k_addr_t addr_to_hi, int vr, u64 ld_rec_opc, int chan, - int qp_load, int atomic_load) + int qp_load, int atomic_load, u32 first_time, + tc_cond_t cond) { if (likely(IS_HV_GM())) - return native_recovery_faulted_move(addr_from, addr_to, + native_recovery_faulted_move(addr_from, addr_to, addr_to_hi, vr, ld_rec_opc, chan, - qp_load, atomic_load); + qp_load, atomic_load, first_time); else - return kvm_recovery_faulted_move(addr_from, addr_to, + kvm_recovery_faulted_move(addr_from, addr_to, addr_to_hi, vr, ld_rec_opc, chan, - qp_load, atomic_load); + qp_load, atomic_load, first_time, cond); } -static inline long +static inline void recovery_faulted_load_to_greg(e2k_addr_t address, u32 greg_num_d, int vr, u64 ld_rec_opc, int chan, int qp_load, int atomic_load, - void *saved_greg_lo, void *saved_greg_hi) + void *saved_greg_lo, void *saved_greg_hi, + tc_cond_t cond) { if (likely(IS_HV_GM())) - return native_recovery_faulted_load_to_greg(address, greg_num_d, + native_recovery_faulted_load_to_greg(address, greg_num_d, vr, ld_rec_opc, chan, qp_load, atomic_load, saved_greg_lo, saved_greg_hi); else - return kvm_recovery_faulted_load_to_greg(address, greg_num_d, + kvm_recovery_faulted_load_to_greg(address, greg_num_d, vr, ld_rec_opc, chan, qp_load, atomic_load, - saved_greg_lo, saved_greg_hi); + saved_greg_lo, saved_greg_hi, cond); } -static inline long +static inline void move_tagged_word(e2k_addr_t addr_from, e2k_addr_t addr_to) { if (likely(IS_HV_GM())) - return native_move_tagged_word(addr_from, addr_to); + native_move_tagged_word(addr_from, addr_to); else - return kvm_move_tagged_word(addr_from, addr_to); + kvm_move_tagged_word(addr_from, addr_to); } -static inline long +static inline void move_tagged_dword(e2k_addr_t addr_from, e2k_addr_t addr_to) { if (likely(IS_HV_GM())) - return native_move_tagged_dword(addr_from, addr_to); + native_move_tagged_dword(addr_from, addr_to); else - return kvm_move_tagged_dword(addr_from, addr_to); + kvm_move_tagged_dword(addr_from, addr_to); } -static inline long +static inline void move_tagged_qword(e2k_addr_t addr_from, e2k_addr_t addr_to) { if (likely(IS_HV_GM())) - return native_move_tagged_qword(addr_from, addr_to); + native_move_tagged_qword(addr_from, addr_to); else - return kvm_move_tagged_qword(addr_from, addr_to); + kvm_move_tagged_qword(addr_from, addr_to); } static inline void diff --git a/arch/e2k/include/asm/kvm/guest/mmu_context.h b/arch/e2k/include/asm/kvm/guest/mmu_context.h index d44dd54..463aa92 100644 --- a/arch/e2k/include/asm/kvm/guest/mmu_context.h +++ b/arch/e2k/include/asm/kvm/guest/mmu_context.h @@ -19,7 +19,8 @@ static inline void deactivate_mm(struct task_struct *dead_task, struct mm_struct *mm) { native_deactivate_mm(dead_task, mm); - HYPERVISOR_switch_to_guest_init_mm(); + if (!dead_task->clear_child_tid || (atomic_read(&mm->mm_users) <= 1)) + HYPERVISOR_switch_to_guest_init_mm(); } #endif /* CONFIG_KVM_GUEST_KERNEL */ diff --git a/arch/e2k/include/asm/kvm/guest/process.h b/arch/e2k/include/asm/kvm/guest/process.h index 05c864d..6da1548 100644 --- a/arch/e2k/include/asm/kvm/guest/process.h +++ b/arch/e2k/include/asm/kvm/guest/process.h @@ -100,6 +100,35 @@ static inline void KVM_COPY_STACKS_TO_MEMORY(void) HYPERVISOR_copy_stacks_to_memory(); } +/* own VCPU state: directly accessible through global registers */ +static inline kvm_vcpu_state_t *kvm_get_vcpu_state(void) +{ + unsigned long vcpu_base; + + KVM_GET_VCPU_STATE_BASE(vcpu_base); + return (kvm_vcpu_state_t *)(vcpu_base); +} + +/* + * Restore proper psize field of WD register + */ +static inline void +kvm_restore_wd_register_psize(e2k_wd_t wd_from) +{ + HYPERVISOR_update_wd_psize(wd_from.WD_psize); +} + +/* + * Preserve current p[c]shtp as they indicate how much to FILL when returning + */ +static inline void +kvm_preserve_user_hw_stacks_to_copy(e2k_stacks_t *u_stacks, + e2k_stacks_t *cur_stacks) +{ + /* guest user hardware stacks sizes to copy should be updated */ + /* after copying and therefore are not preserve */ +} + static inline void kvm_kernel_hw_stack_frames_copy(u64 *dst, const u64 *src, unsigned long size) { @@ -271,7 +300,7 @@ failed: } static __always_inline int -kvm_user_hw_stacks_copy(pt_regs_t *regs, int add_frames_num) +kvm_user_hw_stacks_copy(pt_regs_t *regs) { e2k_psp_lo_t psp_lo; e2k_psp_hi_t psp_hi; @@ -282,10 +311,30 @@ kvm_user_hw_stacks_copy(pt_regs_t *regs, int add_frames_num) e2k_stacks_t *stacks; void __user *dst; void *src; - long to_copy, from, there_are, add_frames_size; + long copyed_ps_size, copyed_pcs_size, to_copy, from, there_are; int ret; - BUG_ON(irqs_disabled()); + if (unlikely(irqs_disabled())) { + pr_err("%s() called with IRQs disabled PSP: 0x%lx UPSR: 0x%lx " + "under UPSR %d\n", + __func__, KVM_READ_PSR_REG_VALUE(), + KVM_READ_UPSR_REG_VALUE(), + kvm_get_vcpu_state()->irqs_under_upsr); + local_irq_enable(); + WARN_ON(true); + } + + stacks = ®s->stacks; + copyed_ps_size = regs->copyed.ps_size; + copyed_pcs_size = regs->copyed.pcs_size; + if (unlikely(copyed_ps_size || copyed_pcs_size)) { + /* stacks have been already copyed */ + BUG_ON(copyed_ps_size != GET_PSHTP_MEM_INDEX(stacks->pshtp) && + GET_PSHTP_MEM_INDEX(stacks->pshtp) != 0); + BUG_ON(copyed_pcs_size != PCSHTP_SIGN_EXTEND(stacks->pcshtp) && + PCSHTP_SIGN_EXTEND(stacks->pcshtp) != SZ_OF_CR); + return 0; + } ret = HYPERVISOR_copy_stacks_to_memory(); if (ret != 0) { @@ -295,7 +344,6 @@ kvm_user_hw_stacks_copy(pt_regs_t *regs, int add_frames_num) } /* copy user part of procedure stack from kernel back to user */ - stacks = ®s->stacks; ATOMIC_READ_HW_STACKS_REGS(psp_lo.PSP_lo_half, psp_hi.PSP_hi_half, pshtp.PSHTP_reg, pcsp_lo.PCSP_lo_half, pcsp_hi.PCSP_hi_half, @@ -339,17 +387,16 @@ kvm_user_hw_stacks_copy(pt_regs_t *regs, int add_frames_num) __func__, src, dst, to_copy, ret); goto failed; } + regs->copyed.ps_size = to_copy; } /* copy user part of chain stack from kernel back to user */ - add_frames_size = add_frames_num * SZ_OF_CR; src = (void *)pcsp_lo.PCSP_lo_base; - DebugUST("chain stack at kernel from %px, size 0x%x + 0x%lx, ind 0x%x, " + DebugUST("chain stack at kernel from %px, size 0x%x, ind 0x%x, " "pcshtp 0x%x\n", - src, pcsp_hi.PCSP_hi_size, add_frames_size, pcsp_hi.PCSP_hi_ind, - pcshtp); - BUG_ON(pcsp_hi.PCSP_hi_ind + PCSHTP_SIGN_EXTEND(pcshtp) + - add_frames_size > pcsp_hi.PCSP_hi_size); + src, pcsp_hi.PCSP_hi_size, pcsp_hi.PCSP_hi_ind, pcshtp); + BUG_ON(pcsp_hi.PCSP_hi_ind + PCSHTP_SIGN_EXTEND(pcshtp) > + pcsp_hi.PCSP_hi_size); if (stacks->pcsp_hi.PCSP_hi_ind >= stacks->pcsp_hi.PCSP_hi_size) { /* chain stack overflow, need expand */ ret = handle_chain_stack_bounds(stacks, regs->trap); @@ -365,7 +412,6 @@ kvm_user_hw_stacks_copy(pt_regs_t *regs, int add_frames_num) from = stacks->pcsp_hi.PCSP_hi_ind - to_copy; BUG_ON(from < 0); dst = (void *)stacks->pcsp_lo.PCSP_lo_base + from; - to_copy += add_frames_size; BUG_ON(to_copy > pcsp_hi.PCSP_hi_ind + PCSHTP_SIGN_EXTEND(pcshtp)); DebugUST("chain stack at user from %px, ind 0x%x, " "pcshtp size to copy 0x%lx\n", @@ -385,10 +431,95 @@ kvm_user_hw_stacks_copy(pt_regs_t *regs, int add_frames_num) __func__, src, dst, to_copy, ret); goto failed; } + regs->copyed.pcs_size = to_copy; } - if (add_frames_size > 0) { + +failed: + if (DEBUG_USER_STACKS_MODE) + debug_ustacks = false; + return ret; +} + +/* + * Copy additional frames injected to the guest kernel stack, but these frames + * are for guest user stack and should be copyed from kernel back to the top + * of user. + */ +static __always_inline int +kvm_copy_injected_pcs_frames_to_user(pt_regs_t *regs, int frames_num) +{ + e2k_size_t pcs_ind, pcs_size; + e2k_addr_t pcs_base; + int pcsh_top; + e2k_stacks_t *stacks; + void __user *dst; + void *src; + long copyed_frames_size, to_copy, from, there_are, frames_size; + int ret; + + BUG_ON(irqs_disabled()); + + frames_size = frames_num * SZ_OF_CR; + copyed_frames_size = regs->copyed.pcs_injected_frames_size; + if (unlikely(copyed_frames_size >= frames_size)) { + /* all frames have been already copyed */ + return 0; + } else { + /* copyed only part of frames - not implemented case */ + BUG_ON(copyed_frames_size != 0); + } + + stacks = ®s->stacks; + ATOMIC_GET_HW_PCS_SIZES_BASE_TOP(pcs_ind, pcs_size, pcs_base, pcsh_top); + + /* guest user stacks part spilled to kernel should be already copyed */ + BUG_ON(PCSHTP_SIGN_EXTEND(regs->copyed.pcs_size != stacks->pcshtp)); + + src = (void *)(pcs_base + regs->copyed.pcs_size); + DebugUST("chain stack at kernel from %px, size 0x%lx + 0x%lx, " + "ind 0x%lx, pcsh top 0x%x\n", + src, pcs_size, frames_size, pcs_ind, pcsh_top); + BUG_ON(regs->copyed.pcs_size + frames_size > pcs_ind + pcsh_top); + if (stacks->pcsp_hi.PCSP_hi_ind + frames_size > + stacks->pcsp_hi.PCSP_hi_size) { + /* user chain stack can overflow, need expand */ + ret = handle_chain_stack_bounds(stacks, regs->trap); + if (unlikely(ret)) { + pr_err("%s(): could not handle process %s (%d) " + "chain stack overflow, error %d\n", + __func__, current->comm, current->pid, ret); + goto failed; + } + } + to_copy = frames_size; + BUG_ON(to_copy < 0); + from = stacks->pcsp_hi.PCSP_hi_ind; + BUG_ON(from < regs->copyed.pcs_size); + dst = (void *)stacks->pcsp_lo.PCSP_lo_base + from; + DebugUST("chain stack at user from %px, ind 0x%x, " + "frames size to copy 0x%lx\n", + dst, stacks->pcsp_hi.PCSP_hi_ind, to_copy); + there_are = stacks->pcsp_hi.PCSP_hi_size - from; + if (there_are < to_copy) { + pr_err("%s(): user chain stack overflow, there are 0x%lx " + "to copy need 0x%lx, not yet implemented\n", + __func__, there_are, to_copy); + BUG_ON(true); + } + if (likely(to_copy > 0)) { + ret = kvm_copy_user_stack_from_kernel(dst, src, to_copy, true); + if (ret != 0) { + pr_err("%s(): chain stack copying from kernel %px " + "to user %px, size 0x%lx failed, error %d\n", + __func__, src, dst, to_copy, ret); + goto failed; + } + regs->copyed.pcs_injected_frames_size = to_copy; /* increment chain stack pointer */ - stacks->pcsp_hi.PCSP_hi_ind += add_frames_size; + stacks->pcsp_hi.PCSP_hi_ind += to_copy; + } else { + BUG_ON(true); + ret = 0; } failed: @@ -451,7 +582,7 @@ static __always_inline int kvm_user_hw_stacks_prepare( * 2) User data copying will be done some later at * kvm_prepare_user_hv_stacks() */ - ret = kvm_user_hw_stacks_copy(regs, 0); + ret = kvm_user_hw_stacks_copy(regs); if (ret != 0) { pr_err("%s(): copying of hardware stacks failed< error %d\n", __func__, ret); @@ -463,7 +594,44 @@ static __always_inline int kvm_user_hw_stacks_prepare( static inline int kvm_ret_from_fork_prepare_hv_stacks(struct pt_regs *regs) { - return kvm_user_hw_stacks_copy(regs, 0); + return kvm_user_hw_stacks_copy(regs); +} + +static __always_inline void +kvm_jump_to_ttable_entry(struct pt_regs *regs, enum restore_caller from) +{ + if (from & FROM_SYSCALL_N_PROT) { + switch (regs->kernel_entry) { + case 1: + case 3: + case 4: + KVM_WRITE_UPSR_REG(E2K_KERNEL_UPSR_ENABLED); + regs->stack_regs_saved = true; + __E2K_JUMP_WITH_ARGUMENTS_8(handle_sys_call, + regs->sys_func, + regs->args[1], regs->args[2], + regs->args[3], regs->args[4], + regs->args[5], regs->args[6], + regs); + default: + BUG(); + } + } else if (from & FROM_SYSCALL_PROT_8) { + /* the syscall restart is not yet implemented */ + BUG(); + } else if (from & FROM_SYSCALL_PROT_10) { + /* the syscall restart is not yet implemented */ + BUG(); + } else { + BUG(); + } +} + +static inline void kvm_clear_virt_thread_struct(thread_info_t *ti) +{ + /* guest PID/MMID's can be received only after registration on host */ + ti->gpid_nr = -1; + ti->gmmid_nr = -1; } static inline void kvm_release_task_struct(struct task_struct *task) @@ -473,6 +641,11 @@ static inline void kvm_release_task_struct(struct task_struct *task) ti = task_thread_info(task); BUG_ON(ti == NULL); + if (ti->gpid_nr == -1) { + /* the process was not registered on host, nothing to do */ + BUG_ON(ti->gmmid_nr != -1); + return; + } ret = HYPERVISOR_release_task_struct(ti->gpid_nr); if (ret != 0) { @@ -527,15 +700,6 @@ static inline kvm_vcpu_state_t *kvm_get_the_vcpu_state(long vcpu_id) return vcpus_state[vcpu_id]; } -/* own VCPU state: directly accessible through global registers */ -static inline kvm_vcpu_state_t *kvm_get_vcpu_state(void) -{ - unsigned long vcpu_base; - - KVM_GET_VCPU_STATE_BASE(vcpu_base); - return (kvm_vcpu_state_t *)(vcpu_base); -} - #define KVM_ONLY_SET_GUEST_GREGS(ti) \ KVM_SET_VCPU_STATE_BASE(kvm_get_the_vcpu_state( \ smp_processor_id())) @@ -646,6 +810,19 @@ static inline void COPY_STACKS_TO_MEMORY(void) KVM_COPY_STACKS_TO_MEMORY(); } +static inline void +restore_wd_register_psize(e2k_wd_t wd_from) +{ + kvm_restore_wd_register_psize(wd_from); +} + +static inline void +preserve_user_hw_stacks_to_copy(e2k_stacks_t *u_stacks, + e2k_stacks_t *cur_stacks) +{ + kvm_preserve_user_hw_stacks_to_copy(u_stacks, cur_stacks); +} + static __always_inline void kernel_hw_stack_frames_copy(u64 *dst, const u64 *src, unsigned long size) { @@ -664,6 +841,13 @@ collapse_kernel_pcs(u64 *dst, const u64 *src, u64 spilled_size) kvm_collapse_kernel_pcs(dst, src, spilled_size); } +static __always_inline int +user_hw_stacks_copy(struct e2k_stacks *stacks, + pt_regs_t *regs, u64 cur_window_q, bool copy_full) +{ + return kvm_user_hw_stacks_copy(regs); +} + static __always_inline void host_user_hw_stacks_prepare( struct e2k_stacks *stacks, pt_regs_t *regs, u64 cur_window_q, enum restore_caller from, int syscall) @@ -676,12 +860,24 @@ static __always_inline void host_user_hw_stacks_prepare( from, syscall); } +static __always_inline void +host_exit_to_usermode_loop(struct pt_regs *regs, bool syscall, bool has_signal) +{ + /* native & guest kernels cannot be as host */ +} + static inline int ret_from_fork_prepare_hv_stacks(struct pt_regs *regs) { return kvm_ret_from_fork_prepare_hv_stacks(regs); } +static __always_inline void +jump_to_ttable_entry(struct pt_regs *regs, enum restore_caller from) +{ + kvm_jump_to_ttable_entry(regs, from); +} + static inline void virt_cpu_thread_init(struct task_struct *boot_task) { @@ -850,6 +1046,7 @@ complete_go2user(thread_info_t *ti, long fn) static inline void clear_virt_thread_struct(thread_info_t *ti) { + kvm_clear_virt_thread_struct(ti); } static inline void virt_setup_arch(void) diff --git a/arch/e2k/include/asm/kvm/guest/regs_state.h b/arch/e2k/include/asm/kvm/guest/regs_state.h index 635dcc6..a39b1ae 100644 --- a/arch/e2k/include/asm/kvm/guest/regs_state.h +++ b/arch/e2k/include/asm/kvm/guest/regs_state.h @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -21,9 +20,10 @@ extern void kvm_save_glob_regs(global_regs_t *gregs); extern void kvm_save_glob_regs_dirty_bgr(global_regs_t *gregs); -extern void kvm_save_local_glob_regs(local_gregs_t *l_gregs); +extern void kvm_save_local_glob_regs(local_gregs_t *l_gregs, bool is_signal); extern void kvm_restore_glob_regs(const global_regs_t *gregs); -extern void kvm_restore_local_glob_regs(const local_gregs_t *l_gregs); +extern void kvm_restore_local_glob_regs(const local_gregs_t *l_gregs, + bool is_signal); extern void kvm_get_all_user_glob_regs(global_regs_t *gregs); static inline void @@ -51,9 +51,9 @@ guest_save_glob_regs_dirty_bgr_v5(global_regs_t *gregs) } static inline void -guest_save_local_glob_regs_v2(local_gregs_t *l_gregs) +guest_save_local_glob_regs_v2(local_gregs_t *l_gregs, bool is_signal) { - kvm_guest_save_local_gregs_v2(l_gregs); + kvm_guest_save_local_gregs_v2(l_gregs, is_signal); if (KERNEL_GREGS_MAX_MASK & LOCAL_GREGS_USER_MASK) copy_k_gregs_to_l_gregs(l_gregs, ¤t_thread_info()->k_gregs); @@ -63,9 +63,9 @@ guest_save_local_glob_regs_v2(local_gregs_t *l_gregs) } static inline void -guest_save_local_glob_regs_v5(local_gregs_t *l_gregs) +guest_save_local_glob_regs_v5(local_gregs_t *l_gregs, bool is_signal) { - kvm_guest_save_local_gregs_v5(l_gregs); + kvm_guest_save_local_gregs_v5(l_gregs, is_signal); if (KERNEL_GREGS_MAX_MASK & LOCAL_GREGS_USER_MASK) copy_k_gregs_to_l_gregs(l_gregs, ¤t_thread_info()->k_gregs); @@ -87,9 +87,9 @@ guest_restore_glob_regs_v5(const global_regs_t *gregs) } static inline void -guest_restore_local_glob_regs_v2(const local_gregs_t *l_gregs) +guest_restore_local_glob_regs_v2(const local_gregs_t *l_gregs, bool is_signal) { - kvm_guest_restore_local_gregs_v2(l_gregs); + kvm_guest_restore_local_gregs_v2(l_gregs, is_signal); if (KERNEL_GREGS_MAX_MASK & LOCAL_GREGS_USER_MASK) get_k_gregs_from_l_regs(¤t_thread_info()->k_gregs, l_gregs); @@ -99,9 +99,9 @@ guest_restore_local_glob_regs_v2(const local_gregs_t *l_gregs) } static inline void -guest_restore_local_glob_regs_v5(const local_gregs_t *l_gregs) +guest_restore_local_glob_regs_v5(const local_gregs_t *l_gregs, bool is_signal) { - kvm_guest_restore_local_gregs_v5(l_gregs); + kvm_guest_restore_local_gregs_v5(l_gregs, is_signal); if (KERNEL_GREGS_MAX_MASK & LOCAL_GREGS_USER_MASK) get_k_gregs_from_l_regs(¤t_thread_info()->k_gregs, l_gregs); @@ -197,8 +197,11 @@ guest_get_all_user_glob_regs(global_regs_t *gregs) do { \ if (IS_HV_GM()) { \ NATIVE_SAVE_STACK_REGS(regs, ti, from_ti, trap); \ - } else { \ + } else if (!(regs)->stack_regs_saved) { \ PREFIX_SAVE_STACK_REGS(KVM, regs, ti, from_ti, trap); \ + } else { \ + /* registers were already saved */ \ + ; \ } \ } while (false) @@ -250,6 +253,8 @@ do { \ KVM_RESTORE_USER_STACK_REGS(regs, true) #define KVM_RESTORE_USER_CUT_REGS(ti, regs) /* CUTD is set by host */ +#define KVM_RESTORE_COMMON_REGS(regs) /* should be restored by host */ + #define KVM_SAVE_TRAP_CELLAR(regs, trap) \ ({ \ kernel_trap_cellar_t *kernel_tcellar = \ @@ -340,6 +345,8 @@ do { \ RESTORE_USER_STACK_REGS(regs, true) #define RESTORE_USER_CUT_REGS(ti, regs, in_sys_call) \ KVM_RESTORE_USER_CUT_REGS(ti, regs) +#define RESTORE_COMMON_REGS(regs) \ + KVM_RESTORE_COMMON_REGS(regs) static inline void save_glob_regs_v2(global_regs_t *gregs) @@ -382,22 +389,22 @@ save_glob_regs_dirty_bgr_v5(global_regs_t *gregs) } static inline void -save_local_glob_regs_v2(local_gregs_t *l_gregs) +save_local_glob_regs_v2(local_gregs_t *l_gregs, bool is_signal) { if (IS_HV_GM()) { - guest_save_local_glob_regs_v2(l_gregs); + guest_save_local_glob_regs_v2(l_gregs, is_signal); } else { - kvm_save_local_glob_regs(l_gregs); + kvm_save_local_glob_regs(l_gregs, is_signal); } } static inline void -save_local_glob_regs_v5(local_gregs_t *l_gregs) +save_local_glob_regs_v5(local_gregs_t *l_gregs, bool is_signal) { if (IS_HV_GM()) { - guest_save_local_glob_regs_v5(l_gregs); + guest_save_local_glob_regs_v5(l_gregs, is_signal); } else { - kvm_save_local_glob_regs(l_gregs); + kvm_save_local_glob_regs(l_gregs, is_signal); } } @@ -422,32 +429,32 @@ restore_glob_regs_v5(const global_regs_t *gregs) } static inline void -restore_local_glob_regs_v2(const local_gregs_t *l_gregs) +restore_local_glob_regs_v2(const local_gregs_t *l_gregs, bool is_signal) { if (IS_HV_GM()) - guest_restore_local_glob_regs_v2(l_gregs); + guest_restore_local_glob_regs_v2(l_gregs, is_signal); else - kvm_restore_local_glob_regs(l_gregs); + kvm_restore_local_glob_regs(l_gregs, is_signal); } static inline void -restore_local_glob_regs_v5(const local_gregs_t *l_gregs) +restore_local_glob_regs_v5(const local_gregs_t *l_gregs, bool is_signal) { if (IS_HV_GM()) - guest_restore_local_glob_regs_v5(l_gregs); + guest_restore_local_glob_regs_v5(l_gregs, is_signal); else - kvm_restore_local_glob_regs(l_gregs); + kvm_restore_local_glob_regs(l_gregs, is_signal); } static inline void -save_local_glob_regs(local_gregs_t *l_gregs) +save_local_glob_regs(local_gregs_t *l_gregs, bool is_signal) { - machine.save_local_gregs(l_gregs); + machine.save_local_gregs(l_gregs, is_signal); } static inline void -restore_local_glob_regs(const local_gregs_t *l_gregs) +restore_local_glob_regs(const local_gregs_t *l_gregs, bool is_signal) { - machine.restore_local_gregs(l_gregs); + machine.restore_local_gregs(l_gregs, is_signal); } static inline void diff --git a/arch/e2k/include/asm/kvm/guest/setup.h b/arch/e2k/include/asm/kvm/guest/setup.h index 9ede913..8b65723 100644 --- a/arch/e2k/include/asm/kvm/guest/setup.h +++ b/arch/e2k/include/asm/kvm/guest/setup.h @@ -8,7 +8,7 @@ extern void __init boot_e2k_virt_setup_arch(void); extern void __init e2k_virt_setup_machine(void); -extern void kvm_bsp_switch_to_init_stack(void); +extern void __init kvm_bsp_switch_to_init_stack(void); extern void kvm_setup_bsp_idle_task(int cpu); extern void setup_guest_interface(void); diff --git a/arch/e2k/include/asm/kvm/guest/string.h b/arch/e2k/include/asm/kvm/guest/string.h index fb36999..7b34106 100644 --- a/arch/e2k/include/asm/kvm/guest/string.h +++ b/arch/e2k/include/asm/kvm/guest/string.h @@ -2,6 +2,7 @@ #define _E2K_KVM_GUEST_STRING_H_ #include +#include #include #include @@ -19,14 +20,11 @@ kvm_do_fast_tagged_memory_copy(void *dst, const void *src, size_t len, { long ret; - if (IS_HOST_KERNEL_ADDRESS((e2k_addr_t)src) || - IS_HOST_KERNEL_ADDRESS((e2k_addr_t)dst)) { - ret = HYPERVISOR_fast_tagged_guest_memory_copy(dst, src, len, - strd_opcode, ldrd_opcode, prefetch); - } else { + do { ret = HYPERVISOR_fast_tagged_memory_copy(dst, src, len, strd_opcode, ldrd_opcode, prefetch); - } + } while (ret == -EAGAIN); + return ret; } static inline unsigned long diff --git a/arch/e2k/include/asm/kvm/guest/switch.h b/arch/e2k/include/asm/kvm/guest/switch.h index 3907b21..852c3ca 100644 --- a/arch/e2k/include/asm/kvm/guest/switch.h +++ b/arch/e2k/include/asm/kvm/guest/switch.h @@ -68,25 +68,18 @@ static inline struct e2k_stacks * kvm_syscall_guest_get_restore_stacks(struct thread_info *ti, struct pt_regs *regs) { - return native_syscall_guest_get_restore_stacks(ti, regs); + return native_syscall_guest_get_restore_stacks(regs); } /* * The function should return bool is the system call from guest */ -static inline bool -kvm_guest_syscall_enter(struct thread_info *ti, struct pt_regs *regs) +static inline bool kvm_guest_syscall_enter(struct pt_regs *regs) { /* guest cannot have own nested guests */ return false; /* it is not nested guest system call */ } -static inline void -kvm_guest_syscall_exit_to(struct thread_info *ti, struct pt_regs *regs, - unsigned flags) -{ - /* nothing guests can be */ -} #ifdef CONFIG_KVM_GUEST_KERNEL /* it is pure guest kernel (not paravrtualized) */ @@ -138,25 +131,28 @@ trap_guest_get_restore_stacks(struct thread_info *ti, struct pt_regs *regs) } static inline struct e2k_stacks * -syscall_guest_get_restore_stacks(struct thread_info *ti, struct pt_regs *regs) +syscall_guest_get_restore_stacks(bool ts_host_at_vcpu_mode, struct pt_regs *regs) { - return kvm_syscall_guest_get_restore_stacks(ti, regs); + return kvm_syscall_guest_get_restore_stacks( + current_thread_info(), regs); } +#define ts_host_at_vcpu_mode() false + /* * The function should return bool is the system call from guest */ -static inline bool -guest_syscall_enter(struct thread_info *ti, struct pt_regs *regs) +static inline bool guest_syscall_enter(struct pt_regs *regs, + bool ts_host_at_vcpu_mode) { - return kvm_guest_syscall_enter(ti, regs); -} -static inline void -guest_syscall_exit_to(struct thread_info *ti, struct pt_regs *regs, - unsigned flags) -{ - kvm_guest_syscall_exit_to(ti, regs, flags); + return kvm_guest_syscall_enter(regs); } + +static inline void guest_exit_intc(struct pt_regs *regs, + bool intc_emul_flag) { } +static inline void guest_syscall_exit_trap(struct pt_regs *regs, + bool ts_host_at_vcpu_mode) { } + #endif /* CONFIG_KVM_GUEST_KERNEL */ #endif /* ! _E2K_KVM_GUEST_SWITCH_H */ diff --git a/arch/e2k/include/asm/kvm/guest/sync_pg_tables.h b/arch/e2k/include/asm/kvm/guest/sync_pg_tables.h new file mode 100644 index 0000000..36f1932 --- /dev/null +++ b/arch/e2k/include/asm/kvm/guest/sync_pg_tables.h @@ -0,0 +1,19 @@ +/* Functions to sync shadow page tables with guest page tables + * without flushing tlb. Used only by guest kernels + * + * Copyright 2021 Andrey Alekhin (alekhin_amcst.ru) + */ + +#ifndef _E2K_GST_SYNC_PG_TABLES_H +#define _E2K_GST_SYNC_PG_TABLES_H + +#include +#include + +static inline void kvm_sync_addr_range(e2k_addr_t start, e2k_addr_t end) +{ + if (!IS_HV_GM()) + HYPERVISOR_sync_addr_range(start, end); +} + +#endif diff --git a/arch/e2k/include/asm/kvm/guest/trap_table.h b/arch/e2k/include/asm/kvm/guest/trap_table.h index 69ebc97..a5198b0 100644 --- a/arch/e2k/include/asm/kvm/guest/trap_table.h +++ b/arch/e2k/include/asm/kvm/guest/trap_table.h @@ -18,20 +18,40 @@ extern long kvm_guest_ttable_entry5(int sys_num, extern long kvm_guest_ttable_entry6(int sys_num, u64 arg1, u64 arg2, u64 arg3, u64 arg4, u64 arg5, u64 arg6); +static __always_inline void kvm_init_pt_regs_copyed_fields(struct pt_regs *regs) +{ +#ifdef CONFIG_KVM_GUEST_KERNEL + if (likely(!regs->stack_regs_saved)) { + regs->copyed.ps_size = 0; + regs->copyed.pcs_size = 0; + regs->copyed.pcs_injected_frames_size = 0; + } else { + /* the regs is reused and all stacks should be already copyed */ + ; + } +#endif /* CONFIG_KVM_GUEST_KERNEL */ +} + +static __always_inline void kvm_init_pt_regs(struct pt_regs *regs) +{ + kvm_init_pt_regs_copyed_fields(regs); +} + static __always_inline void kvm_init_traps_handling(struct pt_regs *regs, bool user_mode_trap) { - regs->deferred_traps = 0; + kvm_init_pt_regs(regs); } static __always_inline void kvm_init_syscalls_handling(struct pt_regs *regs) { kvm_init_traps_handling(regs, true); /* now as traps init */ } -static inline bool -kvm_have_deferred_traps(struct pt_regs *regs) + +static inline void kvm_clear_fork_child_pt_regs(struct pt_regs *childregs) { - return regs->deferred_traps != 0; + native_clear_fork_child_pt_regs(childregs); + kvm_init_pt_regs_copyed_fields(childregs); } #define KVM_FILL_HARDWARE_STACKS() /* host itself will fill */ @@ -99,15 +119,12 @@ kvm_stack_bounds_trap_enable(void) kvm_set_sge(); } -extern void kvm_handle_deferred_traps(struct pt_regs *regs); -extern void kvm_handle_deferred_traps_in_syscall(struct pt_regs *regs); - -static inline void +static inline int kvm_do_aau_page_fault(struct pt_regs *const regs, e2k_addr_t address, const tc_cond_t condition, const tc_mask_t mask, const unsigned int aa_no) { - native_do_aau_page_fault(regs, address, condition, mask, aa_no); + return native_do_aau_page_fault(regs, address, condition, mask, aa_no); } #ifdef CONFIG_KVM_GUEST_KERNEL @@ -178,16 +195,12 @@ is_guest_TIRs_frozen(struct pt_regs *regs) { return false; /* none any guest */ } -static inline bool -have_deferred_traps(struct pt_regs *regs) + +static inline void clear_fork_child_pt_regs(struct pt_regs *childregs) { - return kvm_have_deferred_traps(regs); -} -static inline void -handle_deferred_traps_in_syscall(struct pt_regs *regs) -{ - kvm_handle_deferred_traps_in_syscall(regs); + kvm_clear_fork_child_pt_regs(childregs); } + static inline bool is_proc_stack_bounds(struct thread_info *ti, struct pt_regs *regs) { @@ -229,12 +242,12 @@ handle_guest_last_wish(struct pt_regs *regs) return false; /* none any guest and any wishes from */ } -static inline void +static inline int do_aau_page_fault(struct pt_regs *const regs, e2k_addr_t address, const tc_cond_t condition, const tc_mask_t mask, const unsigned int aa_no) { - kvm_do_aau_page_fault(regs, address, condition, mask, aa_no); + return kvm_do_aau_page_fault(regs, address, condition, mask, aa_no); } /* diff --git a/arch/e2k/include/asm/kvm/host_printk.h b/arch/e2k/include/asm/kvm/host_printk.h index 92650a8..ab6c26b 100644 --- a/arch/e2k/include/asm/kvm/host_printk.h +++ b/arch/e2k/include/asm/kvm/host_printk.h @@ -15,6 +15,19 @@ /* it is native host without any virtualization or */ /* native kernel with virtualization support */ #define host_printk(fmt, args...) printk(fmt, ##args) + +#define host_pr_alert(fmt, args...) pr_alert(fmt, ##args) +#define host_pr_cont(fmt, args...) pr_cont(fmt, ##args) +#define host_pr_info(fmt, args...) pr_info(fmt, ##args) + +#define host_dump_stack() dump_stack() +#define host_print_pt_regs(regs) print_pt_regs(regs) +#define host_print_all_TIRs(TIRs, nr_TIRs) \ + print_all_TIRs(TIRs, nr_TIRs) +#define host_print_tc_record(tcellar, num) \ + print_tc_record(tcellar, num) +#define host_print_all_TC(TC, TC_count) \ + print_all_TC(TC, TC_count) #elif defined(CONFIG_PARAVIRT_GUEST) /* it is paravirtualized host and guest */ #include diff --git a/arch/e2k/include/asm/kvm/hypercall.h b/arch/e2k/include/asm/kvm/hypercall.h index c6e167c..83cf8e5 100644 --- a/arch/e2k/include/asm/kvm/hypercall.h +++ b/arch/e2k/include/asm/kvm/hypercall.h @@ -213,6 +213,8 @@ static inline unsigned long generic_hypercall6(unsigned long nr, /* PCSP_hi register */ #define KVM_HCALL_SETUP_IDLE_TASK 12 /* setup current task of */ /* guest as task */ +#define KVM_HCALL_UPDATE_WD_PSIZE 13 /* write updated psize field */ + /* to the WD register */ #define KVM_HCALL_MOVE_TAGGED_DATA 15 /* move quad value from to */ #define KVM_HCALL_UNFREEZE_TRAPS 16 /* unfreeze TIRs & trap */ /* cellar */ @@ -236,7 +238,7 @@ static inline unsigned long generic_hypercall6(unsigned long nr, /* virtual addresses */ #define KVM_HCALL_MMU_PROBE 29 /* probe MMU entry or */ /* address */ -#define KVM_HCALL_FLUSH_ICACHE_RANGE 30 /* flush ICACHE range */ +#define KVM_HCALL_FLUSH_ICACHE_ALL 30 /* flush all ICACHE */ /* notify host kernel aboout switch to updated procedure stack on guest */ #define KVM_HCALL_SWITCH_TO_EXPANDED_PROC_STACK 31 /* notify host kernel aboout switch to updated procedure chain stack on guest */ @@ -312,6 +314,12 @@ HYPERVISOR_update_psp_hi(unsigned long psp_hi_value) return light_hypercall1(KVM_HCALL_UPDATE_PSP_HI, psp_hi_value); } +static inline unsigned long +HYPERVISOR_update_wd_psize(unsigned long psize_value) +{ + return light_hypercall1(KVM_HCALL_UPDATE_WD_PSIZE, psize_value); +} + static inline unsigned long HYPERVISOR_update_pcsp_hi(unsigned long pcsp_hi_value) { @@ -366,6 +374,7 @@ HYPERVISOR_inject_interrupt(void) { return light_hypercall0(KVM_HCALL_INJECT_INTERRUPT); } +extern unsigned long kvm_hypervisor_inject_interrupt(void); static inline unsigned long HYPERVISOR_virqs_handled(void) { @@ -411,10 +420,9 @@ HYPERVISOR_clear_dcache_l1_range(void *addr, size_t len) (unsigned long)addr, len); } static inline unsigned long -HYPERVISOR_flush_icache_range(e2k_addr_t start, e2k_addr_t end, u64 dummy) +HYPERVISOR_flush_icache_all(void) { - return light_hypercall3(KVM_HCALL_FLUSH_ICACHE_RANGE, - start, end, dummy); + return light_hypercall0(KVM_HCALL_FLUSH_ICACHE_ALL); } typedef enum kvm_mmu_probe { @@ -599,8 +607,10 @@ HYPERVISOR_switch_to_expanded_guest_chain_stack(long delta_size, #define KVM_HCALL_PV_ENABLE_ASYNC_PF 133 /* enable async pf */ /* on current vcpu */ #endif /* CONFIG_KVM_ASYNC_PF */ -#define KVM_HCALL_FLUSH_TLB_RANGE 134 /* flush given address */ - /* range in tlb */ +#define KVM_HCALL_FLUSH_TLB_RANGE 134 /* sync given address range */ + /* in page tables and flush tlb */ +#define KVM_HCALL_SYNC_ADDR_RANGE 135 /* sync ptes in page */ + /* tables without flushing tlb */ #define KVM_HCALL_RECOVERY_FAULTED_TAGGED_STORE 141 /* recovery faulted store */ /* tagged value operations */ @@ -679,9 +689,11 @@ typedef struct kvm_task_info { unsigned long gd_size; /* and size */ unsigned long cut_base; /* CUTD: base */ unsigned long cut_size; /* and size */ - unsigned int cui; /* compilation unit index of code */ + int cui; /* compilation unit index of code */ + bool kernel; /* task in kernel mode */ unsigned long entry_point; /* entry point (address) of task */ - unsigned long tls; /* TLS of new user thread */ + unsigned long gregs; /* pointer to the global registers */ + /* state of the new process */ } kvm_task_info_t; /* hardware stack extention, update and change */ @@ -834,10 +846,11 @@ HYPERVISOR_complete_long_jump(kvm_long_jump_info_t *regs_state) } static inline unsigned long -HYPERVISOR_launch_sig_handler(kvm_stacks_info_t *regs_state, long sys_rval) +HYPERVISOR_launch_sig_handler(kvm_stacks_info_t *regs_state, + unsigned long sigreturn_entry, long sys_rval) { - return generic_hypercall2(KVM_HCALL_LAUNCH_SIG_HANDLER, - (unsigned long)regs_state, sys_rval); + return generic_hypercall3(KVM_HCALL_LAUNCH_SIG_HANDLER, + (unsigned long)regs_state, sigreturn_entry, sys_rval); } static inline unsigned long @@ -1005,16 +1018,16 @@ HYPERVISOR_set_guest_glob_regs_dirty_bgr(unsigned long *gregs[2], (unsigned long)false, (unsigned long)NULL); } static inline unsigned long -HYPERVISOR_get_guest_local_glob_regs(unsigned long *l_gregs[2]) +HYPERVISOR_get_guest_local_glob_regs(unsigned long *l_gregs[2], bool is_signal) { - return generic_hypercall1(KVM_HCALL_GET_GUEST_LOCAL_GLOB_REGS, - (unsigned long)l_gregs); + return generic_hypercall2(KVM_HCALL_GET_GUEST_LOCAL_GLOB_REGS, + (unsigned long)l_gregs, is_signal); } static inline unsigned long -HYPERVISOR_set_guest_local_glob_regs(unsigned long *l_gregs[2]) +HYPERVISOR_set_guest_local_glob_regs(unsigned long *l_gregs[2], bool is_signal) { - return generic_hypercall1(KVM_HCALL_SET_GUEST_LOCAL_GLOB_REGS, - (unsigned long)l_gregs); + return generic_hypercall2(KVM_HCALL_SET_GUEST_LOCAL_GLOB_REGS, + (unsigned long)l_gregs, is_signal); } static inline unsigned long @@ -1051,7 +1064,7 @@ HYPERVISOR_recovery_faulted_guest_load(e2k_addr_t address, static inline unsigned long HYPERVISOR_recovery_faulted_guest_move(e2k_addr_t addr_from, e2k_addr_t addr_to, e2k_addr_t addr_to_hi, int vr, u64 ld_rec_opc, int chan, - int qp_load, int atomic_load) + int qp_load, int atomic_load, u32 first_time) { union recovery_faulted_arg arg = { .vr = vr, @@ -1059,9 +1072,9 @@ HYPERVISOR_recovery_faulted_guest_move(e2k_addr_t addr_from, e2k_addr_t addr_to, .qp = !!qp_load, .atomic = !!atomic_load }; - return generic_hypercall5(KVM_HCALL_RECOVERY_FAULTED_GUEST_MOVE, + return generic_hypercall6(KVM_HCALL_RECOVERY_FAULTED_GUEST_MOVE, addr_from, addr_to, addr_to_hi, - ld_rec_opc, arg.entire); + ld_rec_opc, arg.entire, first_time); } static inline unsigned long HYPERVISOR_recovery_faulted_load_to_guest_greg(e2k_addr_t address, @@ -1108,7 +1121,7 @@ HYPERVISOR_recovery_faulted_load(e2k_addr_t address, u64 *ld_val, static inline unsigned long HYPERVISOR_recovery_faulted_move(e2k_addr_t addr_from, e2k_addr_t addr_to, e2k_addr_t addr_to_hi, int vr, u64 ld_rec_opc, int chan, - int qp_load, int atomic_load) + int qp_load, int atomic_load, u32 first_time) { union recovery_faulted_arg arg = { .vr = vr, @@ -1116,9 +1129,9 @@ HYPERVISOR_recovery_faulted_move(e2k_addr_t addr_from, e2k_addr_t addr_to, .qp = !!qp_load, .atomic = !!atomic_load }; - return generic_hypercall5(KVM_HCALL_RECOVERY_FAULTED_MOVE, + return generic_hypercall6(KVM_HCALL_RECOVERY_FAULTED_MOVE, addr_from, addr_to, addr_to_hi, - ld_rec_opc, arg.entire); + ld_rec_opc, arg.entire, first_time); } static inline unsigned long HYPERVISOR_recovery_faulted_load_to_greg(e2k_addr_t address, u32 greg_num_d, @@ -1449,6 +1462,12 @@ HYPERVISOR_flush_tlb_range(e2k_addr_t start_gva, e2k_addr_t end_gva) return generic_hypercall2(KVM_HCALL_FLUSH_TLB_RANGE, start_gva, end_gva); } +static inline void +HYPERVISOR_sync_addr_range(e2k_addr_t start_gva, e2k_addr_t end_gva) +{ + generic_hypercall2(KVM_HCALL_SYNC_ADDR_RANGE, + start_gva, end_gva); +} /* * arguments: diff --git a/arch/e2k/include/asm/kvm/machdep.h b/arch/e2k/include/asm/kvm/machdep.h index f589aa8..0c8584d 100644 --- a/arch/e2k/include/asm/kvm/machdep.h +++ b/arch/e2k/include/asm/kvm/machdep.h @@ -22,8 +22,10 @@ extern void kvm_save_host_gregs_v2(struct host_gregs *gregs); extern void kvm_save_host_gregs_v5(struct host_gregs *gregs); extern void kvm_restore_host_gregs_v5(const struct host_gregs *gregs); -extern void kvm_guest_save_local_gregs_v2(struct local_gregs *gregs); -extern void kvm_guest_save_local_gregs_v5(struct local_gregs *gregs); +extern void kvm_guest_save_local_gregs_v2(struct local_gregs *gregs, + bool is_signal); +extern void kvm_guest_save_local_gregs_v5(struct local_gregs *gregs, + bool is_signal); extern void kvm_guest_save_kernel_gregs_v2(kernel_gregs_t *gregs); extern void kvm_guest_save_kernel_gregs_v5(kernel_gregs_t *gregs); extern void kvm_guest_save_gregs_v2(struct global_regs *gregs); @@ -34,8 +36,10 @@ extern void kvm_guest_restore_gregs_v2(const global_regs_t *gregs); extern void kvm_guest_restore_gregs_v5(const global_regs_t *gregs); extern void kvm_guest_restore_kernel_gregs_v2(global_regs_t *gregs); extern void kvm_guest_restore_kernel_gregs_v5(global_regs_t *gregs); -extern void kvm_guest_restore_local_gregs_v2(const struct local_gregs *gregs); -extern void kvm_guest_restore_local_gregs_v5(const struct local_gregs *gregs); +extern void kvm_guest_restore_local_gregs_v2(const struct local_gregs *gregs, + bool is_signal); +extern void kvm_guest_restore_local_gregs_v5(const struct local_gregs *gregs, + bool is_signal); #if defined(CONFIG_PARAVIRT_GUEST) || defined(CONFIG_KVM_GUEST_KERNEL) /* it is paravirtualized host and guest kernel */ diff --git a/arch/e2k/include/asm/kvm/mm.h b/arch/e2k/include/asm/kvm/mm.h index a739c31..c1c2313 100644 --- a/arch/e2k/include/asm/kvm/mm.h +++ b/arch/e2k/include/asm/kvm/mm.h @@ -2,6 +2,7 @@ #define __E2K_KVM_HOST_MM_H #include +#include #include #include @@ -27,6 +28,13 @@ typedef struct gmm_struct { /* of guest mm structure */ atomic_t mm_count; /* How many references to guest mm */ /* shared mm */ +#ifdef CONFIG_GUEST_MM_SPT_LIST + struct list_head spt_list; /* shadow page tables pages list */ + spinlock_t spt_list_lock; /* spin lock to access to list */ + size_t spt_list_size; /* current numbers of SPs in list */ + size_t total_released; /* total number of allocated and */ + /* released SPs through list */ +#endif /* CONFIG_GUEST_MM_SPT_LIST */ #ifdef CONFIG_KVM_HV_MMU hpa_t root_hpa; /* physical base of root shadow PT */ /* for guest mm on host */ @@ -44,8 +52,6 @@ typedef struct gmm_struct { /* the guest mm */ cpumask_t cpu_vm_mask; /* mask of CPUs where the mm is */ /* in use or was some early */ - bool in_release; /* guest mm is in release and cannot */ - /* be used as active */ } gmm_struct_t; /* same as accessor for struct mm_struct's cpu_vm_mask but for guest mm */ diff --git a/arch/e2k/include/asm/kvm/mmu.h b/arch/e2k/include/asm/kvm/mmu.h index 866c176..4993a2b 100644 --- a/arch/e2k/include/asm/kvm/mmu.h +++ b/arch/e2k/include/asm/kvm/mmu.h @@ -390,14 +390,15 @@ kvm_mmu_instr_page_fault(struct kvm_vcpu *vcpu, gva_t address, #endif /* CONFIG_KVM_SHADOW_PT_ENABLE */ extern int kvm_guest_addr_to_host(void **addr); -extern void *kvm_guest_ptr_to_host_ptr(void *guest_ptr, int size); +extern void *kvm_guest_ptr_to_host_ptr(void *guest_ptr, int size, + bool need_inject); #ifdef CONFIG_KVM_HOST_MODE /* it is native host kernel with virtualization support */ static inline int -guest_addr_to_host(void **addr, pt_regs_t *regs) +guest_addr_to_host(void **addr, const pt_regs_t *regs) { - if (likely(!host_test_intc_emul_mode((const struct pt_regs *)regs))) { + if (likely(!host_test_intc_emul_mode(regs))) { /* faulted addres is not paravirtualized guest one */ return native_guest_addr_to_host(addr); } @@ -405,14 +406,14 @@ guest_addr_to_host(void **addr, pt_regs_t *regs) return kvm_guest_addr_to_host(addr); } static inline void * -guest_ptr_to_host(void *ptr, int size, pt_regs_t *regs) +guest_ptr_to_host(void *ptr, int size, const pt_regs_t *regs) { - if (likely(!host_test_intc_emul_mode((const struct pt_regs *)regs))) { + if (likely(!host_test_intc_emul_mode(regs))) { /* faulted addres is not paravirtualized guest one */ return native_guest_ptr_to_host(ptr, size); } - return kvm_guest_ptr_to_host_ptr(ptr, size); + return kvm_guest_ptr_to_host_ptr(ptr, size, false); } #endif /* CONFIG_KVM_HOST_MODE */ diff --git a/arch/e2k/include/asm/kvm/mmu_regs_access.h b/arch/e2k/include/asm/kvm/mmu_regs_access.h index a160fae..3b78f6d 100644 --- a/arch/e2k/include/asm/kvm/mmu_regs_access.h +++ b/arch/e2k/include/asm/kvm/mmu_regs_access.h @@ -411,6 +411,16 @@ KVM_READ_CLW_REG(clw_addr_t clw_addr) return -1; } +/* + * Write CLW register + */ + +static inline void +KVM_WRITE_CLW_REG(clw_addr_t clw_addr, clw_reg_t val) +{ + panic("KVM_WRITE_CLW_REG() is not yet implemented\n"); +} + /* * KVM MMU DEBUG registers access */ @@ -731,6 +741,16 @@ READ_CLW_REG(clw_addr_t clw_addr) return KVM_READ_CLW_REG(clw_addr); } +/* + * Write CLW register + */ + +static inline void +WRITE_CLW_REG(clw_addr_t clw_addr, clw_reg_t val) +{ + KVM_WRITE_CLW_REG(clw_addr, val); +} + /* * KVM MMU DEBUG registers access */ diff --git a/arch/e2k/include/asm/kvm/process.h b/arch/e2k/include/asm/kvm/process.h index a26e06f..1b9e77e 100644 --- a/arch/e2k/include/asm/kvm/process.h +++ b/arch/e2k/include/asm/kvm/process.h @@ -131,27 +131,27 @@ do { \ }) static inline void -kvm_clear_virt_thread_struct(thread_info_t *thread_info) +kvm_clear_virt_thread_struct(thread_info_t *ti) { - thread_info->gpid_nr = -1; /* cannot inherit, only set by */ - /* guest/host kernel */ -#ifdef CONFIG_KVM_HOST_MODE - /* clear KVM support fields and flags */ - if (test_ti_thread_flag(thread_info, TIF_VIRTUALIZED_HOST) || - test_ti_thread_flag(thread_info, TIF_VIRTUALIZED_GUEST)) - /* It is clone() on host to create guest */ - /* VCPU or VIRQ VCPU threads */ - kvm_clear_host_thread_info(thread_info); - if (thread_info->gthread_info) { - /* It is guest thread: clear from old process */ - thread_info->gthread_info = NULL; - /* kvm_pv_clear_guest_thread_info(thread_info->gthread_info); */ + if (likely(ti->vcpu == NULL)) { + /* it is not creation of host process */ + /* to support virtualization */ + return; } - /* VCPU host/guest thread flags and VCPU structure cannot inherit */ - /* only to pass */ - clear_ti_thread_flag(thread_info, TIF_VIRTUALIZED_HOST); - thread_info->vcpu = NULL; -#endif /* CONFIG_KVM_HOST_MODE */ + + /* + * Host VCPU thread can be only created by user process (for example + * by qemu) and only user process can clone the thread to handle + * some VCPU running exit reasons. + * But the new thread cannot be one more host VCPU thread, + * so clean up all about VCPU + */ + + /* VCPU thread should be only at host mode (handle exit reason), */ + /* not at running VCPU mode */ + KVM_BUG_ON(test_ti_status_flag(ti, TS_HOST_AT_VCPU_MODE)); + + ti->gthread_info = NULL; } #if !defined(CONFIG_PARAVIRT_GUEST) && !defined(CONFIG_KVM_GUEST_KERNEL) @@ -618,6 +618,14 @@ pv_vcpu_user_hw_stacks_prepare(struct kvm_vcpu *vcpu, pt_regs_t *regs, do_exit(SIGKILL); } +/* Same as for native kernel without virtualization support */ +static __always_inline int +user_hw_stacks_copy(struct e2k_stacks *stacks, + pt_regs_t *regs, u64 cur_window_q, bool copy_full) +{ + return native_user_hw_stacks_copy(stacks, regs, cur_window_q, copy_full); +} + static __always_inline void host_user_hw_stacks_prepare(struct e2k_stacks *stacks, pt_regs_t *regs, u64 cur_window_q, enum restore_caller from, int syscall) @@ -635,7 +643,54 @@ host_user_hw_stacks_prepare(struct e2k_stacks *stacks, pt_regs_t *regs, pv_vcpu_user_hw_stacks_prepare(vcpu, regs, cur_window_q, from, syscall); } -#define SAVE_HOST_KERNEL_GREGS_COPY_TO(__k_gregs, __g_gregs) \ +static __always_inline void +host_exit_to_usermode_loop(struct pt_regs *regs, bool syscall, bool has_signal) +{ + KVM_BUG_ON(!host_test_intc_emul_mode(regs)); + + WRITE_PSR_IRQ_BARRIER(AW(E2K_KERNEL_PSR_ENABLED)); + + /* Check for rescheduling first */ + if (need_resched()) { + schedule(); + } + + if (has_signal) { + /* + * This is guest VCPU interception emulation, but + * there is (are) pending signal for host VCPU mode, + * so it need switch to host VCPU mode to handle + * signal and probably to kill VM + */ + WRITE_PSR_IRQ_BARRIER(AW(E2K_KERNEL_PSR_DISABLED)); + pv_vcpu_switch_to_host_from_intc(current_thread_info()); + } else if (likely(guest_trap_pending(current_thread_info()))) { + /* + * This is guest VCPU interception emulation and + * there is (are) the guest trap(s) to handle + */ + insert_pv_vcpu_traps(current_thread_info(), regs); + } else { + /* + * This is just a return from VCPU interception + * emulation mode to the continue execution + * of the guest paravirtualized VCPU. + * In such case: + * - the currents point to the host qemu-VCPU + * process structures; + * - the regs points to the host guest-VCPU + * process structure. + * So nothing works based on these non-interconnected + * structures cannot be running + */ + } + + WRITE_PSR_IRQ_BARRIER(AW(E2K_KERNEL_PSR_DISABLED)); +} + +#ifdef CONFIG_SMP +#define SAVE_GUEST_KERNEL_GREGS_COPY_TO(__k_gregs, __g_gregs, \ + only_kernel) \ ({ \ kernel_gregs_t *kg = (__k_gregs); \ kernel_gregs_t *gg = (__g_gregs); \ @@ -643,19 +698,45 @@ host_user_hw_stacks_prepare(struct e2k_stacks *stacks, pt_regs_t *regs, unsigned long cpu_id__; \ unsigned long cpu_off__; \ \ + if (likely(!(only_kernel))) { \ + unsigned long vs__; \ + \ + HOST_ONLY_SAVE_VCPU_STATE_GREG(vs__); \ + HOST_ONLY_COPY_TO_VCPU_STATE_GREG(gg, vs__); \ + } \ ONLY_COPY_FROM_KERNEL_GREGS(kg, task__, cpu_id__, cpu_off__); \ ONLY_COPY_TO_KERNEL_GREGS(gg, task__, cpu_id__, cpu_off__); \ }) +#else /* ! CONFIG_SMP */ +#define SAVE_GUEST_KERNEL_GREGS_COPY_TO(__k_gregs, __g_gregs, \ + only_kernel) \ +({ \ + kernel_gregs_t *kg = (__k_gregs); \ + kernel_gregs_t *gg = (__g_gregs); \ + unsigned long task__; \ + \ + if (likely(!(only_kernel))) { \ + unsigned long vs__; \ + \ + HOST_ONLY_SAVE_VCPU_STATE_GREG(vs__); \ + HOST_ONLY_COPY_TO_VCPU_STATE_GREG(gg, vs__); \ + } \ + ONLY_COPY_FROM_KERNEL_CURRENT_GREGS(kg, task__); \ + ONLY_COPY_TO_KERNEL_CURRENT_GREGS(gg, task__); \ +}) +#endif /* CONFIG_SMP */ -#define SAVE_HOST_KERNEL_GREGS_COPY(__ti, __gti) \ +#define SAVE_GUEST_KERNEL_GREGS_COPY(__ti, __gti) \ ({ \ kernel_gregs_t *k_gregs = &(__ti)->k_gregs_light; \ - kernel_gregs_t *g_gregs = &(__gti)->g_gregs; \ + kernel_gregs_t *g_gregs = &(__gti)->gk_gregs; \ \ - SAVE_HOST_KERNEL_GREGS_COPY_TO(k_gregs, g_gregs); \ + SAVE_GUEST_KERNEL_GREGS_COPY_TO(k_gregs, g_gregs, false); \ }) -#define RESTORE_HOST_KERNEL_GREGS_COPY_FROM(__k_gregs, __g_gregs) \ +#ifdef CONFIG_SMP +#define RESTORE_GUEST_KERNEL_GREGS_COPY_FROM(__k_gregs, __g_gregs, \ + only_kernel) \ ({ \ kernel_gregs_t *kg = (__k_gregs); \ kernel_gregs_t *gg = (__g_gregs); \ @@ -663,16 +744,40 @@ host_user_hw_stacks_prepare(struct e2k_stacks *stacks, pt_regs_t *regs, unsigned long cpu_id__; \ unsigned long cpu_off__; \ \ + if (likely(!(only_kernel))) { \ + unsigned long vs__; \ + \ + HOST_ONLY_COPY_FROM_VCPU_STATE_GREG(k_gregs, vs__); \ + HOST_ONLY_RESTORE_VCPU_STATE_GREG(vs__); \ + } \ ONLY_COPY_FROM_KERNEL_GREGS(gg, task__, cpu_id__, cpu_off__); \ ONLY_COPY_TO_KERNEL_GREGS(kg, task__, cpu_id__, cpu_off__); \ }) +#else /* ! CONFIG_SMP */ +#define RESTORE_GUEST_KERNEL_GREGS_COPY_FROM(__k_gregs, __g_gregs, \ + only_kernel) \ +({ \ + kernel_gregs_t *kg = (__k_gregs); \ + kernel_gregs_t *gg = (__g_gregs); \ + unsigned long task__; \ + \ + if (likely(!(only_kernel))) { \ + unsigned long vs__; \ + \ + HOST_ONLY_COPY_FROM_VCPU_STATE_GREG(k_gregs, vs__); \ + HOST_ONLY_RESTORE_VCPU_STATE_GREG(vs__); \ + } \ + ONLY_COPY_FROM_KERNEL_CURRENT_GREGS(gg, task__); \ + ONLY_COPY_TO_KERNEL_CURRENT_GREGS(kg, task__); \ +}) +#endif /* CONFIG_SMP */ -#define RESTORE_HOST_KERNEL_GREGS_COPY(__ti, __gti, __vcpu) \ +#define RESTORE_GUEST_KERNEL_GREGS_COPY(__ti, __gti, __vcpu) \ ({ \ kernel_gregs_t *k_gregs = &(__ti)->k_gregs; \ - kernel_gregs_t *g_gregs = &(__gti)->g_gregs; \ + kernel_gregs_t *g_gregs = &(__gti)->gu_gregs; \ \ - RESTORE_HOST_KERNEL_GREGS_COPY_FROM(k_gregs, g_gregs); \ + RESTORE_GUEST_KERNEL_GREGS_COPY_FROM(k_gregs, g_gregs, true); \ INIT_HOST_VCPU_STATE_GREG_COPY(__ti, __vcpu); \ }) diff --git a/arch/e2k/include/asm/kvm/ptrace.h b/arch/e2k/include/asm/kvm/ptrace.h index c05b345..6471832 100644 --- a/arch/e2k/include/asm/kvm/ptrace.h +++ b/arch/e2k/include/asm/kvm/ptrace.h @@ -557,8 +557,7 @@ check_is_user_address(struct task_struct *task, e2k_addr_t address) pt_regs_t *__regs = (pt_regs); \ bool is_ligh_hypercall; \ \ - is_ligh_hypercall = \ - (__regs->flags & LIGHT_HYPERCALL_FLAG_PT_REGS) != 0; \ + is_ligh_hypercall = __regs->flags.light_hypercall; \ is_ligh_hypercall; \ }) #define TI_LIGHT_HYPERCALL_MODE(thread_info) \ @@ -596,6 +595,7 @@ typedef struct pv_vcpu_ctxt { e2k_psr_t guest_psr; /* guest PSR state before trap */ bool irq_under_upsr; /* is IRQ control under UOSR? */ bool in_sig_handler; /* signal handler in progress */ + unsigned long sigreturn_entry; /* guest signal return start IP */ } pv_vcpu_ctxt_t; #else /* !CONFIG_KVM_HOST_MODE */ diff --git a/arch/e2k/include/asm/kvm/pv-emul.h b/arch/e2k/include/asm/kvm/pv-emul.h index fd8b854..02de005 100644 --- a/arch/e2k/include/asm/kvm/pv-emul.h +++ b/arch/e2k/include/asm/kvm/pv-emul.h @@ -11,29 +11,22 @@ #include #ifdef CONFIG_VIRTUALIZATION - -static __always_inline bool -kvm_host_at_pv_vcpu_mode(thread_info_t *ti) -{ - return ti->vcpu && test_ti_thread_flag(ti, TIF_HOST_AT_VCPU_MODE); -} - static __always_inline void kvm_set_intc_emul_flag(pt_regs_t *regs) { - regs->flags |= TRAP_AS_INTC_EMUL_PT_REGS; + regs->flags.trap_as_intc_emul = 1; } static __always_inline bool kvm_test_intc_emul_flag(pt_regs_t *regs) { - return !!(regs->flags & TRAP_AS_INTC_EMUL_PT_REGS); + return regs->flags.trap_as_intc_emul; } static __always_inline void kvm_clear_intc_emul_flag(pt_regs_t *regs) { - regs->flags &= ~TRAP_AS_INTC_EMUL_PT_REGS; + regs->flags.trap_as_intc_emul = 0; } static __always_inline bool @@ -59,6 +52,15 @@ host_test_intc_emul_mode(const struct pt_regs *regs) return true; } + +extern void pv_vcpu_switch_to_host_from_intc(thread_info_t *ti); +extern void pv_vcpu_return_to_intc_mode(thread_info_t *ti, struct kvm_vcpu *vcpu); + +static inline void return_to_pv_vcpu_intc(struct kvm_vcpu *vcpu) +{ + pv_vcpu_return_to_intc_mode(current_thread_info(), vcpu); +} + #else /* !CONFIG_KVM_HOST_MODE */ /* it is not host kernel */ static inline bool @@ -66,6 +68,12 @@ host_test_intc_emul_mode(const pt_regs_t *regs) { return false; } + +static inline __interrupt void +pv_vcpu_switch_to_host_from_intc(thread_info_t *ti) +{ + /* nothing to do */ +} #endif /* CONFIG_KVM_HOST_MODE */ static inline int kvm_get_vcpu_intc_TIRs_num(struct kvm_vcpu *vcpu) @@ -113,6 +121,8 @@ kvm_clear_vcpu_guest_stacks_pending(struct kvm_vcpu *vcpu, pt_regs_t *regs) } extern noinline void insert_pv_vcpu_traps(thread_info_t *ti, pt_regs_t *regs); +extern void insert_pv_vcpu_sigreturn(struct kvm_vcpu *vcpu, + pv_vcpu_ctxt_t *vcpu_ctxt, pt_regs_t *regs); extern void kvm_emulate_pv_vcpu_intc(struct thread_info *ti, pt_regs_t *regs, trap_pt_regs_t *trap); @@ -235,12 +245,6 @@ static inline mm_context_t *pv_vcpu_get_gmm_context(struct kvm_vcpu *vcpu) } #else /* !CONFIG_VIRTUALIZATION */ -static __always_inline bool -kvm_host_at_pv_vcpu_mode(thread_info_t *ti) -{ - return false; -} - static __always_inline void kvm_set_intc_emul_flag(pt_regs_t *regs) { diff --git a/arch/e2k/include/asm/kvm/regs_state.h b/arch/e2k/include/asm/kvm/regs_state.h index 85df1aa..01970e1 100644 --- a/arch/e2k/include/asm/kvm/regs_state.h +++ b/arch/e2k/include/asm/kvm/regs_state.h @@ -408,7 +408,7 @@ do { \ e2k_cutd_t cutd; \ struct kvm_vcpu *vcpu; \ \ - if (likely(!test_ti_thread_flag((ti), TIF_HOST_AT_VCPU_MODE))) { \ + if (likely(!test_ti_status_flag((ti), TS_HOST_AT_VCPU_MODE))) { \ /* host at native or hypervisor mode */ \ /* so CUT context is alredy set */ \ break; \ @@ -438,11 +438,6 @@ do { \ #error "Undefined virtualization mode" #endif /* !CONFIG_VIRTUALIZATION */ -#define RESTORE_USER_TRAP_CUT_REGS(ti, regs) \ - RESTORE_USER_CUT_REGS(ti, regs, false) -#define RESTORE_USER_SYSCALL_CUT_REGS(ti, regs) \ - RESTORE_USER_CUT_REGS(ti, regs, true) - #if defined(CONFIG_KVM) || defined(CONFIG_KVM_GUEST_KERNEL) #define SAVE_GUEST_VCPU_STATE_GREGS(regs) \ ({ \ diff --git a/arch/e2k/include/asm/kvm/switch.h b/arch/e2k/include/asm/kvm/switch.h index e38861c..7f39f8b 100644 --- a/arch/e2k/include/asm/kvm/switch.h +++ b/arch/e2k/include/asm/kvm/switch.h @@ -8,9 +8,12 @@ #include #include #include +#include #include #include +#define DEBUG_UPSR_FP_DISABLE + /* * See below the 'flags' argument of xxx_guest_enter()/xxx_guest_exit() */ @@ -23,10 +26,12 @@ #define DONT_CU_REGS_SWITCH 0x0010U /* do not save/restore CUT and CU */ /* registers */ #define DONT_MMU_CONTEXT_SWITCH 0x0020U /* do not switch MMU context */ -#define DONT_SAVE_GREGS_SWITCH 0x0040U /* do not save global regs */ +#define DONT_SAVE_KGREGS_SWITCH 0x0040U /* do not save and set kernel global */ + /* regs */ #define DONT_AAU_CONTEXT_SWITCH 0x0080U /* do not switch AAU context */ -#define EXIT_FROM_INTC_SWITCH 0x0100U /* complete intercept emulation mode */ -#define EXIT_FROM_TRAP_SWITCH 0x0200U /* complete trap mode */ +#define DONT_TRAP_MASK_SWITCH 0x0100U /* do not switch OSEM context */ +#define EXIT_FROM_INTC_SWITCH 0x1000U /* complete intercept emulation mode */ +#define EXIT_FROM_TRAP_SWITCH 0x2000U /* complete trap mode */ static inline void native_trap_guest_enter(struct thread_info *ti, struct pt_regs *regs, @@ -77,8 +82,7 @@ native_trap_guest_get_restore_stacks(struct thread_info *ti, } static inline struct e2k_stacks * -native_syscall_guest_get_restore_stacks(struct thread_info *ti, - struct pt_regs *regs) +native_syscall_guest_get_restore_stacks(struct pt_regs *regs) { return ®s->stacks; } @@ -87,55 +91,15 @@ native_syscall_guest_get_restore_stacks(struct thread_info *ti, * The function should return bool is the system call from guest */ static inline bool -native_guest_syscall_enter(struct thread_info *ti, struct pt_regs *regs) +native_guest_syscall_enter(struct pt_regs *regs) { /* nothing guests can be */ return false; /* it is not guest system call */ } -static inline void -native_guest_syscall_exit_to(struct thread_info *ti, struct pt_regs *regs, - unsigned flags) -{ - /* nothing guests can be */ -} #ifdef CONFIG_VIRTUALIZATION -/* - * Normally data stack is switched on interceptions as follows: - * 1) Upon interception guest's USD_hi.size is saved into backup - * stacks (cr1_lo.ussz field). - * 2) Then hardware switches PCSP stack (see Phase 5) and does an - * equivalent of DONE which modifies guest's USD with 'cr1_lo.ussz' - * from the function that called GLAUNCH. - * 3) Hypervisor in software saves this modified USD and restores it - * before GLAUNCH. - * 4) Hardware in GLAUNCH switches PCSP stack (see Phase 4) - * 5) Hardware in GLAUNCH does an equivalent of DONE (see Phase 6) - * which restores proper guest USD. - * - * But if hypervisor sets VIRT_CTRL_CU.glnch.g_th then that DONE is - * skipped and guest's data stack is incorrect. So we manually do - * here what DONE does. For simplicity do it always although it - * actually is needed only in 'g_th' case. - */ -static inline void kvm_correct_guest_data_stack_regs( - struct kvm_sw_cpu_context *sw_ctxt, e2k_cr1_hi_t cr1_hi) -{ - e2k_usd_lo_t usd_lo; - e2k_usd_hi_t usd_hi; - e2k_size_t real_size; - - real_size = cr1_hi.CR1_hi_ussz << 4; - usd_hi = sw_ctxt->usd_hi; - usd_lo = sw_ctxt->usd_lo; - usd_lo.USD_lo_base += (real_size - usd_hi.USD_hi_size); - usd_hi.USD_hi_size = real_size; - sw_ctxt->usd_lo = usd_lo; - sw_ctxt->usd_hi = usd_hi; -} - /* * For interceptions just switch actual registers with saved values * in 'sw_ctxt'. @@ -187,9 +151,6 @@ static inline void kvm_switch_stack_regs(struct kvm_sw_cpu_context *sw_ctxt, } } -#define Compiler_bug_128308_workaround - -#ifndef Compiler_bug_128308_workaround static inline void kvm_switch_fpu_regs(struct kvm_sw_cpu_context *sw_ctxt) { e2k_fpcr_t fpcr; @@ -212,9 +173,6 @@ static inline void kvm_switch_fpu_regs(struct kvm_sw_cpu_context *sw_ctxt) sw_ctxt->pfpfr = pfpfr; sw_ctxt->upsr = upsr; } -#else /* Compiler_bug_128308_workaround */ -extern noinline void kvm_switch_fpu_regs(struct kvm_sw_cpu_context *sw_ctxt); -#endif /* !Compiler_bug_128308_workaround */ static inline void kvm_switch_cu_regs(struct kvm_sw_cpu_context *sw_ctxt) { @@ -377,6 +335,39 @@ static inline void kvm_switch_debug_regs(struct kvm_sw_cpu_context *sw_ctxt, } +#ifdef CONFIG_CLW_ENABLE +static inline void kvm_switch_clw_regs(struct kvm_sw_cpu_context *sw_ctxt, bool guest_enter) +{ + if (guest_enter) { + native_write_US_CL_B(sw_ctxt->us_cl_b); + native_write_US_CL_UP(sw_ctxt->us_cl_up); + native_write_US_CL_M0(sw_ctxt->us_cl_m0); + native_write_US_CL_M1(sw_ctxt->us_cl_m1); + native_write_US_CL_M2(sw_ctxt->us_cl_m2); + native_write_US_CL_M3(sw_ctxt->us_cl_m3); + + NATIVE_WRITE_MMU_US_CL_D(sw_ctxt->us_cl_d); + } else { + sw_ctxt->us_cl_d = NATIVE_READ_MMU_US_CL_D(); + + DISABLE_US_CLW(); + + sw_ctxt->us_cl_b = native_read_US_CL_B(); + sw_ctxt->us_cl_up = native_read_US_CL_UP(); + sw_ctxt->us_cl_m0 = native_read_US_CL_M0(); + sw_ctxt->us_cl_m1 = native_read_US_CL_M1(); + sw_ctxt->us_cl_m2 = native_read_US_CL_M2(); + sw_ctxt->us_cl_m3 = native_read_US_CL_M3(); + } +} +#else +static inline void kvm_switch_clw_regs(struct kvm_sw_cpu_context *sw_ctxt, bool guest_enter) +{ + /* Nothing to do */ +} +#endif + + static inline void switch_ctxt_trap_enable_mask(struct kvm_sw_cpu_context *sw_ctxt) { @@ -392,11 +383,23 @@ static inline void host_guest_enter(struct thread_info *ti, { struct kvm_sw_cpu_context *sw_ctxt = &vcpu->sw_ctxt; - switch_ctxt_trap_enable_mask(sw_ctxt); - /* In full virtualization mode guest sets his own OSEM in thread_init() */ - if (!vcpu->is_hv) - KVM_BUG_ON((NATIVE_READ_OSEM_REG_VALUE() & HYPERCALLS_TRAPS_MASK) != - HYPERCALLS_TRAPS_MASK); + if (likely(!(flags & DONT_TRAP_MASK_SWITCH))) { + switch_ctxt_trap_enable_mask(sw_ctxt); + /* In full virtualization mode guest sets his own OSEM */ + /* in thread_init() */ + if (!vcpu->is_hv) { + KVM_BUG_ON((NATIVE_READ_OSEM_REG_VALUE() & + HYPERCALLS_TRAPS_MASK) != + HYPERCALLS_TRAPS_MASK); + } + } else { + /* In full virtualization mode guest sets his own OSEM */ + /* in thread_init() */ + if (!vcpu->is_hv) { + KVM_BUG_ON((NATIVE_READ_OSEM_REG_VALUE() & + HYPERCALLS_TRAPS_MASK) != 0); + } + } if (flags & FROM_HYPERCALL_SWITCH) { /* @@ -427,8 +430,13 @@ static inline void host_guest_enter(struct thread_info *ti, machine.calculate_aau_aaldis_aaldas(NULL, ti, &sw_ctxt->aau_context); #endif - /* For interceptions restore extended part */ - NATIVE_RESTORE_KERNEL_GREGS(&ti->k_gregs); + if (machine.flushts) + machine.flushts(); + + if (likely(!(flags & DONT_SAVE_KGREGS_SWITCH))) { + /* For interceptions restore extended part */ + NATIVE_RESTORE_KERNEL_GREGS(&ti->k_gregs); + } NATIVE_RESTORE_INTEL_REGS(sw_ctxt); @@ -438,7 +446,9 @@ static inline void host_guest_enter(struct thread_info *ti, * the list in sw_ctxt definition */ kvm_switch_fpu_regs(sw_ctxt); kvm_switch_cu_regs(sw_ctxt); - kvm_switch_mmu_regs(sw_ctxt, vcpu->is_hv); + if (likely(!(flags & DONT_MMU_CONTEXT_SWITCH))) { + kvm_switch_mmu_regs(sw_ctxt, vcpu->is_hv); + } #ifdef CONFIG_USE_AAU if (!(flags & DONT_AAU_CONTEXT_SWITCH)) { @@ -477,6 +487,9 @@ static inline void host_guest_enter(struct thread_info *ti, /* restore saved source pointers of host stack */ kvm_switch_stack_regs(sw_ctxt, false, true); } + + if (vcpu->is_hv) + kvm_switch_clw_regs(sw_ctxt, true); } } @@ -509,11 +522,16 @@ static inline void host_guest_exit(struct thread_info *ti, { struct kvm_sw_cpu_context *sw_ctxt = &vcpu->sw_ctxt; - switch_ctxt_trap_enable_mask(sw_ctxt); + if (likely(!(flags & DONT_TRAP_MASK_SWITCH))) { + switch_ctxt_trap_enable_mask(sw_ctxt); + } KVM_BUG_ON(NATIVE_READ_OSEM_REG_VALUE() & HYPERCALLS_TRAPS_MASK); /* Switch data stack before all function calls */ if (flags & USD_CONTEXT_SWITCH) { + if (vcpu->is_hv) + kvm_switch_clw_regs(sw_ctxt, false); + if (!(flags & FROM_HYPERCALL_SWITCH) || !vcpu->is_hv) { kvm_switch_stack_regs(sw_ctxt, false, false); } else { @@ -593,21 +611,23 @@ static inline void host_guest_exit(struct thread_info *ti, if (cpu_has(CPU_HWBUG_L1I_STOPS_WORKING)) E2K_DISP_CTPRS(); - /* For interceptions save extended part. */ - machine.save_kernel_gregs(&ti->k_gregs); - ONLY_SET_KERNEL_GREGS(ti); + if (likely(!(flags & DONT_SAVE_KGREGS_SWITCH))) { + /* For interceptions save extended part. */ + machine.save_kernel_gregs(&ti->k_gregs); + ONLY_SET_KERNEL_GREGS(ti); + } NATIVE_SAVE_INTEL_REGS(sw_ctxt); #ifdef CONFIG_MLT_STORAGE machine.invalidate_MLT(); #endif - if (machine.flushts) - machine.flushts(); /* Isolate from QEMU */ kvm_switch_fpu_regs(sw_ctxt); kvm_switch_cu_regs(sw_ctxt); - kvm_switch_mmu_regs(sw_ctxt, vcpu->is_hv); + if (likely(!(flags & DONT_MMU_CONTEXT_SWITCH))) { + kvm_switch_mmu_regs(sw_ctxt, vcpu->is_hv); + } } else { /* * Starting emulation of interseption of paravirtualized vcpu @@ -708,6 +728,7 @@ pv_vcpu_switch_guest_host_context(struct kvm_vcpu *vcpu, static inline void pv_vcpu_switch_host_context(struct kvm_vcpu *vcpu) { kvm_host_context_t *host_ctxt = &vcpu->arch.host_ctxt; + struct kvm_sw_cpu_context *sw_ctxt = &vcpu->arch.sw_ctxt; unsigned long *stack; pt_regs_t *regs; e2k_usd_hi_t k_usd_hi; @@ -717,13 +738,16 @@ static inline void pv_vcpu_switch_host_context(struct kvm_vcpu *vcpu) e2k_psp_hi_t k_psp_hi; e2k_pcsp_lo_t k_pcsp_lo; e2k_pcsp_hi_t k_pcsp_hi; + e2k_upsr_t upsr; unsigned long base; unsigned long size; unsigned long used; + unsigned osem; /* keep current state of context */ stack = current->stack; regs = current_thread_info()->pt_regs; + upsr = current_thread_info()->upsr; k_usd_lo = current_thread_info()->k_usd_lo; k_usd_hi = current_thread_info()->k_usd_hi; k_sbr.SBR_reg = (unsigned long)stack + KERNEL_C_STACK_SIZE + @@ -736,6 +760,7 @@ static inline void pv_vcpu_switch_host_context(struct kvm_vcpu *vcpu) /* restore VCPU thread context */ current->stack = host_ctxt->stack; current_thread_info()->pt_regs = host_ctxt->pt_regs; + current_thread_info()->upsr = host_ctxt->upsr; current_thread_info()->k_usd_hi = host_ctxt->k_usd_hi; current_thread_info()->k_usd_lo = host_ctxt->k_usd_lo; current_thread_info()->k_psp_lo = host_ctxt->k_psp_lo; @@ -746,6 +771,7 @@ static inline void pv_vcpu_switch_host_context(struct kvm_vcpu *vcpu) /* save VCPU thread context */ host_ctxt->stack = stack; host_ctxt->pt_regs = regs; + host_ctxt->upsr = upsr; host_ctxt->k_usd_lo = k_usd_lo; host_ctxt->k_usd_hi = k_usd_hi; host_ctxt->k_sbr = k_sbr; @@ -754,6 +780,11 @@ static inline void pv_vcpu_switch_host_context(struct kvm_vcpu *vcpu) host_ctxt->k_pcsp_lo = k_pcsp_lo; host_ctxt->k_pcsp_hi = k_pcsp_hi; + /* remember host/guest OSEM registers state & restore guest/host state */ + osem = host_ctxt->osem; + host_ctxt->osem = sw_ctxt->osem; + sw_ctxt->osem = osem; + /* keep current signal stack state */ base = current_thread_info()->signal_stack.base; size = current_thread_info()->signal_stack.size; @@ -778,6 +809,20 @@ static inline void pv_vcpu_exit_to_host(struct kvm_vcpu *vcpu) /* save VCPU guest thread context */ /* restore VCPU host thread context */ pv_vcpu_switch_host_context(vcpu); +#ifdef DEBUG_UPSR_FP_DISABLE + if (unlikely(!current_thread_info()->upsr.UPSR_fe)) { + pr_err("%s(): switch to host QEMU process with disabled " + "FloatPoint mask, UPSR 0x%x\n", + __func__, current_thread_info()->upsr.UPSR_reg); + /* correct UPSR to enable float pointing */ + current_thread_info()->upsr.UPSR_fe = 1; + } + if (unlikely(!vcpu->arch.host_ctxt.upsr.UPSR_fe)) { + pr_err("%s(): switch from host VCPU process where disabled " + "FloatPoint mask, UPSR 0x%x\n", + __func__, vcpu->arch.host_ctxt.upsr.UPSR_reg); + } +#endif /* DEBUG_UPSR_FP_DISABLE */ } static inline void pv_vcpu_enter_to_guest(struct kvm_vcpu *vcpu) @@ -785,6 +830,14 @@ static inline void pv_vcpu_enter_to_guest(struct kvm_vcpu *vcpu) /* save VCPU host thread context */ /* restore VCPU guest thread context */ pv_vcpu_switch_host_context(vcpu); +#ifdef DEBUG_UPSR_FP_DISABLE + if (unlikely(!current_thread_info()->upsr.UPSR_fe)) { + pr_err("%s(): switch to host VCPU process with disabled " + "FloatPoint mask, UPSR 0x%x\n", + __func__, current_thread_info()->upsr.UPSR_reg); + /* do not correct UPSR, maybe it should be */ + } +#endif /* DEBUG_UPSR_FP_DISABLE */ } static inline void @@ -879,11 +932,12 @@ host_syscall_from_guest_user(struct thread_info *ti) static inline void host_trap_guest_exit_intc(struct thread_info *ti, struct pt_regs *regs) { - if (!kvm_test_and_clear_intc_emul_flag(regs)) { + if (likely(!kvm_test_intc_emul_flag(regs))) { /* it is not paravirtualized guest VCPU intercepts*/ /* emulation mode, so nothing to do more */ return; } + kvm_clear_intc_emul_flag(regs); /* * Return from trap on paravirtualized guest VCPU which was @@ -972,31 +1026,20 @@ host_syscall_guest_get_pv_vcpu_restore_stacks(struct thread_info *ti, return ®s->g_stacks; } else { /* it need switch to guest user context */ - return native_syscall_guest_get_restore_stacks(ti, regs); + return native_syscall_guest_get_restore_stacks(regs); } } static inline struct e2k_stacks * host_trap_guest_get_restore_stacks(struct thread_info *ti, struct pt_regs *regs) { - if (test_ti_thread_flag(ti, TIF_HOST_AT_VCPU_MODE)) { + if (test_ti_status_flag(ti, TS_HOST_AT_VCPU_MODE)) { /* host return to paravirtualized guest (VCPU) mode */ return host_trap_guest_get_pv_vcpu_restore_stacks(ti, regs); } return native_trap_guest_get_restore_stacks(ti, regs); } -static inline struct e2k_stacks * -host_syscall_guest_get_restore_stacks(struct thread_info *ti, - struct pt_regs *regs) -{ - if (test_ti_thread_flag(ti, TIF_HOST_AT_VCPU_MODE)) { - /* host return to paravirtualized guest (VCPU) mode */ - return host_syscall_guest_get_pv_vcpu_restore_stacks(ti, regs); - } - return native_syscall_guest_get_restore_stacks(ti, regs); -} - static inline void host_trap_pv_vcpu_exit_trap(struct thread_info *ti, struct pt_regs *regs) { @@ -1021,7 +1064,7 @@ host_trap_pv_vcpu_exit_trap(struct thread_info *ti, struct pt_regs *regs) static inline void host_trap_guest_exit_trap(struct thread_info *ti, struct pt_regs *regs) { - if (test_ti_thread_flag(ti, TIF_HOST_AT_VCPU_MODE)) { + if (test_ti_status_flag(ti, TS_HOST_AT_VCPU_MODE)) { /* host return to paravirtualized guest (VCPU) mode */ host_trap_pv_vcpu_exit_trap(ti, regs); } @@ -1065,29 +1108,7 @@ host_syscall_pv_vcpu_exit_trap(struct thread_info *ti, struct pt_regs *regs) atomic_inc(&vcpu->arch.host_ctxt.signal.in_syscall); } -static inline void -host_syscall_guest_exit_trap(struct thread_info *ti, struct pt_regs *regs) -{ - if (!test_ti_thread_flag(ti, TIF_HOST_AT_VCPU_MODE)) - return; - - /* host return to paravirtualized guest (VCPU) mode */ - host_syscall_pv_vcpu_exit_trap(ti, regs); - - host_switch_trap_enable_mask(ti, regs, true); -} - -static inline void -host_guest_syscall_exit_to(struct thread_info *ti, struct pt_regs *regs, - unsigned flags) -{ - if (flags & EXIT_FROM_INTC_SWITCH) { - host_trap_guest_exit_intc(ti, regs); - } - if (flags & EXIT_FROM_TRAP_SWITCH) { - host_syscall_guest_exit_trap(ti, regs); - } -} +extern void host_syscall_guest_exit_trap(struct thread_info *, struct pt_regs *); extern void kvm_init_pv_vcpu_intc_handling(struct kvm_vcpu *vcpu, pt_regs_t *regs); extern int last_light_hcall; @@ -1096,13 +1117,14 @@ static inline void host_trap_guest_exit(struct thread_info *ti, struct pt_regs *regs, trap_pt_regs_t *trap, unsigned flags) { - if (!test_and_clear_ti_thread_flag(ti, TIF_HOST_AT_VCPU_MODE)) + if (likely(!test_ti_status_flag(ti, TS_HOST_AT_VCPU_MODE))) return; + clear_ti_status_flag(ti, TS_HOST_AT_VCPU_MODE); + /* * Trap on paravirtualized guest VCPU is interpreted as intercept */ - kvm_emulate_pv_vcpu_intc(ti, regs, trap); /* only after switch to host MMU context at previous function */ @@ -1112,13 +1134,14 @@ host_trap_guest_exit(struct thread_info *ti, struct pt_regs *regs, /* * The function should return bool 'is the system call from guest?' */ -static inline bool -host_guest_syscall_enter(struct thread_info *ti, struct pt_regs *regs) +static inline bool host_guest_syscall_enter(struct pt_regs *regs, + bool ts_host_at_vcpu_mode) { - if (!test_and_clear_ti_thread_flag(ti, TIF_HOST_AT_VCPU_MODE)) + if (likely(!ts_host_at_vcpu_mode)) return false; /* it is not guest system call */ - return pv_vcpu_syscall_intc(ti, regs); + clear_ts_flag(TS_HOST_AT_VCPU_MODE); + return pv_vcpu_syscall_intc(current_thread_info(), regs); } #endif /* CONFIG_VIRTUALIZATION */ @@ -1188,25 +1211,26 @@ trap_guest_get_restore_stacks(struct thread_info *ti, struct pt_regs *regs) } static inline struct e2k_stacks * -syscall_guest_get_restore_stacks(struct thread_info *ti, struct pt_regs *regs) +syscall_guest_get_restore_stacks(bool ts_host_at_vcpu_mode, struct pt_regs *regs) { - return native_syscall_guest_get_restore_stacks(ti, regs); + return native_syscall_guest_get_restore_stacks(regs); } +#define ts_host_at_vcpu_mode() false + /* * The function should return bool is the system call from guest */ -static inline bool -guest_syscall_enter(struct thread_info *ti, struct pt_regs *regs) +static inline bool guest_syscall_enter(struct pt_regs *regs, + bool ts_host_at_vcpu_mode) { - return native_guest_syscall_enter(ti, regs); -} -static inline void -guest_syscall_exit_to(struct thread_info *ti, struct pt_regs *regs, - unsigned flags) -{ - native_guest_syscall_exit_to(ti, regs, flags); + return native_guest_syscall_enter(regs); } +static inline void guest_exit_intc(struct pt_regs *regs, + bool intc_emul_flag) { } +static inline void guest_syscall_exit_trap(struct pt_regs *regs, + bool ts_host_at_vcpu_mode) { } + #else /* CONFIG_VIRTUALIZATION */ /* it is only host kernel with virtualization support */ static inline void __guest_enter(struct thread_info *ti, @@ -1268,25 +1292,47 @@ trap_guest_get_restore_stacks(struct thread_info *ti, struct pt_regs *regs) } static inline struct e2k_stacks * -syscall_guest_get_restore_stacks(struct thread_info *ti, struct pt_regs *regs) +syscall_guest_get_restore_stacks(bool ts_host_at_vcpu_mode, struct pt_regs *regs) { - return host_syscall_guest_get_restore_stacks(ti, regs); + if (unlikely(ts_host_at_vcpu_mode)) { + /* host return to paravirtualized guest (VCPU) mode */ + return host_syscall_guest_get_pv_vcpu_restore_stacks( + current_thread_info(), regs); + } + return native_syscall_guest_get_restore_stacks(regs); } +#define ts_host_at_vcpu_mode() unlikely(!!test_ts_flag(TS_HOST_AT_VCPU_MODE)) + /* * The function should return bool is the system call from guest */ -static inline bool -guest_syscall_enter(struct thread_info *ti, struct pt_regs *regs) +static inline bool guest_syscall_enter(struct pt_regs *regs, + bool ts_host_at_vcpu_mode) { - return host_guest_syscall_enter(ti, regs); + return host_guest_syscall_enter(regs, ts_host_at_vcpu_mode); } -static inline void -guest_syscall_exit_to(struct thread_info *ti, struct pt_regs *regs, - unsigned flags) + +static inline void guest_exit_intc(struct pt_regs *regs, bool intc_emul_flag) { - host_guest_syscall_exit_to(ti, regs, flags); + if (unlikely(intc_emul_flag)) { + kvm_clear_intc_emul_flag(regs); + + /* + * Return from trap on paravirtualized guest VCPU which was + * interpreted as interception + */ + return_from_pv_vcpu_intc(current_thread_info(), regs); + } } + +static inline void guest_syscall_exit_trap(struct pt_regs *regs, + bool ts_host_at_vcpu_mode) +{ + if (unlikely(ts_host_at_vcpu_mode)) + host_syscall_guest_exit_trap(current_thread_info(), regs); +} + #endif /* ! CONFIG_VIRTUALIZATION */ #endif /* CONFIG_PARAVIRT_GUEST */ diff --git a/arch/e2k/include/asm/kvm/thread_info.h b/arch/e2k/include/asm/kvm/thread_info.h index 3748ab3..943ad33 100644 --- a/arch/e2k/include/asm/kvm/thread_info.h +++ b/arch/e2k/include/asm/kvm/thread_info.h @@ -147,8 +147,13 @@ typedef struct gthread_info { vcpu_l_gregs_t l_gregs; /* guest user "local" global */ /* registers to save updated on page */ /* fault values */ - kernel_gregs_t g_gregs; /* guest kernel global resiters state */ + kernel_gregs_t gk_gregs; /* guest kernel global resiters state */ + /* some registers can be updated only */ /* after migration to other VCPU */ + kernel_gregs_t gu_gregs; /* guest user global resiters state */ + /* only for global registers which */ + /* used by the guest kernel for its */ + /* own purposes */ /* the following flags to mark event: */ /* hardware stacks bounds trap occured, but 'sge' on guest */ @@ -236,6 +241,8 @@ typedef struct gthread_info { /* thread */ #define GTIF_THREAD_MIGRATED 2 /* the thread was migrated from one */ /* VCPU to other */ +#define GTIF_USER_THREAD 4 /* the process is user thread on */ + /* common virtual memory (gmm) */ #define GTIF_HW_PS_LOCKED 16 /* hardware procedure stack */ /* was locked by host */ #define GTIF_HW_PCS_LOCKED 17 /* hardware chain stack */ @@ -258,6 +265,7 @@ typedef struct gthread_info { #define _GTIF_VCPU_START_THREAD (1UL << GTIF_VCPU_START_THREAD) #define _GTIF_KERNEL_THREAD (1UL << GTIF_KERNEL_THREAD) #define _GTIF_THREAD_MIGRATED (1UL << GTIF_THREAD_MIGRATED) +#define _GTIF_USER_THREAD (1UL << GTIF_USER_THREAD) #define _GTIF_HW_PS_LOCKED (1UL << GTIF_HW_PS_LOCKED) #define _GTIF_HW_PCS_LOCKED (1UL << GTIF_HW_PCS_LOCKED) #define _GTIF_HW_PS_PRESENTED (1UL << GTIF_HW_PS_PRESENTED) diff --git a/arch/e2k/include/asm/kvm/trace_kvm_hv.h b/arch/e2k/include/asm/kvm/trace_kvm_hv.h index 153b160..b617c13 100644 --- a/arch/e2k/include/asm/kvm/trace_kvm_hv.h +++ b/arch/e2k/include/asm/kvm/trace_kvm_hv.h @@ -444,6 +444,41 @@ TRACE_EVENT( TP_printk("gpa 0x%lx, data 0x%lx\n", __entry->gpa, __entry->data) ); +TRACE_EVENT( + intc_clw, + + TP_PROTO(bool us_cl_d, unsigned long us_cl_b, unsigned long us_cl_up, + unsigned long us_cl_m0, unsigned long us_cl_m1, + unsigned long us_cl_m2, unsigned long us_cl_m3), + + TP_ARGS(us_cl_d, us_cl_b, us_cl_up, us_cl_m0, us_cl_m1, us_cl_m2, us_cl_m3), + + TP_STRUCT__entry( + __field( bool, us_cl_d ) + __field( unsigned long, us_cl_b ) + __field( unsigned long, us_cl_up ) + __field( unsigned long, us_cl_m0 ) + __field( unsigned long, us_cl_m1 ) + __field( unsigned long, us_cl_m2 ) + __field( unsigned long, us_cl_m3 ) + ), + + TP_fast_assign( + __entry->us_cl_d = us_cl_d; + __entry->us_cl_b = us_cl_b; + __entry->us_cl_up = us_cl_up; + __entry->us_cl_m0 = us_cl_m0; + __entry->us_cl_m1 = us_cl_m1; + __entry->us_cl_m2 = us_cl_m2; + __entry->us_cl_m3 = us_cl_m3; + ), + + TP_printk("us_cl_d %d, us_cl_b 0x%lx, us_cl_up 0x%lx\n" + "us_cl_m0 0x%lx us_cl_m1 0x%lx us_cl_m2 0x%lx, us_cl_m3 0x%lx\n", + __entry->us_cl_d, __entry->us_cl_b, __entry->us_cl_up, + __entry->us_cl_m0, __entry->us_cl_m1, __entry->us_cl_m2, __entry->us_cl_m3) +); + #endif /* _TRACE_KVM_HV_H */ /* This part must be outside protection */ diff --git a/arch/e2k/include/asm/kvm/trap_table.S.h b/arch/e2k/include/asm/kvm/trap_table.S.h index a20e1ae..caa3b89 100644 --- a/arch/e2k/include/asm/kvm/trap_table.S.h +++ b/arch/e2k/include/asm/kvm/trap_table.S.h @@ -143,39 +143,29 @@ } .endm /* GOTO_GUEST_KERNEL_TTABLE */ +# ifdef CONFIG_PARAVIRT_GUEST /* * goto guest kernel system call table entry, if system call is from guest user * rti: register of current_thread_info() * rtmp0 rtmp1 rtmp2: temporary registers * ptmp0 ptmp1: temporary predicates */ -.macro GOTO_PV_VCPU_KERNEL_TTABLE entry_num rti rtmp0 rtmp1 rtmp2 \ - ptmp0 ptmp1 +.macro GOTO_PV_VCPU_KERNEL_TTABLE rti rtmp0 rtmp1 rtmp2 ptmp1 /* thread_info_t *ti = %dr7 */ /* e2k_cr1_lo_t cr1_lo = NATIVE_READ_CR1_LO_REG(); */ /* */ - /* if ((ti->flags & TIF_HOST_AT_VCPU_MODE)) { */ /* if (ti->flags & _TIF_PARAVIRT_GUEST) { */ /* DO_SWITCH_TO_KERNEL_IMAGE_PGD() */ /* } */ - /* goto goto_guest_kernel_ttable_C( */ - /* sys_num << 32 | entry, */ - /* arg1, arg2, arg3, arg4, */ - /* arg5, arg6); */ - /* } */ { ldd [\rti + TI_FLAGS], \rtmp0; sxt 2, %r0, %dr0; } { - cmpandedb \rtmp0, _TIF_HOST_AT_VCPU_MODE, \ptmp0; cmpandedb \rtmp0, _TIF_PARAVIRT_GUEST, \ptmp1; } { - pass \ptmp0, @p0; - landp ~@p0, ~@p0, @p4; - pass @p4, \ptmp0; pass \ptmp1, @p2; landp ~@p2, ~@p2, @p5; pass @p5, \ptmp1; @@ -185,6 +175,10 @@ /* rtmp0, rtmp1, rtmp2: temporary registers */ DO_SWITCH_TO_KERNEL_IMAGE_PGD \rti, \ptmp1, \rtmp0, \rtmp1, \rtmp2 .endm /* GOTO_GUEST_KERNEL_TTABLE */ +# else +.macro GOTO_PV_VCPU_KERNEL_TTABLE rti rtmp0 rtmp1 rtmp2 ptmp1 +.endm +# endif /* CONFIG_PARAVIRT_GUEST */ /* * goto guest kernel fast system call table entry, if system call is @@ -329,64 +323,6 @@ #ifdef CONFIG_KVM_HOST_MODE /* it is host kernel with virtualization support */ /* or paravirtualized host and guest kernel */ -.macro NEED_SAVE_CUR_AND_VCPU_STATE_GREGS drti, predV5, \ - drtmp0, drtmp1, predtmp, \ - predCUR, predVCPU, predEXTk - /* - * drti - pointer to thread_info - * predV5 - ISET is V5 - * predCUR - is now set to true (trap from user) and can be updated - * to does not save kernel global regs and set current - * Trap at host mode and host kernel currents and other global registers - * (GCURTI & GCURTASK & CPU_ID & CPU_OFF) - * should not be saved to not invalidate guest kernel or user state of - * global registers, which were or will be saved at thread info - * %predVCPU - save VCPU state pointer regs - * predEXTk - need save kernel (predCUR) & need save extention (!predV5) - * - * predCUR = test_thread_flag(TIF_HOST_AT_VCPU_MODE) && - * !test_thread_flag(TIF_LIGHT_HYPERCALL) || - * !test_thread_flag(TIF_HOST_AT_VCPU_MODE) && - * (cr0_hi.CR0_hi_IP >= NATIVE_TASK_SIZE) - * predVCPU = predCUR; - * predEXTk = predCUR & !predV5 - */ - { - rrd %cr0.hi, \drtmp0; /* %drtmp0: cr0_hi.IP */ - ldd [\drti + TI_FLAGS], \drtmp1; /* %drtmp1: ti->flags */ - } - { - cmpbdb \drtmp0, NATIVE_TASK_SIZE, \predtmp; - cmpandedb \drtmp1, _TIF_LIGHT_HYPERCALL, \predCUR; - cmpandedb \drtmp1, _TIF_HOST_AT_VCPU_MODE, \predVCPU; - } - { - nop 1; - pass \predtmp, @p2; - pass \predCUR, @p0; - pass \predVCPU, @p1; - landp @p0, ~@p1, @p4; - pass @p4, \predCUR; - } - { - nop 1; - pass \predVCPU, @p0; - pass \predCUR, @p2; - pass \predtmp, @p1; - landp @p0, ~@p1, @p4; - landp ~@p2, ~@p4, @p5; - landp ~@p2, ~@p4, @p6; - pass @p5, \predCUR; - pass @p6, \predVCPU; - } - { - pass \predV5, @p0; - pass \predCUR, @p1; - landp ~@p0, @p1, @p4; - pass @p4, \predEXTk; - } -.endm /* NEED_SAVE_CUR_AND_VCPU_STATE_GREGS */ - .macro DO_SAVE_HOST_GREGS_V2 gvcpu_lo, gvcpu_hi, hvcpu_lo, hvcpu_hi \ drti, predSAVE, drtmp, rtmp0, rtmp1 /* drtmp: thread_info->h_gregs.g */ @@ -439,27 +375,6 @@ #include #else /* ! CONFIG_KVM_HOST_MODE && ! CONFIG_KVM_GUEST_KERNEL */ /* It is native host kernel without any virtualization */ -.macro NEED_SAVE_CUR_AND_VCPU_STATE_GREGS drti, predV5, \ - drtmp0, drtmp1, predtmp, \ - predCUR, predVCPU, predEXTk - /* - * drti - pointer to thread_info (unused) - * predV5 - ISET is V5 - * predCUR - save kernel global regs and set current (already - * calculated, don't update) - * %predVCPU - set to false (none any VCPUs) - * predEXTk - need save kernel (predCUR) & need save extention (!predV5) - */ - { - pass \predV5, @p0; - pass \predCUR, @p1; - landp ~@p0, @p1, @p4; - landp ~@p1, @p1, @p5; - pass @p4, \predEXTk; - pass @p5, \predVCPU; - } -.endm /* NEED_SAVE_CUR_AND_VCPU_STATE_GREGS */ - .macro SAVE_HOST_GREGS_TO_VIRT_V2 drti, predSAVE, drtmp, rtmp0, rtmp1 /* not used */ .endm /* SAVE_VCPU_STATE_GREGS */ diff --git a/arch/e2k/include/asm/kvm/trap_table.h b/arch/e2k/include/asm/kvm/trap_table.h index efb0548..4a59da8 100644 --- a/arch/e2k/include/asm/kvm/trap_table.h +++ b/arch/e2k/include/asm/kvm/trap_table.h @@ -156,11 +156,6 @@ is_guest_TIRs_frozen(struct pt_regs *regs) { return false; /* none any guest */ } -static inline bool -have_deferred_traps(struct pt_regs *regs) -{ - return native_have_deferred_traps(regs); -} static inline bool handle_guest_last_wish(struct pt_regs *regs) @@ -261,7 +256,6 @@ typedef enum trap_hndl { } trap_hndl_t; extern trap_hndl_t kvm_do_handle_guest_traps(struct pt_regs *regs); -extern trap_hndl_t kvm_handle_guest_deferred_traps(struct pt_regs *regs); extern bool kvm_is_guest_TIRs_frozen(struct pt_regs *regs); extern bool kvm_is_guest_proc_stack_bounds(struct pt_regs *regs); @@ -285,8 +279,7 @@ extern unsigned long kvm_pass_page_fault_to_guest(struct pt_regs *regs, trap_cellar_t *tcellar); extern void kvm_complete_page_fault_to_guest(unsigned long what_complete); -extern noinline notrace int do_hret_last_wish_intc(struct kvm_vcpu *vcpu, - struct pt_regs *regs); +extern int do_hret_last_wish_intc(struct kvm_vcpu *vcpu, struct pt_regs *regs); extern void trap_handler_trampoline(void); extern void syscall_handler_trampoline(void); @@ -302,12 +295,10 @@ kvm_init_guest_traps_handling(struct pt_regs *regs, bool user_mode_trap) { regs->traps_to_guest = 0; /* only for host */ regs->is_guest_user = false; /* only for host */ - regs->deferred_traps = 0; /* for host and guest */ regs->g_stacks_valid = false; /* only for host */ - if (user_mode_trap && - test_thread_flag(TIF_LIGHT_HYPERCALL) && + if (user_mode_trap && test_thread_flag(TIF_LIGHT_HYPERCALL) && (NATIVE_NV_READ_CR1_LO_REG().CR1_lo_pm)) { - regs->flags |= LIGHT_HYPERCALL_FLAG_PT_REGS; + regs->flags.light_hypercall = 1; } } @@ -316,14 +307,8 @@ kvm_init_guest_syscalls_handling(struct pt_regs *regs) { regs->traps_to_guest = 0; /* only for host */ regs->is_guest_user = true; /* only for host */ - regs->deferred_traps = 0; /* only for guest */ regs->g_stacks_valid = false; /* only for host */ } -static inline bool -kvm_have_guest_deferred_traps(struct pt_regs *regs) -{ - return regs->deferred_traps != 0; -} static inline void kvm_exit_handle_syscall(e2k_addr_t sbr, e2k_usd_hi_t usd_hi, @@ -348,6 +333,12 @@ kvm_handle_guest_last_wish(struct pt_regs *regs) /* it is not guest VCPU thread, or completed */ return false; } + if (vcpu->arch.trap_wish) { + /* some trap was injected, goto trap handling */ + regs->traps_to_guest |= vcpu->arch.trap_mask_wish; + vcpu->arch.trap_mask_wish = 0; + return true; + } if (vcpu->arch.virq_wish) { /* trap is only to interrupt guest kernel on guest mode */ /* to provide injection of pending VIRQs on guest */ @@ -407,8 +398,11 @@ kvm_should_pass_the_trap_to_guest(struct pt_regs *regs, int trap_no) } } else if (vcpu->arch.is_pv) { if (vcpu->arch.virq_wish) { - /* it is paravirtualized guest, pass trap */ - /* to guest, if it is enabled */ + /* it is paravirtualized guest, pass */ + /* interrupt to guest, if it is enabled */ + ; + } else if (vcpu->arch.trap_wish) { + /* it is wish to inject some trap to guest */ ; } else { /* there is not any wish for guest */ @@ -479,15 +473,8 @@ static inline bool kvm_handle_guest_traps(struct pt_regs *regs) "created\n"); return false; } - regs->flags |= GUEST_FLAG_PT_REGS; ret = kvm_do_handle_guest_traps(regs); regs->traps_to_guest = 0; - if (regs->deferred_traps) { - /* New traps (VIRQs interrupt) occured to pass to guest */ - ret = kvm_handle_guest_deferred_traps(regs); - regs->deferred_traps = 0; - } - regs->flags &= ~GUEST_FLAG_PT_REGS; if (ret == GUEST_TRAP_HANDLED) { DebugKVMGT("the guest trap handled\n"); @@ -552,11 +539,6 @@ is_guest_TIRs_frozen(struct pt_regs *regs) return kvm_is_guest_TIRs_frozen(regs); } -static inline bool -have_deferred_traps(struct pt_regs *regs) -{ - return kvm_have_guest_deferred_traps(regs); -} static inline bool handle_guest_last_wish(struct pt_regs *regs) @@ -580,18 +562,18 @@ kvm_host_instr_page_fault(struct pt_regs *regs, tc_fault_type_t ftype, kvm_pv_mmu_instr_page_fault(vcpu, regs, ftype, async_instr); } -static inline void +static inline int kvm_host_do_aau_page_fault(struct pt_regs *const regs, e2k_addr_t address, const tc_cond_t condition, const tc_mask_t mask, const unsigned int aa_no) { if (likely(!kvm_test_intc_emul_flag(regs))) { - native_do_aau_page_fault(regs, address, condition, mask, aa_no); - return; + return native_do_aau_page_fault(regs, address, condition, mask, + aa_no); } - kvm_pv_mmu_aau_page_fault(current_thread_info()->vcpu, regs, - address, condition, aa_no); + return kvm_pv_mmu_aau_page_fault(current_thread_info()->vcpu, regs, + address, condition, aa_no); } /* @@ -830,12 +812,13 @@ instr_page_fault(struct pt_regs *regs, tc_fault_type_t ftype, kvm_host_instr_page_fault(regs, ftype, async_instr); } -static inline void +static inline int do_aau_page_fault(struct pt_regs *const regs, e2k_addr_t address, const tc_cond_t condition, const tc_mask_t mask, const unsigned int aa_no) { - kvm_host_do_aau_page_fault(regs, address, condition, mask, aa_no); + return kvm_host_do_aau_page_fault(regs, address, condition, mask, + aa_no); } #endif /* CONFIG_VIRTUALIZATION */ diff --git a/arch/e2k/include/asm/kvm/ttable-help.h b/arch/e2k/include/asm/kvm/ttable-help.h new file mode 100644 index 0000000..f4704be --- /dev/null +++ b/arch/e2k/include/asm/kvm/ttable-help.h @@ -0,0 +1,51 @@ +/* + * + * Copyright (C) 2020 MCST + * + * Definitions of KVM traps handling routines. + */ + +#ifndef _E2K_KVM_TTABLE_HELP_H +#define _E2K_KVM_TTABLE_HELP_H + +#ifdef CONFIG_KVM_HOST_MODE +/* it is native kernel with virtualization support (hypervisor) */ + +#ifdef CONFIG_CPU_HW_CLEAR_RF + +# ifdef GENERATING_HEADER +# define RETURN_PV_VCPU_TRAP_SIZE 0x1 +# define HANDLE_PV_VCPU_SYS_CALL_SIZE 0x1 +# define HANDLE_PV_VCPU_SYS_FORK_SIZE 0x1 +# endif + +# define CLEAR_RETURN_PV_VCPU_TRAP_WINDOW() E2K_DONE() +# define CLEAR_HANDLE_PV_VCPU_SYS_CALL_WINDOW(r0) E2K_SYSCALL_RETURN(r0) +# define CLEAR_HANDLE_PV_VCPU_SYS_FORK_WINDOW(r0) E2K_SYSCALL_RETURN(r0) + +#else /* ! CONFIG_CPU_HW_CLEAR_RF */ + +# ifdef GENERATING_HEADER +# define CLEAR_RETURN_PV_VCPU_TRAP_WINDOW() E2K_EMPTY_CMD(: "ctpr3") +# define CLEAR_HANDLE_PV_VCPU_SYS_CALL_WINDOW(r0) \ + E2K_EMPTY_CMD([_r0] "ir" (r0) : "ctpr3") +# define CLEAR_HANDLE_PV_VCPU_SYS_FORK_WINDOW(r0) \ + E2K_EMPTY_CMD([_r0] "ir" (r0) : "ctpr3") +# define RETURN_PV_VCPU_TRAP_SIZE 0x1 +# define HANDLE_PV_VCPU_SYS_CALL_SIZE 0x1 +# define HANDLE_PV_VCPU_SYS_FORK_SIZE 0x1 +# endif + +#endif /* CONFIG_CPU_HW_CLEAR_RF */ + +#else /* !CONFIG_KVM_HOST_MODE */ +/* It is native guest kernel whithout virtualization support */ +/* Virtualiztion in guest mode cannot be supported */ + +# define CLEAR_RETURN_PV_VCPU_TRAP_WINDOW() +# define CLEAR_HANDLE_PV_VCPU_SYS_CALL_WINDOW(rval) +# define CLEAR_HANDLE_PV_VCPU_SYS_FORK_WINDOW(rval) + +#endif /* CONFIG_KVM_HOST_MODE */ + +#endif /* _E2K_KVM_TTABLE_HELP_H */ diff --git a/arch/e2k/include/asm/kvm/uaccess.h b/arch/e2k/include/asm/kvm/uaccess.h index 5a757fa..2e86d1a 100644 --- a/arch/e2k/include/asm/kvm/uaccess.h +++ b/arch/e2k/include/asm/kvm/uaccess.h @@ -54,24 +54,38 @@ native_copy_from_user_with_tags(void *to, const void __user *from, ({ \ __typeof__(*(uptr)) __user *___pu_ptr = (uptr); \ int sz_uptr = sizeof(*(uptr)); \ + long res; \ \ ___pu_ptr = (!host_test_intc_emul_mode(hregs)) ? \ (uptr) \ : \ - kvm_guest_ptr_to_host_ptr((uptr), sz_uptr); \ - (___pu_ptr) ? native_get_user(kval, ___pu_ptr) : -EFAULT; \ + kvm_guest_ptr_to_host_ptr((uptr), sz_uptr, \ + true); \ + if (PTR_ERR(___pu_ptr) == -EAGAIN) \ + res = -EAGAIN; \ + else \ + res = (___pu_ptr) ? native_get_user(kval, ___pu_ptr) : \ + -EFAULT; \ + (res); \ }) #define host_put_user(kval, uptr, hregs) \ ({ \ __typeof__(*(uptr)) __user *___pu_ptr = (uptr); \ int sz_uptr = sizeof(*(uptr)); \ + long res; \ \ ___pu_ptr = (!host_test_intc_emul_mode(hregs)) ? \ (uptr) \ : \ - kvm_guest_ptr_to_host_ptr((uptr), sz_uptr); \ - (___pu_ptr) ? native_put_user(kval, ___pu_ptr) : -EFAULT; \ + kvm_guest_ptr_to_host_ptr((uptr), sz_uptr, \ + true); \ + if (PTR_ERR(___pu_ptr) == -EAGAIN) \ + res = -EAGAIN; \ + else \ + res = (___pu_ptr) ? native_put_user(kval, ___pu_ptr) : \ + -EFAULT; \ + (res); \ }) extern unsigned long kvm_copy_in_user_with_tags(void __user *to, diff --git a/arch/e2k/include/asm/kvm/vcpu-regs-debug-inline.h b/arch/e2k/include/asm/kvm/vcpu-regs-debug-inline.h new file mode 100644 index 0000000..660a356 --- /dev/null +++ b/arch/e2k/include/asm/kvm/vcpu-regs-debug-inline.h @@ -0,0 +1,86 @@ +/* + * KVM guest kernel processes support + * Copyright 2011 Salavat S. Guiliazov (atic@mcst.ru) + */ + +#ifndef _E2K_KVM_VCPU_REGS_DEBUG_INLINE_H +#define _E2K_KVM_VCPU_REGS_DEBUG_INLINE_H + +#include +#include +#include + +#ifdef VCPU_REGS_DEBUG + +static inline void init_vcpu_regs_trace(void) +{ + vcpu_regs_trace_t *trace; + + trace = get_vcpu_regs_trace_struct(); + atomic_set(&trace->count, 0); + vcpu_regs_trace_on = true; +} + +static inline void dump_vcpu_regs_entry(vcpu_regs_t *regs, int entry_no) +{ + u32 updated = regs->updated; + + pr_alert("[%02d] : PSR %02x\tUPSR %03x\tunder UPSR %d\n", + entry_no, regs->psr, regs->upsr, regs->under_upsr); + pr_alert(" updated : %s %s %s %s\n", + (updated & PSR_UPDATE_MASK) ? "PSR" : "", + (updated & UPSR_UPDATE_MASK) ? "UPSR" : "", + (updated & UNDER_UPSR_UPDATE_MASK) ? "under UPSR" : "", + (regs->injected) ? "injected IRQs" : ""); + pr_alert(" IP %pF called from IP %pF\n", + (void *)regs->IP, (void *)regs->IP_from); + pr_alert(" clock : start 0x%llx end 0x%llx delta 0x%llx\n", + regs->clock_start, regs->clock_end, + regs->clock_end - regs->clock_start); +} + +static inline void dump_vcpu_regs_trace(void) +{ + vcpu_regs_trace_t *trace; + vcpu_regs_t *regs; + int count, ent, num, entries; + + /* stop tracing */ + vcpu_regs_trace_on = false; + + trace = get_vcpu_regs_trace_struct(); + count = atomic_read(&trace->count); + pr_alert("CPU #%d : Trace of VCPU #%d some registers update history\n", + raw_smp_processor_id(), KVM_READ_VCPU_ID()); + if (count == 0) { + pr_alert(" trace is empty\n"); + return; + } + + entries = (count > MAX_VCPU_REGS_TRACE_NUM) ? + MAX_VCPU_REGS_TRACE_NUM : count; + for (ent = VCPU_REGS_TRACE_INDEX(count), num = 0; + num < entries; + ent = VCPU_REGS_TRACE_INDEX(ent - 1), num++) { + regs = &trace->regs[ent]; + dump_vcpu_regs_entry(regs, ent); + } +} +#else /* !VCPU_REGS_DEBUG */ + +#define vcpu_regs_trace_on false + +static inline void init_vcpu_regs_trace(void) +{ +} + +#define trace_vcpu_upsr_update(upsr_val, injected_irqs) +#define trace_vcpu_psr_update(psr_val, under_upsr) + +static inline void dump_vcpu_regs_trace(void) +{ +} + +#endif /* VCPU_REGS_DEBUG */ + +#endif /* ! _E2K_KVM_VCPU_REGS_DEBUG_INLINE_H */ diff --git a/arch/e2k/include/asm/kvm/vcpu-regs-debug.h b/arch/e2k/include/asm/kvm/vcpu-regs-debug.h new file mode 100644 index 0000000..386e226 --- /dev/null +++ b/arch/e2k/include/asm/kvm/vcpu-regs-debug.h @@ -0,0 +1,130 @@ +/* + * KVM guest kernel processes support + * Copyright 2011 Salavat S. Guiliazov (atic@mcst.ru) + */ + +#ifndef _E2K_KVM_VCPU_REGS_DEBUG_H +#define _E2K_KVM_VCPU_REGS_DEBUG_H + +/* do not include this header directly, only through asm/kvm/guest.h */ + +#include + +#undef VCPU_REGS_DEBUG + +typedef struct vcpu_regs { + u64 IP; + u64 IP_from; + u64 clock_start; + u64 clock_end; + u32 psr; + u32 upsr; + bool under_upsr; + u8 updated; + bool injected; +} vcpu_regs_t; + +#define MAX_VCPU_REGS_TRACE_NUM 32 +#define VCPU_REGS_TRACE_MASK (MAX_VCPU_REGS_TRACE_NUM - 1) +#define VCPU_REGS_TRACE_INDEX(count) ((count) & VCPU_REGS_TRACE_MASK) + +typedef struct vcpu_regs_trace { + atomic_t count; + vcpu_regs_t regs[MAX_VCPU_REGS_TRACE_NUM]; +} vcpu_regs_trace_t; + +#define PSR_UPDATE_MASK 0x01U +#define UPSR_UPDATE_MASK 0x02U +#define UNDER_UPSR_UPDATE_MASK 0x04U + +#define GET_CLOCK_REG() NATIVE_READ_CLKR_REG_VALUE() + +#define GUEST_GET_IRQS_UNDER_UPSR() \ +({ \ + kvm_vcpu_state_t *vcpu_state; \ + bool under_upsr; \ + \ + KVM_GET_VCPU_STATE_BASE(vcpu_state); \ + under_upsr = vcpu_state->irqs_under_upsr; \ + under_upsr; \ +}) + +#ifdef VCPU_REGS_DEBUG + +#undef PSR +extern bool vcpu_regs_trace_on; +extern int vcpu_regs_trace_on_num; + +#define get_vcpu_regs_trace_struct() \ +({ \ + struct kvm_vcpu_state *vcpu_state; \ + \ + KVM_GET_VCPU_STATE_BASE(vcpu_state); \ + &vcpu_state->trace; \ +}) + +#define get_next_vcpu_regs_trace() \ +({ \ + vcpu_regs_trace_t *trace; \ + vcpu_regs_t *regs; \ + int count; \ + \ + if (likely(!vcpu_regs_trace_on)) { \ + regs = NULL; \ + } else { \ + trace = get_vcpu_regs_trace_struct(); \ + count = atomic_inc_return(&trace->count); \ + regs = &trace->regs[VCPU_REGS_TRACE_INDEX(count)]; \ + regs->clock_start = GET_CLOCK_REG(); \ + regs->IP = NATIVE_READ_IP_REG_VALUE(); \ + regs->IP_from = NATIVE_NV_READ_CR0_HI_REG_VALUE(); \ + regs->updated = 0; \ + regs->psr = GUEST_GET_CPU_SREG(E2K_PSR); \ + regs->upsr = GUEST_GET_CPU_SREG(UPSR); \ + regs->under_upsr = GUEST_GET_IRQS_UNDER_UPSR(); \ + regs->injected = false; \ + } \ + regs; \ +}) + +#define trace_vcpu_upsr_update(upsr_val, injected_irqs) \ +do { \ + vcpu_regs_t *regs; \ + if (likely(!vcpu_regs_trace_on)) \ + break; \ + regs = get_next_vcpu_regs_trace(); \ + if (unlikely(regs == NULL)) \ + break; \ + regs->upsr = (upsr_val); \ + regs->updated |= UPSR_UPDATE_MASK; \ + if (injected_irqs) { \ + regs->injected = (injected_irqs); \ + } \ + E2K_CMD_SEPARATOR; \ + regs->clock_end = GET_CLOCK_REG(); \ +} while (false) + +#define trace_vcpu_psr_update(psr_val, under_upsr_mode) \ +do { \ + vcpu_regs_t *regs; \ + if (likely(!vcpu_regs_trace_on)) \ + break; \ + regs = get_next_vcpu_regs_trace(); \ + if (unlikely(regs == NULL)) \ + break; \ + regs->psr = (psr_val); \ + regs->updated |= PSR_UPDATE_MASK | UNDER_UPSR_UPDATE_MASK; \ + regs->under_upsr = (under_upsr_mode); \ + E2K_CMD_SEPARATOR; \ + regs->clock_end = GET_CLOCK_REG(); \ +} while (false) + +#else /* !VCPU_REGS_DEBUG */ + +#define vcpu_regs_trace_on false +#define trace_vcpu_upsr_update(upsr_val, injected_irqs) +#define trace_vcpu_psr_update(psr_val, under_upsr) + +#endif /* VCPU_REGS_DEBUG */ + +#endif /* ! _E2K_KVM_VCPU_REGS_DEBUG_H */ diff --git a/arch/e2k/include/asm/kvm_host.h b/arch/e2k/include/asm/kvm_host.h index 94c05bc..b057001 100644 --- a/arch/e2k/include/asm/kvm_host.h +++ b/arch/e2k/include/asm/kvm_host.h @@ -109,7 +109,8 @@ kvm_is_hw_pv_vm_available(void) #undef E2K_INVALID_PAGE #define E2K_INVALID_PAGE (~(hpa_t)0) -#define UNMAPPED_GVA (~(gpa_t)0) +#define UNMAPPED_GVA (~(gpa_t)0) +#define arch_is_error_gpa(gpa) ((gpa_t)(gpa) == UNMAPPED_GVA) /* * See include/linux/kvm_host.h @@ -387,6 +388,10 @@ typedef struct kvm_mmu_page { /* Number of writes since the last time traversal visited this page. */ atomic_t write_flooding_count; +#ifdef CONFIG_GUEST_MM_SPT_LIST + struct list_head gmm_entry; /* entry at the gmm list of SPs */ + gmm_struct_t *gmm; /* the gmm in whose list the entry */ +#endif /* CONFIG_GUEST_MM_SPT_LIST */ } kvm_mmu_page_t; /* page fault handling results */ @@ -533,9 +538,9 @@ typedef struct kvm_mmu { struct kvm_arch_exception *exception); void (*update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, pgprot_t *spte, const void *pte); - void (*flush_gva)(struct kvm_vcpu *vcpu, gva_t gva); - void (*flush_gva_range)(struct kvm_vcpu *vcpu, gva_t gva_start, - gva_t gva_end); + void (*sync_gva)(struct kvm_vcpu *vcpu, gva_t gva); + void (*sync_gva_range)(struct kvm_vcpu *vcpu, gva_t gva_start, + gva_t gva_end, bool flush_tlb); int (*sync_page)(struct kvm_vcpu *vcpu, kvm_mmu_page_t *sp); } kvm_mmu_t; @@ -649,6 +654,18 @@ typedef struct kvm_sw_cpu_context { e2k_sbr_t sbr; } saved; + /* + * Host VCPU local data stack pointer registers state (to save/restore). + * It is relevant only for paravirtualization, since in this case + * there is one VCPU process, but there are two mode of its execution: + * as host part of VCPU (qemu) + * as guest part of VCPU + * and, accordingly, two stacks: host & guest. + */ + e2k_usd_lo_t host_usd_lo; + e2k_usd_hi_t host_usd_hi; + e2k_sbr_t host_sbr; + e2k_mem_crs_t crs; /* only for PV guest */ /* @@ -726,6 +743,14 @@ typedef struct kvm_sw_cpu_context { u64 rpr_lo; u64 rpr_hi; u64 tcd; + + mmu_reg_t us_cl_d; + clw_reg_t us_cl_b; + clw_reg_t us_cl_up; + clw_reg_t us_cl_m0; + clw_reg_t us_cl_m1; + clw_reg_t us_cl_m2; + clw_reg_t us_cl_m3; } kvm_sw_cpu_context_t; /* @@ -873,6 +898,9 @@ typedef struct kvm_host_context { /* pointer to the top of 'pt_regs' structures list */ pt_regs_t *pt_regs; + /* some additional items of processes context */ + e2k_upsr_t upsr; /* user UPSR register state */ + unsigned osem; /* OSEM register state */ /* the host kernel's signal/trap stack of contexts */ kvm_signal_context_t signal; } kvm_host_context_t; @@ -896,6 +924,9 @@ struct kvm_vcpu_arch { /* support */ bool is_hv; /* VCPU is under hardware virtualized */ /* support */ + /* host switch to vcpu-host mode from host interception emulation mode */ + /* (trap or system call on PV mode) */ + bool from_pv_intc; kvm_vcpu_state_t *vcpu_state; kvm_vcpu_state_t *kmap_vcpu_state; /* alias of VCPU state */ @@ -990,6 +1021,9 @@ struct kvm_vcpu_arch { bool on_virqs_handling; /* VCPU is handling pending VIRQs */ bool vm_exit_wish; /* VCPU is need to VM exit and */ /* exit reason handling */ + bool trap_wish; /* VCPU is need to inject traps */ + bool hcall_irqs_disabled; /* VCPU entered HCALL with disabled interrupts */ + unsigned long trap_mask_wish; /* mask of traps to wish */ struct completion exited; /* guest VCPU thread completed */ struct completion released; /* all VCPU threads completed and */ /* VCPU can be freed */ @@ -1371,16 +1405,12 @@ static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) #define KVM_ARCH_WANT_MMU_NOTIFIER #ifdef KVM_ARCH_WANT_MMU_NOTIFIER -int kvm_unmap_hva_range(struct kvm *kvm, - unsigned long start, unsigned long end); +int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, unsigned flags); int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); #endif /* KVM_ARCH_WANT_MMU_NOTIFIER */ -extern int kvm_mmu_load(struct kvm_vcpu *vcpu, unsigned flags); -extern void kvm_mmu_unload(struct kvm_vcpu *vcpu, unsigned flags); - extern void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask); extern int kvm_wake_up_vcpu_host(struct kvm_vcpu *vcpu, int wait); diff --git a/arch/e2k/include/asm/l-iommu.h b/arch/e2k/include/asm/l-iommu.h index 6d54d0a..a251ba4 100644 --- a/arch/e2k/include/asm/l-iommu.h +++ b/arch/e2k/include/asm/l-iommu.h @@ -52,7 +52,7 @@ typedef struct { unsigned iopte; } iopte_t; #define addr_to_flush(__a) ((__a) >> IO_PAGE_SHIFT) -static inline void l_iommu_write(unsigned node, u32 val, unsigned long addr) +static inline void __l_iommu_write(unsigned node, u32 val, unsigned long addr) { sic_write_node_iolink_nbsr_reg(node, 0, addr, val); } @@ -62,10 +62,13 @@ static inline u32 l_iommu_read(unsigned node, unsigned long addr) return sic_read_node_iolink_nbsr_reg(node, 0, addr); } -static inline void l_iommu_set_ba(unsigned node, unsigned long *ba) +#define __l_iommu_set_ba __l_iommu_set_ba +static inline void __l_iommu_set_ba(unsigned node, unsigned long *ba) { - l_iommu_write(node, pa_to_iopte(ba[IOMMU_LOW_TABLE]), SIC_iommu_ba_lo); - l_iommu_write(node, pa_to_iopte(ba[IOMMU_HIGH_TABLE]), SIC_iommu_ba_hi); + __l_iommu_write(node, pa_to_iopte(ba[IOMMU_LOW_TABLE]), + SIC_iommu_ba_lo); + __l_iommu_write(node, pa_to_iopte(ba[IOMMU_HIGH_TABLE]), + SIC_iommu_ba_hi); } #define l_prefetch_iopte_supported l_prefetch_iopte_supported @@ -82,9 +85,9 @@ static inline void l_prefetch_iopte(iopte_t *iopte, int prefetch) iopte_val(iopte[0]) |= IOPTE_STP_PREF_IOPTE; } -static inline void *l_iommu_map_table(void *va, unsigned long size) +static inline void *l_iommu_map_table(unsigned long pa, unsigned long size) { - phys_addr_t start = __pa(va); + phys_addr_t start = pa; pgprot_t prot = pgprot_writecombine(PAGE_KERNEL); struct page **pages; phys_addr_t page_start; @@ -93,7 +96,7 @@ static inline void *l_iommu_map_table(void *va, unsigned long size) void *vaddr; if (!cpu_has(CPU_HWBUG_IOMMU)) - return va; + return __va(pa); page_start = start - offset_in_page(start); page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); diff --git a/arch/e2k/include/asm/l-mcmonitor.h b/arch/e2k/include/asm/l-mcmonitor.h index 360ae88..e69eacf 100644 --- a/arch/e2k/include/asm/l-mcmonitor.h +++ b/arch/e2k/include/asm/l-mcmonitor.h @@ -41,27 +41,20 @@ static inline bool l_mcmonitor_eec_enabled(void) #define L_MC_ECC_WORDS_NR 4 #define L_MCMONITOR_TEST_SIZE (256 * L_MC_ECC_WORDS_NR) -static inline void local_set_mc_ecc(void *node_nbsr, int num, unsigned int reg_value) -{ - nbsr_write(reg_value, node_nbsr + SIC_mc0_ecc + num * 0x40); -} - static inline void l_mcmonitor_fill_data(u64 *a, bool make_error) { int i, mc = SIC_MC_COUNT; int sz = L_MCMONITOR_TEST_SIZE / L_MC_ECC_WORDS_NR / sizeof(*a); e2k_mc_ecc_struct_t mc_ecc[SIC_MAX_MC_COUNT]; a = (void *)__pa(a); - - void *node_nbsr = sic_get_node_nbsr_base(0); - + for (i = 0; i < mc; i++) mc_ecc[i].E2K_MC_ECC_reg = sic_get_mc_ecc(0, i); - + for (i = 0; i < mc; i++) { l_mc_ecc_struct_t e = mc_ecc[i]; e.E2K_MC_ECC_dmode = 1; - local_set_mc_ecc(node_nbsr, i, e.E2K_MC_ECC_reg); + sic_set_mc_ecc(0, i, e.E2K_MC_ECC_reg); } mb(); @@ -78,7 +71,7 @@ static inline void l_mcmonitor_fill_data(u64 *a, bool make_error) } for (i = 0; i < mc; i++) - local_set_mc_ecc(node_nbsr, i, mc_ecc[i].E2K_MC_ECC_reg); + sic_set_mc_ecc(0, i, mc_ecc[i].E2K_MC_ECC_reg); mb(); } diff --git a/arch/e2k/include/asm/machdep.h b/arch/e2k/include/asm/machdep.h index 9e532e8..61bb325 100644 --- a/arch/e2k/include/asm/machdep.h +++ b/arch/e2k/include/asm/machdep.h @@ -122,12 +122,12 @@ typedef struct machdep { void (*save_kernel_gregs)(struct kernel_gregs *); void (*save_gregs)(struct global_regs *); - void (*save_local_gregs)(struct local_gregs *); + void (*save_local_gregs)(struct local_gregs *, bool is_signal); void (*save_gregs_dirty_bgr)(struct global_regs *); void (*save_gregs_on_mask)(struct global_regs *, bool dirty_bgr, unsigned long not_save_gregs_mask); void (*restore_gregs)(const struct global_regs *); - void (*restore_local_gregs)(const struct local_gregs *); + void (*restore_local_gregs)(const struct local_gregs *, bool is_signal); void (*restore_gregs_on_mask)(struct global_regs *, bool dirty_bgr, unsigned long not_restore_gregs_mask); @@ -598,8 +598,8 @@ extern void save_kernel_gregs_v2(struct kernel_gregs *); extern void save_kernel_gregs_v5(struct kernel_gregs *); extern void save_gregs_v2(struct global_regs *); extern void save_gregs_v5(struct global_regs *); -extern void save_local_gregs_v2(struct local_gregs *); -extern void save_local_gregs_v5(struct local_gregs *); +extern void save_local_gregs_v2(struct local_gregs *, bool is_signal); +extern void save_local_gregs_v5(struct local_gregs *, bool is_signal); extern void save_gregs_dirty_bgr_v2(struct global_regs *); extern void save_gregs_dirty_bgr_v5(struct global_regs *); extern void save_gregs_on_mask_v2(struct global_regs *, bool dirty_bgr, @@ -608,8 +608,8 @@ extern void save_gregs_on_mask_v5(struct global_regs *, bool dirty_bgr, unsigned long mask_not_save); extern void restore_gregs_v2(const struct global_regs *); extern void restore_gregs_v5(const struct global_regs *); -extern void restore_local_gregs_v2(const struct local_gregs *); -extern void restore_local_gregs_v5(const struct local_gregs *); +extern void restore_local_gregs_v2(const struct local_gregs *, bool is_signal); +extern void restore_local_gregs_v5(const struct local_gregs *, bool is_signal); extern void restore_gregs_on_mask_v2(struct global_regs *, bool dirty_bgr, unsigned long mask_not_restore); extern void restore_gregs_on_mask_v5(struct global_regs *, bool dirty_bgr, diff --git a/arch/e2k/include/asm/mas.h b/arch/e2k/include/asm/mas.h index 5430298..2481fbb 100644 --- a/arch/e2k/include/asm/mas.h +++ b/arch/e2k/include/asm/mas.h @@ -101,6 +101,56 @@ static inline bool is_mas_secondary_lock_trap_on_load_store(unsigned int mas) return (mas & 3) == 2; } +/* Note that 'chan', 'spec' and 'store' must also be checked */ +static inline bool is_mas_special_mmu_aau(unsigned int mas) +{ + return (mas & 7) == 7; +} + +/* mas is conflict check between ld and st */ +static inline bool is_mas_check(unsigned int mas) +{ + unsigned int m1 = (mas >> 3) & 0x2; + unsigned int big_endian = (mas >> 3) & 0x1; + unsigned int mod = mas & 0x7; + + return m1 == 0x0 && mod == 0x2 && !big_endian; +} + +/* mas is conflict check with unlock between ld and st */ +static inline bool is_mas_check_unlock(unsigned int mas) +{ + unsigned int m1 = (mas >> 3) & 0x2; + unsigned int big_endian = (mas >> 3) & 0x1; + unsigned int mod = mas & 0x7; + + return m1 == 0x0 && mod == 0x3 && !big_endian; +} + +/* mas is semi-speculative conflict lock check between ld and st */ +static inline bool is_mas_lock_check(unsigned int mas) +{ + unsigned int m1 = (mas >> 3) & 0x2; + unsigned int big_endian = (mas >> 3) & 0x1; + unsigned int mod = mas & 0x7; + unsigned int m2 = mas & 0x3; + + return (m1 == 0x0 && mod == 0x4 || m1 == 0x1 && m2 == 0x1) && + !big_endian; +} + +/* mas is speculative conflict lock check between ld and st */ +static inline bool is_mas_spec_lock_check(unsigned int mas) +{ + unsigned int m1 = (mas >> 3) & 0x2; + unsigned int big_endian = (mas >> 3) & 0x1; + unsigned int mod = mas & 0x7; + unsigned int m2 = mas & 0x3; + + return (m1 == 0x0 && mod == 0x7 || m1 == 0x1 && m2 == 0x3) && + !big_endian; +} + #endif #endif /* _E2K_MAS_H_ */ diff --git a/arch/e2k/include/asm/mman.h b/arch/e2k/include/asm/mman.h index bf5ca54..7937a46 100644 --- a/arch/e2k/include/asm/mman.h +++ b/arch/e2k/include/asm/mman.h @@ -57,23 +57,9 @@ static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot, vm_flags = cui << VM_CUI_SHIFT; - /* - * Check if we are allocating hardware stacks. - */ - if (current_thread_info()->status & TS_MMAP_DONTEXPAND) { - /* - * VM_DONTEXPAND makes sure that even if VM_MLOCK - * is set, this area won't be populated on mmap(). - */ - vm_flags |= VM_DONTEXPAND; - } - if (current_thread_info()->status & TS_MMAP_PRIVILEGED) vm_flags |= VM_PRIVILEGED; - if (current_thread_info()->status & TS_MMAP_DONTCOPY) - vm_flags |= VM_DONTCOPY; - if (current_thread_info()->status & TS_MMAP_PS) vm_flags |= VM_HW_STACK_PS; @@ -83,9 +69,6 @@ static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot, if (current_thread_info()->status & TS_MMAP_SIGNAL_STACK) vm_flags |= VM_SIGNAL_STACK; - if (current_thread_info()->status & TS_MMAP_NOHUGEPAGE) - vm_flags |= VM_NOHUGEPAGE; - return vm_flags; } #define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey) @@ -136,13 +119,13 @@ enum exec_mmu_ret { /* Trap cellar record should be executed again */ EXEC_MMU_REPEAT }; -extern int execute_mmu_operations(trap_cellar_t *tcellar, +extern enum exec_mmu_ret execute_mmu_operations(trap_cellar_t *tcellar, trap_cellar_t *next_tcellar, struct pt_regs *regs, int rg, int zeroing, e2k_addr_t *addr, bool (*is_spill_fill_recovery)(tc_cond_t cond, e2k_addr_t address, bool s_f, struct pt_regs *regs), - int (*calculate_rf_frame)(struct pt_regs *regs, + enum exec_mmu_ret (*calculate_rf_frame)(struct pt_regs *regs, tc_cond_t cond, u64 **radr, bool *load_to_rf)); diff --git a/arch/e2k/include/asm/mmu_context.h b/arch/e2k/include/asm/mmu_context.h index 777fe71..c417d88 100644 --- a/arch/e2k/include/asm/mmu_context.h +++ b/arch/e2k/include/asm/mmu_context.h @@ -182,65 +182,10 @@ enter_lazy_tlb (struct mm_struct *mm, struct task_struct *tsk) { } -/* - * Initialize a new mmu context. This is invoked when a new - * address space instance (unique or shared) is instantiated. - * This just needs to set mm->context[] to an invalid context. - */ -static inline int -__init_new_context(struct task_struct *p, struct mm_struct *mm, - mm_context_t *context) -{ - bool is_fork = p && (p != current); - int ret; - memset(&context->cpumsk, 0, nr_cpu_ids * sizeof(context->cpumsk[0])); - - if (is_fork) { - /* - * Copy data on user fork - */ - mm_context_t *curr_context = ¤t->mm->context; - - /* - * Copy cut mask from the context of parent process - * to the context of new process - */ - mutex_lock(&curr_context->cut_mask_lock); - bitmap_copy((unsigned long *) &context->cut_mask, - (unsigned long *) &curr_context->cut_mask, - USER_CUT_AREA_SIZE/sizeof(e2k_cute_t)); - mutex_unlock(&curr_context->cut_mask_lock); - } else { - /* - * Initialize by zero cut_mask of new process - */ - mutex_init(&context->cut_mask_lock); - bitmap_zero((unsigned long *) &context->cut_mask, - USER_CUT_AREA_SIZE/sizeof(e2k_cute_t)); - } - - atomic_set(&context->tstart, 1); - - init_rwsem(&context->sival_ptr_list_sem); - INIT_LIST_HEAD(&context->sival_ptr_list_head); - - INIT_LIST_HEAD(&context->delay_free_stacks); - init_rwsem(&context->core_lock); - - INIT_LIST_HEAD(&context->cached_stacks); - spin_lock_init(&context->cached_stacks_lock); - context->cached_stacks_size = 0; - - if (mm == NULL) - return 0; - - ret = hw_contexts_init(p, context, is_fork); - return ret; -} - -static inline int -init_new_context(struct task_struct *p, struct mm_struct *mm) +extern int __init_new_context(struct task_struct *p, struct mm_struct *mm, + mm_context_t *context); +static inline int init_new_context(struct task_struct *p, struct mm_struct *mm) { return __init_new_context(p, mm, &mm->context); } diff --git a/arch/e2k/include/asm/mmu_fault.h b/arch/e2k/include/asm/mmu_fault.h index 7e49779..a234ca2 100644 --- a/arch/e2k/include/asm/mmu_fault.h +++ b/arch/e2k/include/asm/mmu_fault.h @@ -32,7 +32,7 @@ native_guest_ptr_to_host(void *ptr, int size) return ptr; } -static inline long +static inline void native_recovery_faulted_tagged_store(e2k_addr_t address, u64 wr_data, u32 data_tag, u64 st_rec_opc, u64 data_ext, u32 data_ext_tag, u64 opc_ext, int chan, int qp_store, int atomic_store) @@ -45,10 +45,8 @@ native_recovery_faulted_tagged_store(e2k_addr_t address, u64 wr_data, st_rec_opc, data_ext, data_ext_tag, opc_ext, chan, qp_store); } - - return 0; } -static inline long +static inline void native_recovery_faulted_load(e2k_addr_t address, u64 *ld_val, u8 *data_tag, u64 ld_rec_opc, int chan) { @@ -58,25 +56,23 @@ native_recovery_faulted_load(e2k_addr_t address, u64 *ld_val, u8 *data_tag, NATIVE_RECOVERY_TAGGED_LOAD_TO(address, ld_rec_opc, val, tag, chan); *ld_val = val; *data_tag = tag; - return 0; } -static inline long +static inline void native_recovery_faulted_move(e2k_addr_t addr_from, e2k_addr_t addr_to, e2k_addr_t addr_to_hi, int vr, u64 ld_rec_opc, int chan, - int qp_load, int atomic_load) + int qp_load, int atomic_load, u32 first_time) { if (atomic_load) { NATIVE_MOVE_TAGGED_DWORD_WITH_OPC_VR_ATOMIC(addr_from, addr_to, addr_to_hi, vr, ld_rec_opc); } else { NATIVE_MOVE_TAGGED_DWORD_WITH_OPC_CH_VR(addr_from, addr_to, - addr_to_hi, vr, ld_rec_opc, chan, qp_load); + addr_to_hi, vr, ld_rec_opc, chan, qp_load, + first_time); } - - return 0; } -static inline long +static inline void native_recovery_faulted_load_to_cpu_greg(e2k_addr_t address, u32 greg_num_d, int vr, u64 ld_rec_opc, int chan_opc, int qp_load, int atomic_load) @@ -88,24 +84,23 @@ native_recovery_faulted_load_to_cpu_greg(e2k_addr_t address, u32 greg_num_d, NATIVE_RECOVERY_LOAD_TO_A_GREG_CH_VR(address, ld_rec_opc, greg_num_d, chan_opc, vr, qp_load); } - - return 0; } -static inline long +static inline void native_recovery_faulted_load_to_greg(e2k_addr_t address, u32 greg_num_d, int vr, u64 ld_rec_opc, int chan_opc, int qp_load, int atomic_load, u64 *saved_greg_lo, u64 *saved_greg_hi) { if (!saved_greg_lo) { - return native_recovery_faulted_load_to_cpu_greg(address, + native_recovery_faulted_load_to_cpu_greg(address, greg_num_d, vr, ld_rec_opc, chan_opc, qp_load, atomic_load); } else { - return native_recovery_faulted_move(address, + native_recovery_faulted_move(address, (u64) saved_greg_lo, (u64) saved_greg_hi, - vr, ld_rec_opc, chan_opc, qp_load, atomic_load); + vr, ld_rec_opc, chan_opc, qp_load, + atomic_load, 1); } } @@ -118,27 +113,21 @@ native_is_guest_kernel_gregs(struct thread_info *ti, return false; } -static inline long +static inline void native_move_tagged_word(e2k_addr_t addr_from, e2k_addr_t addr_to) { NATIVE_MOVE_TAGGED_WORD(addr_from, addr_to); - - return 0; } -static inline long +static inline void native_move_tagged_dword(e2k_addr_t addr_from, e2k_addr_t addr_to) { NATIVE_MOVE_TAGGED_DWORD(addr_from, addr_to); - - return 0; } -static inline long +static inline void native_move_tagged_qword(e2k_addr_t addr_from, e2k_addr_t addr_to) { NATIVE_MOVE_TAGGED_QWORD(addr_from, addr_from + sizeof(long), addr_to, addr_to + sizeof(long)); - - return 0; } extern void native_handle_mpdma_fault(e2k_addr_t hva); @@ -153,39 +142,40 @@ extern e2k_addr_t print_address_ptes(pgd_t *pgdp, e2k_addr_t address, #if !defined(CONFIG_PARAVIRT_GUEST) && !defined(CONFIG_KVM_GUEST_KERNEL) /* it is native kernel without any virtualization */ /* or it is native host kernel with virtualization support */ -static inline long +static inline void recovery_faulted_tagged_store(e2k_addr_t address, u64 wr_data, u32 data_tag, u64 st_rec_opc, u64 data_ext, u32 data_ext_tag, u64 opc_ext, int chan, int qp_store, int atomic_store) { - return native_recovery_faulted_tagged_store(address, wr_data, data_tag, + native_recovery_faulted_tagged_store(address, wr_data, data_tag, st_rec_opc, data_ext, data_ext_tag, opc_ext, chan, qp_store, atomic_store); } -static inline long +static inline void recovery_faulted_load(e2k_addr_t address, u64 *ld_val, u8 *data_tag, - u64 ld_rec_opc, int chan) + u64 ld_rec_opc, int chan, tc_cond_t cond) { - return native_recovery_faulted_load(address, ld_val, data_tag, + native_recovery_faulted_load(address, ld_val, data_tag, ld_rec_opc, chan); } -static inline long +static inline void recovery_faulted_load_to_greg(e2k_addr_t address, u32 greg_num_d, int vr, u64 ld_rec_opc, int chan, int qp_load, int atomic_load, u64 *saved_greg_lo, - u64 *saved_greg_hi) + u64 *saved_greg_hi, tc_cond_t cond) { - return native_recovery_faulted_load_to_greg(address, greg_num_d, + native_recovery_faulted_load_to_greg(address, greg_num_d, vr, ld_rec_opc, chan, qp_load, atomic_load, saved_greg_lo, saved_greg_hi); } -static inline long +static inline void recovery_faulted_move(e2k_addr_t addr_from, e2k_addr_t addr_to, e2k_addr_t addr_to_hi, int vr, u64 ld_rec_opc, int chan, - int qp_load, int atomic_load) + int qp_load, int atomic_load, u32 first_time, + tc_cond_t cond) { - return native_recovery_faulted_move(addr_from, addr_to, addr_to_hi, vr, - ld_rec_opc, chan, qp_load, atomic_load); + native_recovery_faulted_move(addr_from, addr_to, addr_to_hi, vr, + ld_rec_opc, chan, qp_load, atomic_load, first_time); } static inline bool @@ -217,12 +207,12 @@ handle_mpdma_fault(e2k_addr_t hva) # ifndef CONFIG_VIRTUALIZATION /* it is native kernel without any virtualization */ -static inline int guest_addr_to_host(void **addr, pt_regs_t *regs) +static inline int guest_addr_to_host(void **addr, const pt_regs_t *regs) { return native_guest_addr_to_host(addr); } -static inline void *guest_ptr_to_host(void *ptr, int size, pt_regs_t *regs) +static inline void *guest_ptr_to_host(void *ptr, int size, const pt_regs_t *regs) { return native_guest_ptr_to_host(ptr, size); } @@ -241,31 +231,31 @@ static inline void *guest_ptr_to_host(void *ptr, int size, pt_regs_t *regs) #error "Unknown virtualization type" #endif /* ! CONFIG_PARAVIRT_GUEST && ! CONFIG_KVM_GUEST_KERNEL */ -static inline long +static inline void store_tagged_dword(void *address, u64 data, u32 tag) { - return recovery_faulted_tagged_store((e2k_addr_t) address, data, tag, + recovery_faulted_tagged_store((e2k_addr_t) address, data, tag, TAGGED_MEM_STORE_REC_OPC, 0, 0, 0, 1, 0, 0); } -static inline long +static inline void load_value_and_tagd(const void *address, u64 *ld_val, u8 *ld_tag) { - return recovery_faulted_load((e2k_addr_t) address, ld_val, ld_tag, - TAGGED_MEM_LOAD_REC_OPC, 0); + recovery_faulted_load((e2k_addr_t) address, ld_val, ld_tag, + TAGGED_MEM_LOAD_REC_OPC, 0, + (tc_cond_t) {.word = 0}); } -static inline long +static inline void load_qvalue_and_tagq(e2k_addr_t address, u64 *val_lo, u64 *val_hi, u8 *tag_lo, u8 *tag_hi) { - long ret; - - ret = recovery_faulted_load(address, val_lo, tag_lo, - TAGGED_MEM_LOAD_REC_OPC, 0); - ret |= recovery_faulted_load(address + sizeof(long), val_hi, tag_hi, - TAGGED_MEM_LOAD_REC_OPC, 0); - return ret; + recovery_faulted_load(address, val_lo, tag_lo, + TAGGED_MEM_LOAD_REC_OPC, 0, + (tc_cond_t) {.word = 0}); + recovery_faulted_load(address + sizeof(long), val_hi, tag_hi, + TAGGED_MEM_LOAD_REC_OPC, 0, + (tc_cond_t) {.word = 0}); } #endif /* _E2K_MMU_FAULT_H_ */ diff --git a/arch/e2k/include/asm/mmu_regs.h b/arch/e2k/include/asm/mmu_regs.h index 8556630..8fd7e0b 100644 --- a/arch/e2k/include/asm/mmu_regs.h +++ b/arch/e2k/include/asm/mmu_regs.h @@ -760,29 +760,103 @@ read_CLW_reg(clw_addr_t clw_addr) return READ_CLW_REG(clw_addr); } +static inline clw_reg_t +native_read_CLW_reg(clw_addr_t clw_addr) +{ + DebugCLW("Read CLW reg 0x%lx\n", clw_addr); + return NATIVE_READ_CLW_REG(clw_addr); +} + /* * Read CLW bottom register */ -#define read_US_CL_B() read_CLW_reg(ADDR_US_CL_B) -#define READ_US_CL_B() READ_CLW_REG(ADDR_US_CL_B) +#define read_US_CL_B() read_CLW_reg(ADDR_US_CL_B) +#define READ_US_CL_B() READ_CLW_REG(ADDR_US_CL_B) +#define native_read_US_CL_B() native_read_CLW_reg(ADDR_US_CL_B) +#define NATIVE_READ_US_CL_B() NATIVE_READ_CLW_REG(ADDR_US_CL_B) /* * Read CLW up register */ -#define read_US_CL_UP() read_CLW_reg(ADDR_US_CL_UP) -#define READ_US_CL_UP() READ_CLW_REG(ADDR_US_CL_UP) +#define read_US_CL_UP() read_CLW_reg(ADDR_US_CL_UP) +#define READ_US_CL_UP() READ_CLW_REG(ADDR_US_CL_UP) +#define native_read_US_CL_UP() native_read_CLW_reg(ADDR_US_CL_UP) +#define NATIVE_READ_US_CL_UP() NATIVE_READ_CLW_REG(ADDR_US_CL_UP) /* * Read CLW bit-mask registers */ -#define read_US_CL_M0() read_CLW_reg(ADDR_US_CL_M0) -#define READ_US_CL_M0() READ_CLW_REG(ADDR_US_CL_M0) -#define read_US_CL_M1() read_CLW_reg(ADDR_US_CL_M1) -#define READ_US_CL_M1() READ_CLW_REG(ADDR_US_CL_M1) -#define read_US_CL_M2() read_CLW_reg(ADDR_US_CL_M2) -#define READ_US_CL_M2() READ_CLW_REG(ADDR_US_CL_M2) -#define read_US_CL_M3() read_CLW_reg(ADDR_US_CL_M3) -#define READ_US_CL_M3() READ_CLW_REG(ADDR_US_CL_M3) +#define read_US_CL_M0() read_CLW_reg(ADDR_US_CL_M0) +#define READ_US_CL_M0() READ_CLW_REG(ADDR_US_CL_M0) +#define read_US_CL_M1() read_CLW_reg(ADDR_US_CL_M1) +#define READ_US_CL_M1() READ_CLW_REG(ADDR_US_CL_M1) +#define read_US_CL_M2() read_CLW_reg(ADDR_US_CL_M2) +#define READ_US_CL_M2() READ_CLW_REG(ADDR_US_CL_M2) +#define read_US_CL_M3() read_CLW_reg(ADDR_US_CL_M3) +#define READ_US_CL_M3() READ_CLW_REG(ADDR_US_CL_M3) +#define native_read_US_CL_M0() native_read_CLW_reg(ADDR_US_CL_M0) +#define NATIVE_READ_US_CL_M0() NATIVE_READ_CLW_REG(ADDR_US_CL_M0) +#define native_read_US_CL_M1() native_read_CLW_reg(ADDR_US_CL_M1) +#define NATIVE_READ_US_CL_M1() NATIVE_READ_CLW_REG(ADDR_US_CL_M1) +#define native_read_US_CL_M2() native_read_CLW_reg(ADDR_US_CL_M2) +#define NATIVE_READ_US_CL_M2() NATIVE_READ_CLW_REG(ADDR_US_CL_M2) +#define native_read_US_CL_M3() native_read_CLW_reg(ADDR_US_CL_M3) +#define NATIVE_READ_US_CL_M3() NATIVE_READ_CLW_REG(ADDR_US_CL_M3) + +/* + * Write CLW register + */ + +static inline void +write_CLW_reg(clw_addr_t clw_addr, clw_reg_t val) +{ + DebugCLW("Write CLW reg 0x%lx value 0x%lx\n", clw_addr, val); + WRITE_CLW_REG(clw_addr, val); +} + +static inline void +native_write_CLW_reg(clw_addr_t clw_addr, clw_reg_t val) +{ + DebugCLW("Write CLW reg 0x%lx value 0x%lx\n", clw_addr, val); + NATIVE_WRITE_CLW_REG(clw_addr, val); +} + +/* + * Write CLW bottom register + */ +#define write_US_CL_B(val) write_CLW_reg(ADDR_US_CL_B, val) +#define WRITE_US_CL_B(val) WRITE_CLW_REG(ADDR_US_CL_B, val) +#define native_write_US_CL_B(val) native_write_CLW_reg(ADDR_US_CL_B, val) +#define NATIVE_WRITE_US_CL_B(val) NATIVE_WRITE_CLW_REG(ADDR_US_CL_B, val) + +/* + * Write CLW up register + */ +#define write_US_CL_UP(val) write_CLW_reg(ADDR_US_CL_UP, val) +#define WRITE_US_CL_UP(val) WRITE_CLW_REG(ADDR_US_CL_UP, val) +#define native_write_US_CL_UP(val) native_write_CLW_reg(ADDR_US_CL_UP, val) +#define NATIVE_WRITE_US_CL_UP(val) NATIVE_WRITE_CLW_REG(ADDR_US_CL_UP, val) + +/* + * Write CLW bit-mask registers + */ +#define write_US_CL_M0(val) write_CLW_reg(ADDR_US_CL_M0, val) +#define WRITE_US_CL_M0(val) WRITE_CLW_REG(ADDR_US_CL_M0, val) +#define write_US_CL_M1(val) write_CLW_reg(ADDR_US_CL_M1, val) +#define WRITE_US_CL_M1(val) WRITE_CLW_REG(ADDR_US_CL_M1, val) +#define write_US_CL_M2(val) write_CLW_reg(ADDR_US_CL_M2, val) +#define WRITE_US_CL_M2(val) WRITE_CLW_REG(ADDR_US_CL_M2, val) +#define write_US_CL_M3(val) write_CLW_reg(ADDR_US_CL_M3, val) +#define WRITE_US_CL_M3(val) WRITE_CLW_REG(ADDR_US_CL_M3, val) +#define native_write_US_CL_M0(val) native_write_CLW_reg(ADDR_US_CL_M0, val) +#define NATIVE_WRITE_US_CL_M0(val) NATIVE_WRITE_CLW_REG(ADDR_US_CL_M0, val) +#define native_write_US_CL_M1(val) native_write_CLW_reg(ADDR_US_CL_M1, val) +#define NATIVE_WRITE_US_CL_M1(val) NATIVE_WRITE_CLW_REG(ADDR_US_CL_M1, val) +#define native_write_US_CL_M2(val) native_write_CLW_reg(ADDR_US_CL_M2, val) +#define NATIVE_WRITE_US_CL_M2(val) NATIVE_WRITE_CLW_REG(ADDR_US_CL_M2, val) +#define native_write_US_CL_M3(val) native_write_CLW_reg(ADDR_US_CL_M3, val) +#define NATIVE_WRITE_US_CL_M3(val) NATIVE_WRITE_CLW_REG(ADDR_US_CL_M3, val) + #endif /* ! __ASSEMBLY__ */ diff --git a/arch/e2k/include/asm/mmu_regs_access.h b/arch/e2k/include/asm/mmu_regs_access.h index 7170638..946ce5f 100644 --- a/arch/e2k/include/asm/mmu_regs_access.h +++ b/arch/e2k/include/asm/mmu_regs_access.h @@ -227,6 +227,12 @@ extern void boot_native_write_MMU_OS_VAB_reg_value(unsigned long value); #define READ_CLW_REG(clw_addr) \ NATIVE_READ_CLW_REG(clw_addr) +/* + * Write CLW register + */ +#define WRITE_CLW_REG(clw_addr, val) \ + NATIVE_WRITE_CLW_REG(clw_addr, val) + /* * MMU DEBUG registers access */ diff --git a/arch/e2k/include/asm/mmu_regs_types.h b/arch/e2k/include/asm/mmu_regs_types.h index a47f6b6..7dcf688 100644 --- a/arch/e2k/include/asm/mmu_regs_types.h +++ b/arch/e2k/include/asm/mmu_regs_types.h @@ -320,14 +320,6 @@ typedef unsigned long long mmu_reg_t; #define mmu_trap_count_get(mmu_reg) MMU_TRAP_COUNT_GET(mmu_reg) -/* - * MMU Memory Protection Table Base MMU_MPT_B - * The base address of Memory Protection Table, - * aligned to table size - */ - -#define _MMU_MPT_B 0x000000fffffff000UL - /* * MMU PCI Low Bound MMU_PCI_L_B * Fix the boundary between PCIand main memory addresses @@ -688,12 +680,12 @@ typedef e2k_addr_t flush_addr_t; typedef e2k_addr_t clw_addr_t; #endif /* __ASSEMBLY__ */ -#define US_CL_B_NO 0x024 /* User stack bottom to clean */ -#define US_CL_UP_NO 0x124 /* User stack up to clean */ -#define US_CL_M0_NO 0x004 /* User stack bit-mask [0:63] */ -#define US_CL_M1_NO 0x084 /* User stack bit-mask [64:127] */ -#define US_CL_M2_NO 0x104 /* User stack bit-mask [128:195] */ -#define US_CL_M3_NO 0x184 /* User stack bit-mask [196:255] */ +#define ADDR_US_CL_B 0x024 /* User stack bottom to clean */ +#define ADDR_US_CL_UP 0x124 /* User stack up to clean */ +#define ADDR_US_CL_M0 0x004 /* User stack bit-mask [0:63] */ +#define ADDR_US_CL_M1 0x084 /* User stack bit-mask [64:127] */ +#define ADDR_US_CL_M2 0x104 /* User stack bit-mask [128:195] */ +#define ADDR_US_CL_M3 0x184 /* User stack bit-mask [196:255] */ /* CLW internel register contents */ diff --git a/arch/e2k/include/asm/mmu_types.h b/arch/e2k/include/asm/mmu_types.h index 1e60924..d85b8f0 100644 --- a/arch/e2k/include/asm/mmu_types.h +++ b/arch/e2k/include/asm/mmu_types.h @@ -411,6 +411,11 @@ struct mmu_tc_opcode { #ifndef __ASSEMBLY__ +static inline bool tc_fmt_has_valid_mask(int fmt) +{ + return fmt == LDST_QP_FMT || fmt == TC_FMT_QWORD_QP || fmt == TC_FMT_DWORD_QP; +} + typedef union { unsigned word; @@ -500,6 +505,56 @@ typedef union { #define TC_COND_FMT_FULL(cond) (AS(cond).fmt | (AS(cond).fmtc << 3)) +static inline bool tc_cond_is_special_mmu_aau(tc_cond_t cond) +{ + unsigned int mas = AS(cond).mas; + int chan = AS(cond).chan; + int store = AS(cond).store; + int spec_mode = AS(cond).spec; + + if (unlikely(is_mas_special_mmu_aau(mas) && (store || + !store && !spec_mode && (chan == 1 || chan == 3)))) + return true; + + return false; +} + +static inline bool tc_cond_is_check_ld(tc_cond_t cond) +{ + unsigned int mas = AS(cond).mas; + int store = AS(cond).store; + int spec_mode = AS(cond).spec; + + return is_mas_check(mas) && !spec_mode && !store; +} + +static inline bool tc_cond_is_check_unlock_ld(tc_cond_t cond) +{ + unsigned int mas = AS(cond).mas; + int store = AS(cond).store; + int spec_mode = AS(cond).spec; + + return is_mas_check_unlock(mas) && !spec_mode && !store; +} + +static inline bool tc_cond_is_lock_check_ld(tc_cond_t cond) +{ + unsigned int mas = AS(cond).mas; + int store = AS(cond).store; + int spec_mode = AS(cond).spec; + + return is_mas_lock_check(mas) && spec_mode && !store; +} + +static inline bool tc_cond_is_spec_lock_check_ld(tc_cond_t cond) +{ + unsigned int mas = AS(cond).mas; + int store = AS(cond).store; + int spec_mode = AS(cond).spec; + + return is_mas_spec_lock_check(mas) && spec_mode && !store; +} + /* * Caveat: for qword accesses this will return 16 bytes for * the first entry in trap cellar and 8 bytes for the second one. @@ -746,35 +801,6 @@ typedef union { #define LD_ST_REC_OPC_mask(ld_st_rec) (ld_st_rec.mask) #define LD_ST_REC_OPC_reg(ld_st_rec) (ld_st_rec.word) -typedef enum ld_st_rec_mode { - primary_rec_mode = 0, /* primary, privileged, */ - primary_prot_rec_mode = 1, /* primary, privileged, protected */ - secondary_rec_mode = 2, /* secondary, privileged */ - guest_physical_rec_mode = 3, /* guest, physical, privileged */ - primary_user_rec_mode = 4, /* primary */ - guest_primary_rec_mode = 5, /* guest, primary, privileged, prot */ - secondary_user_rec_mode = 6, /* secondary, not privileged */ -} ld_st_rec_mode_t; - -static inline ld_st_rec_mode_t -get_ld_st_rec_opc_mode(ldst_rec_op_t rec_opcode) -{ - unsigned mode = 0; - - mode |= LD_ST_REC_OPC_prot(rec_opcode) ? 0x01 : 0x00; - mode |= LD_ST_REC_OPC_root(rec_opcode) ? 0x02 : 0x00; - mode |= LD_ST_REC_OPC_mode_h(rec_opcode) ? 0x04 : 0x00; - return (ld_st_rec_mode_t)mode; -} -static inline ldst_rec_op_t -set_ld_st_rec_opc_mode(ldst_rec_op_t rec_opcode, ld_st_rec_mode_t mode) -{ - LD_ST_REC_OPC_prot(rec_opcode) = (mode & 0x01) ? 1 : 0; - LD_ST_REC_OPC_root(rec_opcode) = (mode & 0x02) ? 1 : 0; - LD_ST_REC_OPC_mode_h(rec_opcode) = (mode & 0x04) ? 1 : 0; - return rec_opcode; -} - #endif /* ! __ASSEMBLY__ */ #define LDST_REC_OPC_BYPASS_L1 (MAS_BYPASS_L1_CACHE << \ @@ -790,6 +816,7 @@ set_ld_st_rec_opc_mode(ldst_rec_op_t rec_opcode, ld_st_rec_mode_t mode) MAS_FILL_OPERATION << LDST_REC_OPC_MAS_SHIFT) #define TAGGED_MEM_STORE_REC_OPC (LDST_QWORD_FMT << LDST_REC_OPC_FMT_SHIFT) #define TAGGED_MEM_STORE_REC_OPC_W (LDST_WORD_FMT << LDST_REC_OPC_FMT_SHIFT) +#define MEM_STORE_REC_OPC_B (LDST_BYTE_FMT << LDST_REC_OPC_FMT_SHIFT) #endif /* _E2K_MMU_TYPES_H_ */ diff --git a/arch/e2k/include/asm/native_mmu_regs_access.h b/arch/e2k/include/asm/native_mmu_regs_access.h index a949b76..fcc700c 100644 --- a/arch/e2k/include/asm/native_mmu_regs_access.h +++ b/arch/e2k/include/asm/native_mmu_regs_access.h @@ -47,6 +47,13 @@ #define NATIVE_READ_MMU_TRAP_POINT() \ NATIVE_READ_MMU_REG( \ _MMU_REG_NO_TO_MMU_ADDR_VAL(_MMU_TRAP_POINT_NO)) +#define NATIVE_WRITE_MMU_US_CL_D(us_cl_d) \ + NATIVE_WRITE_MMU_REG( \ + _MMU_REG_NO_TO_MMU_ADDR_VAL(_MMU_US_CL_D_NO), \ + mmu_reg_val(us_cl_d)) +#define NATIVE_READ_MMU_US_CL_D() \ + NATIVE_READ_MMU_REG( \ + _MMU_REG_NO_TO_MMU_ADDR_VAL(_MMU_US_CL_D_NO)) #define NATIVE_WRITE_MMU_OS_PPTB_REG_VALUE(mmu_phys_ptb) \ NATIVE_WRITE_MMU_REG( \ _MMU_REG_NO_TO_MMU_ADDR_VAL(_MMU_OS_PPTB_NO), \ @@ -226,6 +233,12 @@ native_flush_ICACHE_all(void) #define NATIVE_READ_CLW_REG(clw_addr) \ NATIVE_READ_MAS_D_5((clw_addr), MAS_CLW_REG) +/* + * Write CLW register + */ +#define NATIVE_WRITE_CLW_REG(clw_addr, val) \ + NATIVE_WRITE_MAS_D((clw_addr), (val), MAS_CLW_REG) + /* * native MMU DEBUG registers access */ diff --git a/arch/e2k/include/asm/page.h b/arch/e2k/include/asm/page.h index bb17b84..27f44a4 100644 --- a/arch/e2k/include/asm/page.h +++ b/arch/e2k/include/asm/page.h @@ -249,7 +249,6 @@ extern struct page *e2k_virt_to_page(const void *kaddr); #define VM_HW_STACK_PCS 0x00400000000UL /* chain stack area */ #define VM_WRITECOMBINED 0x00800000000UL #define VM_PRIVILEGED 0x04000000000UL /* pages are privileged */ -#define VM_GUARD 0x08000000000UL /* guard page(s) mapping */ #define VM_MPDMA 0x10000000000UL /* pages are under MPDMA */ /* hardware protection */ #define VM_SIGNAL_STACK 0x20000000000UL /* Signal stack area */ diff --git a/arch/e2k/include/asm/paravirt/mmu_regs_access.h b/arch/e2k/include/asm/paravirt/mmu_regs_access.h index 212ef96..67d76d3 100644 --- a/arch/e2k/include/asm/paravirt/mmu_regs_access.h +++ b/arch/e2k/include/asm/paravirt/mmu_regs_access.h @@ -215,6 +215,15 @@ PV_READ_CLW_REG(clw_addr_t clw_addr) return pv_mmu_ops.read_clw_reg(clw_addr); } +/* + * Write CLW register + */ +static inline void +PV_WRITE_CLW_REG(clw_addr_t clw_addr, clw_reg_t val) +{ + pv_mmu_ops.write_clw_reg(clw_addr, val); +} + /* * MMU DEBUG registers access */ @@ -531,6 +540,16 @@ READ_CLW_REG(clw_addr_t clw_addr) return PV_READ_CLW_REG(clw_addr); } +/* + * Write CLW register + */ + +static inline void +WRITE_CLW_REG(clw_addr_t clw_addr, clw_reg_t val) +{ + PV_WRITE_CLW_REG(clw_addr, val); +} + /* * KVM MMU DEBUG registers access */ diff --git a/arch/e2k/include/asm/paravirt/pv_ops.h b/arch/e2k/include/asm/paravirt/pv_ops.h index 546c770..96c28c3 100644 --- a/arch/e2k/include/asm/paravirt/pv_ops.h +++ b/arch/e2k/include/asm/paravirt/pv_ops.h @@ -367,8 +367,6 @@ typedef struct pv_cpu_ops { bool proc_bounds, bool chain_bounds); irqreturn_t (*handle_interrupt)(struct pt_regs *regs); void (*init_guest_system_handlers_table)(void); - void (*handle_deferred_traps_in_syscall)(struct pt_regs *regs, - bool use_pt_regs, bool new_hs); void (*fix_process_pt_regs)(struct thread_info *ti, struct e2k_stacks *stacks, struct pt_regs *regs, struct pt_regs *old_regs); @@ -390,8 +388,10 @@ typedef struct pv_cpu_ops { unsigned long (*fast_tagged_memory_set)(void *addr, u64 val, u64 tag, size_t len, u64 strd_opcode); unsigned long (*extract_tags_32)(u16 *dst, const void *src); - void (*save_local_glob_regs)(struct local_gregs *l_gregs); - void (*restore_local_glob_regs)(struct local_gregs *l_gregs); + void (*save_local_glob_regs)(struct local_gregs *l_gregs, + bool is_signal); + void (*restore_local_glob_regs)(struct local_gregs *l_gregs, + bool is_signal); void (*restore_kernel_gregs_in_syscall)(struct thread_info *ti); void (*get_all_user_glob_regs)(struct global_regs *gregs); void (*arch_setup_machine)(void); @@ -574,6 +574,7 @@ typedef struct pv_mmu_ops { probe_entry_t (*entry_probe_mmu_op)(e2k_addr_t virt_addr); probe_entry_t (*address_probe_mmu_op)(e2k_addr_t virt_addr); clw_reg_t (*read_clw_reg)(clw_addr_t clw_addr); + void (*write_clw_reg)(clw_addr_t clw_addr, clw_reg_t val); void (*save_DAM)(unsigned long long *dam); void (*write_mmu_debug_reg)(int reg_no, mmu_reg_t mmu_reg); mmu_reg_t (*read_mmu_debug_reg)(int reg_no); diff --git a/arch/e2k/include/asm/paravirt/regs_state.h b/arch/e2k/include/asm/paravirt/regs_state.h index 3a0e8aa..b7294b9 100644 --- a/arch/e2k/include/asm/paravirt/regs_state.h +++ b/arch/e2k/include/asm/paravirt/regs_state.h @@ -15,14 +15,14 @@ }) static inline void -pv_save_local_glob_regs(local_gregs_t *l_gregs) +pv_save_local_glob_regs(local_gregs_t *l_gregs, is_signal) { - pv_cpu_ops.save_local_glob_regs(l_gregs); + pv_cpu_ops.save_local_glob_regs(l_gregs, is_signal); } static inline void -pv_restore_local_glob_regs(local_gregs_t *l_gregs) +pv_restore_local_glob_regs(local_gregs_t *l_gregs, bool is_signal) { - pv_cpu_ops.restore_local_glob_regs(l_gregs); + pv_cpu_ops.restore_local_glob_regs(l_gregs, is_signal); } static inline void pv_get_all_user_glob_regs(struct global_regs *gregs) @@ -46,14 +46,14 @@ pv_restore_kernel_gregs_in_syscall(struct thread_info *ti) #define INIT_G_REGS() PV_INIT_G_REGS() static inline void -save_local_glob_regs(local_gregs_t *l_gregs) +save_local_glob_regs(local_gregs_t *l_gregs, bool is_signal) { - pv_save_local_glob_regs(l_gregs); + pv_save_local_glob_regs(l_gregs, is_signal); } static inline void -restore_local_glob_regs(local_gregs_t *l_gregs) +restore_local_glob_regs(local_gregs_t *l_gregs, bool is_signal) { - pv_restore_local_glob_regs(l_gregs); + pv_restore_local_glob_regs(l_gregs, is_signal); } static inline void get_all_user_glob_regs(struct global_regs *gregs) diff --git a/arch/e2k/include/asm/paravirt/trap_table.h b/arch/e2k/include/asm/paravirt/trap_table.h index a8a9be7..11539c0 100644 --- a/arch/e2k/include/asm/paravirt/trap_table.h +++ b/arch/e2k/include/asm/paravirt/trap_table.h @@ -9,15 +9,6 @@ #define pv_ttable_entry3 (pv_cpu_ops.trap_table_entry3) #define pv_ttable_entry4 (pv_cpu_ops.trap_table_entry4) -static inline void -pv_handle_deferred_traps_in_syscall(struct pt_regs *regs, - bool use_pt_regs, bool new_hs) -{ - if (pv_cpu_ops.handle_deferred_traps_in_syscall) - pv_cpu_ops.handle_deferred_traps_in_syscall(regs, - use_pt_regs, new_hs); -} - static inline void pv_exit_handle_syscall(e2k_addr_t sbr, e2k_usd_hi_t usd_hi, e2k_usd_lo_t usd_lo, e2k_upsr_t upsr) @@ -80,13 +71,6 @@ exit_handle_syscall(e2k_addr_t sbr, e2k_usd_hi_t usd_hi, pv_exit_handle_syscall(sbr, usd_hi, usd_lo, upsr); } -static inline void -handle_deferred_traps_in_syscall(struct pt_regs *regs, - bool use_pt_regs, bool new_hs) -{ - pv_handle_deferred_traps_in_syscall(regs, use_pt_regs, new_hs); -} - static inline bool is_proc_stack_bounds(struct thread_info *ti, struct pt_regs *regs) { diff --git a/arch/e2k/include/asm/perf_event.h b/arch/e2k/include/asm/perf_event.h index 8f989f7..347ae52 100644 --- a/arch/e2k/include/asm/perf_event.h +++ b/arch/e2k/include/asm/perf_event.h @@ -2,6 +2,7 @@ #define _ASM_E2K_PERF_EVENT_H #include +#include #include static inline void set_perf_event_pending(void) {} diff --git a/arch/e2k/include/asm/pgalloc.h b/arch/e2k/include/asm/pgalloc.h index 0045da6..f29f948 100644 --- a/arch/e2k/include/asm/pgalloc.h +++ b/arch/e2k/include/asm/pgalloc.h @@ -42,8 +42,6 @@ extern void __init *node_early_get_zeroed_page(int nid); extern int mem_init_done; -#define check_pgt_cache() do { } while (0) - static inline void pgd_ctor(pgd_t *pgd) { int root_pt_index; diff --git a/arch/e2k/include/asm/pgtable.h b/arch/e2k/include/asm/pgtable.h index 092f74d..77f912b 100644 --- a/arch/e2k/include/asm/pgtable.h +++ b/arch/e2k/include/asm/pgtable.h @@ -377,14 +377,18 @@ static inline void untrack_pfn_moved(struct vm_area_struct *vma) #define NATIVE_VMALLOC_START (NATIVE_KERNEL_IMAGE_AREA_BASE + \ 0x020000000000UL) /* 0x0000 e400 0000 0000 */ -#define NATIVE_VMALLOC_END (NATIVE_VMALLOC_START + 0x010000000000UL) - /* 0x0000 e500 0000 0000 */ -#define NATIVE_VMEMMAP_START (NATIVE_VMALLOC_END + 0x010000000000UL) - /* 0x0000 e600 0000 0000 */ +/* We need big enough vmalloc area since usage of pcpu_embed_first_chunk() + * on e2k leads to having pcpu area span large ranges, and vmalloc area + * should be able to span those same ranges (see pcpu_embed_first_chunk()). */ +#define NATIVE_VMALLOC_END (NATIVE_VMALLOC_START + 0x100000000000UL) + /* 0x0000 f400 0000 0000 */ +#define NATIVE_VMEMMAP_START NATIVE_VMALLOC_END + /* 0x0000 f400 0000 0000 */ #define NATIVE_VMEMMAP_END (NATIVE_VMEMMAP_START + \ - (1ULL << (E2K_MAX_PHYS_BITS - \ - PAGE_SHIFT)) * \ - sizeof(struct page)) + (1ULL << (E2K_MAX_PHYS_BITS - PAGE_SHIFT)) * \ + sizeof(struct page)) + /* 0x0000 f800 0000 0000 - for 64 bytes struct page */ + /* 0x0000 fc00 0000 0000 - for 128 bytes struct page */ #ifdef CONFIG_SMP static inline void @@ -821,7 +825,7 @@ extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, - unsigned long address, pmd_t *pmdp) + unsigned long addr, pmd_t *pmdp) { # ifdef CONFIG_SMP u64 newval; @@ -829,7 +833,8 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, newval = (test_ts_flag(TS_KEEP_PAGES_VALID)) ? _PAGE_INIT_VALID : 0UL; - return __pmd(__api_xchg_return(newval, &pmdp->pmd, d, RELAXED_MB)); + return __pmd(pt_get_and_xchg_atomic(mm, addr, newval, + (pgprot_t *)pmdp)); # else pmd_t pmd = *pmdp; pmd_clear(pmdp); @@ -841,8 +846,8 @@ static inline pmd_t pmdp_huge_get_and_clear_as_valid(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { # ifdef CONFIG_SMP - return __pmd(__api_xchg_return(_PAGE_INIT_VALID, &pmdp->pmd, d, - RELAXED_MB)); + return __pmd(pt_get_and_xchg_atomic(mm, addr, _PAGE_INIT_VALID, + (pgprot_t *)pmdp)); # else pmd_t pmd = *pmdp; set_pmd_at(mm, addr, pmdp, __pmd(_PAGE_INIT_VALID)); diff --git a/arch/e2k/include/asm/process.h b/arch/e2k/include/asm/process.h index e1c4383..5a62028 100644 --- a/arch/e2k/include/asm/process.h +++ b/arch/e2k/include/asm/process.h @@ -12,8 +12,6 @@ #include #include #include -#include -#include #include #include @@ -794,6 +792,13 @@ clear_virt_thread_struct(thread_info_t *thread_info) { /* virtual machines is not supported */ } + +static __always_inline void +host_exit_to_usermode_loop(struct pt_regs *regs, bool syscall, bool has_signal) +{ + /* native & guest kernels cannot be as host */ +} + static __always_inline __interrupt void complete_switch_to_user_func(void) { @@ -821,6 +826,32 @@ static inline void free_virt_task_struct(struct task_struct *task) */ #endif /* ! CONFIG_VIRTUALIZATION */ +/* + * Restore proper psize field of WD register + */ +static inline void +native_restore_wd_register_psize(e2k_wd_t wd_from) +{ + e2k_wd_t wd; + + raw_all_irq_disable(); + wd = NATIVE_READ_WD_REG(); + wd.psize = wd_from.WD_psize; + NATIVE_WRITE_WD_REG(wd); + raw_all_irq_enable(); +} + +/* + * Preserve current p[c]shtp as they indicate how much to FILL when returning + */ +static inline void +native_preserve_user_hw_stacks_to_copy(e2k_stacks_t *u_stacks, + e2k_stacks_t *cur_stacks) +{ + u_stacks->pshtp = cur_stacks->pshtp; + u_stacks->pcshtp = cur_stacks->pcshtp; +} + static __always_inline void native_kernel_hw_stack_frames_copy(u64 *dst, const u64 *src, unsigned long size) { @@ -1049,6 +1080,19 @@ extern e2k_addr_t get_nested_kernel_IP(pt_regs_t *regs, int n); #define ONLY_SET_GUEST_GREGS(ti) NATIVE_ONLY_SET_GUEST_GREGS(ti) +static inline void +restore_wd_register_psize(e2k_wd_t wd_from) +{ + native_restore_wd_register_psize(wd_from); +} + +static inline void +preserve_user_hw_stacks_to_copy(e2k_stacks_t *u_stacks, + e2k_stacks_t *cur_stacks) +{ + native_preserve_user_hw_stacks_to_copy(u_stacks, cur_stacks); +} + static __always_inline void kernel_hw_stack_frames_copy(u64 *dst, const u64 *src, unsigned long size) { @@ -1277,9 +1321,9 @@ user_hw_stack_frames_copy(void __user *dst, void *src, unsigned long copy_size, } static __always_inline int -user_crs_frames_copy(e2k_mem_crs_t __user *u_frame, pt_regs_t *regs) +user_crs_frames_copy(e2k_mem_crs_t __user *u_frame, pt_regs_t *regs, + e2k_mem_crs_t *crs) { - e2k_mem_crs_t *crs = ®s->crs; unsigned long ts_flag; int ret; @@ -1397,7 +1441,7 @@ static inline void apply_graph_tracer_delta(unsigned long delta) * data from user space is spilled to kernel space. */ static __always_inline int -user_hw_stacks_copy(struct e2k_stacks *stacks, +native_user_hw_stacks_copy(struct e2k_stacks *stacks, pt_regs_t *regs, u64 cur_window_q, bool copy_full) { trap_pt_regs_t *trap = regs->trap; @@ -1533,65 +1577,6 @@ static inline void collapse_kernel_hw_stacks(struct e2k_stacks *stacks) raw_all_irq_restore(flags); } -/** - * user_hw_stacks_copy_full - copy part of user stacks that was SPILLed - * into kernel back to user stacks. - * @stacks - saved user stack registers - * @regs - pt_regs pointer - * @crs - last frame to copy - * - * If @crs is not NULL then the frame pointed to by it will also be copied - * to userspace. Note that 'stacks->pcsp_hi.ind' is _not_ updated after - * copying since it would leave stack in inconsistent state (with two - * copies of the same @crs frame), this is left to the caller. * - * - * Inlining this reduces the amount of memory to copy in - * collapse_kernel_hw_stacks(). - */ -static inline int user_hw_stacks_copy_full(struct e2k_stacks *stacks, - pt_regs_t *regs, e2k_mem_crs_t *crs) -{ - int ret; - - /* - * Copy part of user stacks that were SPILLed into kernel stacks - */ - ret = user_hw_stacks_copy(stacks, regs, 0, true); - if (unlikely(ret)) - return ret; - - /* - * Nothing to FILL so remove the resulting hole from kernel stacks. - * - * IMPORTANT: there is always at least one user frame at the top of - * kernel stack - the one that issued a system call (in case of an - * exception we uphold this rule manually, see user_hw_stacks_prepare()) - * We keep this ABI and _always_ leave space for one user frame, - * this way we can later FILL using return trick (otherwise there - * would be no space in chain stack for the trick). - */ - collapse_kernel_hw_stacks(stacks); - - /* - * Copy saved %cr registers - * - * Caller must take care of filling of resulting hole - * (last user frame from pcshtp == SZ_OF_CR). - */ - if (crs) { - e2k_mem_crs_t __user *u_frame; - int ret; - - u_frame = (void __user *) (AS(stacks->pcsp_lo).base + - AS(stacks->pcsp_hi).ind); - ret = user_crs_frames_copy(u_frame, regs); - if (unlikely(ret)) - return ret; - } - - return 0; -} - /** * user_hw_stacks_prepare - prepare user hardware stacks that have been * SPILLed to kernel back to user space @@ -1691,13 +1676,20 @@ static __always_inline void native_user_hw_stacks_prepare( /* * 2) Copy user data that cannot be FILLed */ - ret = user_hw_stacks_copy(stacks, regs, cur_window_q, false); + ret = native_user_hw_stacks_copy(stacks, regs, cur_window_q, false); if (unlikely(ret)) do_exit(SIGKILL); } #ifndef CONFIG_VIRTUALIZATION /* native kernel without virtualization support */ +static __always_inline int +user_hw_stacks_copy(struct e2k_stacks *stacks, + pt_regs_t *regs, u64 cur_window_q, bool copy_full) +{ + return native_user_hw_stacks_copy(stacks, regs, cur_window_q, copy_full); +} + static __always_inline void host_user_hw_stacks_prepare(struct e2k_stacks *stacks, pt_regs_t *regs, u64 cur_window_q, enum restore_caller from, int syscall) @@ -1718,6 +1710,64 @@ host_user_hw_stacks_prepare(struct e2k_stacks *stacks, pt_regs_t *regs, #error "unknown virtualization mode" #endif /* !CONFIG_VIRTUALIZATION */ +/** + * user_hw_stacks_copy_full - copy part of user stacks that was SPILLed + * into kernel back to user stacks. + * @stacks - saved user stack registers + * @regs - pt_regs pointer + * @crs - last frame to copy + * + * If @crs is not NULL then the frame pointed to by it will also be copied + * to userspace. Note that 'stacks->pcsp_hi.ind' is _not_ updated after + * copying since it would leave stack in inconsistent state (with two + * copies of the same @crs frame), this is left to the caller. * + * + * Inlining this reduces the amount of memory to copy in + * collapse_kernel_hw_stacks(). + */ +static inline int user_hw_stacks_copy_full(struct e2k_stacks *stacks, + pt_regs_t *regs, e2k_mem_crs_t *crs) +{ + int ret; + + /* + * Copy part of user stacks that were SPILLed into kernel stacks + */ + ret = user_hw_stacks_copy(stacks, regs, 0, true); + if (unlikely(ret)) + return ret; + + /* + * Nothing to FILL so remove the resulting hole from kernel stacks. + * + * IMPORTANT: there is always at least one user frame at the top of + * kernel stack - the one that issued a system call (in case of an + * exception we uphold this rule manually, see user_hw_stacks_prepare()) + * We keep this ABI and _always_ leave space for one user frame, + * this way we can later FILL using return trick (otherwise there + * would be no space in chain stack for the trick). + */ + collapse_kernel_hw_stacks(stacks); + + /* + * Copy saved %cr registers + * + * Caller must take care of filling of resulting hole + * (last user frame from pcshtp == SZ_OF_CR). + */ + if (crs) { + e2k_mem_crs_t __user *u_frame; + int ret; + + u_frame = (void __user *) (AS(stacks->pcsp_lo).base + + AS(stacks->pcsp_hi).ind); + ret = user_crs_frames_copy(u_frame, regs, ®s->crs); + if (unlikely(ret)) + return ret; + } + + return 0; +} extern e2k_addr_t get_nested_kernel_IP(pt_regs_t *regs, int n); extern unsigned long remap_e2k_stack(unsigned long addr, diff --git a/arch/e2k/include/asm/processor.h b/arch/e2k/include/asm/processor.h index d611eea..1b2ce4f 100644 --- a/arch/e2k/include/asm/processor.h +++ b/arch/e2k/include/asm/processor.h @@ -144,6 +144,7 @@ typedef struct thread_struct { /* protected mode */ #define BIN_COMP_CODE_TASK_FLAG_BIT 4 /* task is binary application */ /* compiler code */ +#define CLONE_SETTLS_TASK_FLAG_BIT 5 /* set new TLS for thread */ #define DO_PRESENT_HW_STACKS_TASK_FLAG_BIT 8 /* hardware stacks should be */ /* made present (populated) */ #define DO_LOCK_HW_STACKS_TASK_FLAG_BIT 9 /* hardware stacks should be */ @@ -163,6 +164,7 @@ typedef struct thread_struct { #define BIN_32_CODE_TASK_FLAG (1UL << BIN_32_CODE_TASK_FLAG_BIT) #define BIN_COMP_CODE_TASK_FLAG (1UL << BIN_COMP_CODE_TASK_FLAG_BIT) #define PROTECTED_CODE_TASK_FLAG (1UL << PROTECTED_CODE_TASK_FLAG_BIT) +#define CLONE_SETTLS_TASK_FLAG (1UL << CLONE_SETTLS_TASK_FLAG_BIT) #define DO_PRESENT_HW_STACKS_TASK_FLAG \ (1UL << DO_PRESENT_HW_STACKS_TASK_FLAG_BIT) #define DO_LOCK_HW_STACKS_TASK_FLAG \ diff --git a/arch/e2k/include/asm/protected_syscalls.h b/arch/e2k/include/asm/protected_syscalls.h index ab547be..8273693 100644 --- a/arch/e2k/include/asm/protected_syscalls.h +++ b/arch/e2k/include/asm/protected_syscalls.h @@ -7,9 +7,13 @@ /****************** PROTECTED SYSTEM CALL DEBUG DEFINES *******************/ +#ifndef _E2K_PROTECTED_SYSCALLS_H_ +#define _E2K_PROTECTED_SYSCALLS_H_ + #ifdef CONFIG_PROTECTED_MODE #include +#include #undef DYNAMIC_DEBUG_SYSCALLP_ENABLED #define DYNAMIC_DEBUG_SYSCALLP_ENABLED 1 /* Dynamic prot. syscalls control */ @@ -123,6 +127,112 @@ do { \ /**************************** END of DEBUG DEFINES ***********************/ + +static inline +long make_ap_lo(e2k_addr_t base, long size, long offset, int access) +{ + return MAKE_AP_LO(base, size, offset, access); +} + +static inline +long make_ap_hi(e2k_addr_t base, long size, long offset, int access) +{ + return MAKE_AP_HI(base, size, offset, access); +} + +static inline +int e2k_ptr_itag(long low) +{ + e2k_ptr_t ptr; + + AW(ptr).lo = low; + + return AS(ptr).itag; +} + +static inline +int e2k_ptr_rw(long low) +{ + e2k_ptr_t ptr; + + AW(ptr).lo = low; + + return AS(ptr).rw; +} + +static inline +unsigned long e2k_ptr_ptr(long low, long hiw, unsigned int min_size) +{ + e2k_ptr_t ptr; + unsigned int ptr_size; + + AW(ptr).lo = low; + AW(ptr).hi = hiw; + ptr_size = AS(ptr).size - AS(ptr).curptr; + + if (ptr_size < min_size) { + DbgSCP_ALERT(" Pointer is too small: %d < %d\n", + ptr_size, min_size); + return 0; + } else { + return E2K_PTR_PTR(ptr, GET_SBR_HI()); + } +} + +static inline +unsigned long e2k_ptr_curptr(long low, long hiw) +{ + e2k_ptr_t ptr; + + AW(ptr).lo = low; + AW(ptr).hi = hiw; + + return AS(ptr).curptr; +} + +static inline +unsigned long e2k_ptr_size(long low, long hiw, unsigned int min_size) +{ + e2k_ptr_hi_t hi; + unsigned int ptr_size; + + AW(hi) = hiw; + ptr_size = AS(hi).size - AS(hi).curptr; + + if (ptr_size < min_size) { + DbgSCP_ALERT(" Pointer is too small: %d < %d\n", + ptr_size, min_size); + return 0; + } else { + return ptr_size; + } +} + +static inline int e2k_ptr_str_check(char __user *str, u64 max_size) +{ + long slen; + + slen = strnlen_user(str, max_size); + + if (unlikely(!slen || slen > max_size)) + return 1; + + return 0; +} + +static inline char __user *e2k_ptr_str(long low, long hiw, u64 sbr_hi) +{ + char __user *str; + e2k_ptr_hi_t hi = { .word = hiw }; + + str = (char __user *) __E2K_PTR_PTR(low, hiw, sbr_hi); + + if (!e2k_ptr_str_check(str, AS(hi).size - AS(hi).curptr)) + return str; + + return NULL; +} + #else /* #ifndef CONFIG_PROTECTED_MODE */ #define DbgSCP(...) @@ -131,3 +241,7 @@ do { \ #define DbgSC_ALERT(...) #endif /* CONFIG_PROTECTED_MODE */ + + +#endif /* _E2K_PROTECTED_SYSCALLS_H_ */ + diff --git a/arch/e2k/include/asm/ptrace.h b/arch/e2k/include/asm/ptrace.h index 8e6b2a5..a182329 100644 --- a/arch/e2k/include/asm/ptrace.h +++ b/arch/e2k/include/asm/ptrace.h @@ -94,7 +94,6 @@ typedef struct trap_pt_regs { u64 TIR_lo; int TIR_no; /* current handled TIRs # */ s8 nr_TIRs; - bool irqs_disabled; /* IRQs are disabled while trap */ s8 tc_count; s8 curr_cnt; char ignore_user_tc; @@ -114,11 +113,29 @@ typedef struct trap_pt_regs { #endif } trap_pt_regs_t; -/* - * WARNING: 'usd_lo' field in the 'pt_regs' structure should have offset - * JB_USD_LO = 22 (in format of long long) as defined by e2k GLIBC header - * /usr/include/bits/setjmp.h - */ +union pt_regs_flags { + struct { + /* execute_mmu_operations() is working */ + u32 exec_mmu_op : 1; + /* nested exception appeared while + * execute_mmu_operations() was working */ + u32 exec_mmu_op_nested : 1; + /* A signal's handler will be called upon return to userspace */ + u32 sig_call_handler : 1; + /* System call should be restarted after signal's handler */ + u32 sig_restart_syscall : 1; + /* Used to distinguish between entry8 and entry10 for protected syscalls */ + u32 protected_entry10 : 1; + /* From hardware guest interception */ + u32 kvm_hw_intercept : 1; + /* trap or system call is on or from guest */ + u32 trap_as_intc_emul : 1; + /* Trap occurred in light hypercall */ + u32 light_hypercall : 1; + }; + u32 word; +}; + typedef struct pt_regs { struct pt_regs *next; /* the previous regs structure */ struct trap_pt_regs *trap; @@ -131,9 +148,7 @@ typedef struct pt_regs { e2k_wd_t wd; /* current window descriptor */ int sys_num; /* to restart sys_call */ int kernel_entry; - u32 flags; /* Trap occured on the instruction */ - /* with "Store recovery point" flag */ - bool irqs_disabled; /* IRQs are disabled while trap */ + union pt_regs_flags flags; e2k_ctpr_t ctpr1; /* CTPRj for control transfer */ e2k_ctpr_t ctpr2; e2k_ctpr_t ctpr3; @@ -168,12 +183,15 @@ typedef struct pt_regs { u64 rpr_lo; u64 rpr_hi; #ifdef CONFIG_VIRTUALIZATION + u64 sys_func; /* need only for guest */ e2k_stacks_t g_stacks; /* current state of guest kernel */ /* stacks registers */ bool g_stacks_valid; /* the state of guest kernel stacks */ /* registers is valid */ bool g_stacks_active; /* the guest kernel stacks */ /* registers is in active work */ + bool stack_regs_saved; /* stack state regs was already */ + /* saved */ bool need_inject; /* flag for unconditional injection */ /* trap to guest to avoid acces to */ /* guest user space in trap context */ @@ -183,11 +201,19 @@ typedef struct pt_regs { unsigned long traps_to_guest; /* mask of traps passed to guest */ /* and are not yet handled by guest */ /* need only for host */ - unsigned long deferred_traps; /* mask of deffered traps, which */ - /* cannot be handled immediately */ - /* (for guest) or occured while */ - /* guest is handling previous traps */ - /* (for host, for example interrupts) */ +#ifdef CONFIG_KVM_GUEST_KERNEL +/* only for guest kernel */ + /* already copyed back part of guest user hardware stacks */ + /* spilled to guest kernel stacks */ + struct { + e2k_size_t ps_size; /* procedure stack copyed size */ + e2k_size_t pcs_size; /* chain stack copyesd size */ + /* The frames injected to support 'signal stack' */ + /* and trampolines to return from user to kernel */ + e2k_size_t pcs_injected_frames_size; + } copyed; +#endif /* CONFIG_KVM_GUEST_KERNEL */ + #endif /* CONFIG_VIRTUALIZATION */ #if defined(CONFIG_KVM) || defined(CONFIG_KVM_GUEST_KERNEL) @@ -199,23 +225,6 @@ typedef struct pt_regs { #endif /* CONFIG_KERNEL_TIMES_ACCOUNT */ } pt_regs_t; -#define E_MMU_OP_FLAG_PT_REGS 0x2U /* execute_mmu_operations is */ - /* working */ -#define E_MMU_NESTED_OP_FLAG_PT_REGS 0x4U /* nested exception appeared */ - /* while */ - /* execute_mmu_operations is */ - /* working */ -#define SIG_CALL_HANDLER_FLAG_PT_REGS 0x8U -#define SIG_RESTART_SYSCALL_FLAG_PT_REGS 0x10U -#define PROT_10_FLAG_PT_REGS 0x20U /* marked 10 pm sys_call */ -#define TRAP_AS_INTC_EMUL_PT_REGS 0x0100 /* trap or system call */ - /* is on or from guest */ -#define GUEST_FLAG_PT_REGS 0x10000U /* Trap occurred on the */ - /* guest and should be */ - /* handled by guest */ -#define LIGHT_HYPERCALL_FLAG_PT_REGS 0x20000U /* Trap occurred in */ - /* hypercall */ - static inline struct trap_pt_regs * pt_regs_to_trap_regs(struct pt_regs *regs) { @@ -580,8 +589,9 @@ struct signal_stack_context { do { \ struct pt_regs *__pt_regs = current_thread_info()->pt_regs; \ if (__pt_regs) { \ - user_hw_stacks_copy_full(&__pt_regs->stacks, \ - __pt_regs, NULL); \ + if (!test_ts_flag(TS_USER_EXECVE)) \ + user_hw_stacks_copy_full(&__pt_regs->stacks, \ + __pt_regs, NULL); \ SAVE_AAU_REGS_FOR_PTRACE(__pt_regs, current_thread_info()); \ if (!paravirt_enabled()) { \ /* FIXME: it need implement for guest kernel */ \ @@ -709,7 +719,7 @@ static inline struct pt_regs *find_user_regs(const struct pt_regs *regs) do { CHECK_PT_REGS_LOOP(regs); - if (user_mode(regs)) + if (user_mode(regs) && !regs->flags.kvm_hw_intercept) break; regs = regs->next; @@ -731,7 +741,7 @@ static inline struct pt_regs *find_entry_regs(const struct pt_regs *regs) do { CHECK_PT_REGS_LOOP(regs); - if (user_mode(regs)) + if (user_mode(regs) && !regs->flags.kvm_hw_intercept) goto found; prev_regs = regs; @@ -745,12 +755,26 @@ found: return (struct pt_regs *) regs; } -static inline struct pt_regs *find_trap_regs(const struct pt_regs *regs) +static inline struct pt_regs *find_host_regs(const struct pt_regs *regs) { while (regs) { CHECK_PT_REGS_LOOP(regs); - if (from_trap(regs)) + if (likely(!regs->flags.kvm_hw_intercept)) + break; + + regs = regs->next; + }; + + return (struct pt_regs *) regs; +} + +static inline struct pt_regs *find_trap_host_regs(const struct pt_regs *regs) +{ + while (regs) { + CHECK_PT_REGS_LOOP(regs); + + if (from_trap(regs) && !regs->flags.kvm_hw_intercept) break; regs = regs->next; diff --git a/arch/e2k/include/asm/regs_state.h b/arch/e2k/include/asm/regs_state.h index a23e14e..70e8bb8 100644 --- a/arch/e2k/include/asm/regs_state.h +++ b/arch/e2k/include/asm/regs_state.h @@ -182,19 +182,6 @@ do { \ (thread_info)->u_stack.size = stk_sz; \ }) -/* - * Remeber state of IRQs at trap point - * Now it usefull to analyze can be traps passed to guest handler - * immediately or should be deferred - */ -#define SAVE_IRQS_STATE(regs, upsr) \ -({ \ - unsigned long psr_val = (regs)->crs.cr1_lo.CR1_lo_psr; \ - unsigned long upsr_val = (upsr).UPSR_reg; \ - (regs)->irqs_disabled = \ - psr_and_upsr_irqs_disabled_flags(psr_val, upsr_val); \ -}) - /* * Interrupts should be disabled by caller to read all hardware * stacks registers in coordinated state @@ -579,24 +566,24 @@ do { \ #endif /* E2K_MAXGR_d */ static inline void -native_save_local_glob_regs(local_gregs_t *l_gregs) +native_save_local_glob_regs(local_gregs_t *l_gregs, bool is_signal) { - void (*save_local_gregs)(struct local_gregs *); + void (*save_local_gregs)(struct local_gregs *, bool is_signal); save_local_gregs = machine.save_local_gregs; copy_k_gregs_to_l_gregs(l_gregs, ¤t_thread_info()->k_gregs); - save_local_gregs(l_gregs); + save_local_gregs(l_gregs, is_signal); } static inline void -native_restore_local_glob_regs(local_gregs_t *l_gregs) +native_restore_local_glob_regs(local_gregs_t *l_gregs, bool is_signal) { - void (*restore_local_gregs)(const struct local_gregs *); + void (*restore_local_gregs)(const struct local_gregs *, bool is_signal); restore_local_gregs = machine.restore_local_gregs; get_k_gregs_from_l_regs(¤t_thread_info()->k_gregs, l_gregs); - restore_local_gregs(l_gregs); + restore_local_gregs(l_gregs, is_signal); } static inline void @@ -767,7 +754,7 @@ do { \ (trap)->tc_count = cnt * 3; \ if (unlikely(GET_CLW_REQUEST_COUNT(regs) && \ cpu_has(CPU_HWBUG_CLW_STALE_L1_ENTRY))) \ - (regs)->clw_cpu = raw_smp_processor_id(); \ + SET_CLW_CPU(regs, raw_smp_processor_id()); \ if (cs_req_num > 0) { \ /* recover chain stack pointers to repeat FILL */ \ e2k_pcshtp_t pcshtp = NATIVE_READ_PCSHTP_REG_SVALUE(); \ @@ -798,6 +785,7 @@ do { \ # define GET_CLW_REQUEST_COUNT(regs) ((regs)->clw_count) # define SET_CLW_FIRST_REQUEST(regs, cnt) ((regs)->clw_first = (cnt)) # define GET_CLW_FIRST_REQUEST(regs) ((regs)->clw_first) +# define SET_CLW_CPU(regs, cpu) ((regs)->clw_cpu = (cpu)) #define ENABLE_US_CLW() \ do { \ if (!cpu_has(CPU_HWBUG_CLW)) \ @@ -810,6 +798,7 @@ do { \ # define GET_CLW_REQUEST_COUNT(regs) (0) # define SET_CLW_FIRST_REQUEST(regs, cnt) # define GET_CLW_FIRST_REQUEST(regs) (0) +# define SET_CLW_CPU(regs, cpu) # define ENABLE_US_CLW() # define DISABLE_US_CLW() #endif /* CONFIG_CLW_ENABLE */ @@ -862,10 +851,6 @@ do { \ PREFIX_RESTORE_USER_CRs(NATIVE, regs) #define NATIVE_RESTORE_USER_STACK_REGS(regs, insyscall) \ PREFIX_RESTORE_USER_STACK_REGS(NATIVE, regs, insyscall) -#define NATIVE_RESTORE_USER_TRAP_STACK_REGS(regs) \ - NATIVE_RESTORE_USER_STACK_REGS(regs, false) -#define NATIVE_RESTORE_USER_SYSCALL_STACK_REGS(regs) \ - NATIVE_RESTORE_USER_STACK_REGS(regs, true) #if defined(CONFIG_PARAVIRT_GUEST) /* it is paravirtualized host and guest */ @@ -887,19 +872,21 @@ do { \ RESTORE_USER_STACK_REGS(regs, false) #define RESTORE_USER_SYSCALL_STACK_REGS(regs) \ RESTORE_USER_STACK_REGS(regs, true) +#define RESTORE_COMMON_REGS(regs) \ + NATIVE_RESTORE_COMMON_REGS(regs) #define INIT_G_REGS() NATIVE_INIT_G_REGS() #define BOOT_INIT_G_REGS() NATIVE_BOOT_INIT_G_REGS() static inline void -save_local_glob_regs(local_gregs_t *l_gregs) +save_local_glob_regs(local_gregs_t *l_gregs, bool is_signal) { - native_save_local_glob_regs(l_gregs); + native_save_local_glob_regs(l_gregs, is_signal); } static inline void -restore_local_glob_regs(local_gregs_t *l_gregs) +restore_local_glob_regs(local_gregs_t *l_gregs, bool is_signal) { - native_restore_local_glob_regs(l_gregs); + native_restore_local_glob_regs(l_gregs, is_signal); } static inline void @@ -1080,7 +1067,11 @@ NATIVE_DO_SAVE_TASK_USER_REGS_TO_SWITCH(struct sw_regs *sw_regs, static inline void NATIVE_SAVE_TASK_REGS_TO_SWITCH(struct task_struct *task) { +#ifdef CONFIG_VIRTUALIZATION + const int task_is_binco = TASK_IS_BINCO(task) || task_thread_info(task)->vcpu; +#else const int task_is_binco = TASK_IS_BINCO(task); +#endif struct mm_struct *mm = task->mm; struct sw_regs *sw_regs = &task->thread.sw_regs; @@ -1166,7 +1157,11 @@ NATIVE_RESTORE_TASK_REGS_TO_SWITCH(struct task_struct *task, u64 pcsp_lo = AS_WORD(sw_regs->pcsp_lo); u64 pcsp_hi = AS_WORD(sw_regs->pcsp_hi); e2k_mem_crs_t crs = sw_regs->crs; +#ifdef CONFIG_VIRTUALIZATION + const int task_is_binco = TASK_IS_BINCO(task) || ti->vcpu; +#else const int task_is_binco = TASK_IS_BINCO(task); +#endif struct mm_struct *mm = task->mm; NATIVE_FLUSHCPU; diff --git a/arch/e2k/include/asm/secondary_space.h b/arch/e2k/include/asm/secondary_space.h index eaaad3b..2877cb6 100644 --- a/arch/e2k/include/asm/secondary_space.h +++ b/arch/e2k/include/asm/secondary_space.h @@ -79,6 +79,8 @@ s64 sys_el_binary(s64 work, s64 arg2, s64 arg3, s64 arg4); #define SET_SECONDARY_64BIT_MODE 11 #define GET_PROTOCOL_VERSION 12 #define SET_IC_NEED_FLUSH_ON_SWITCH 13 +#define GET_UPT_SEC_AD_SHIFT_DSBL 14 +#define SET_UPT_SEC_AD_SHIFT_DSBL 15 /* Selector numbers for GET_SECONDARY_SPACE_OFFSET */ enum sel_num { diff --git a/arch/e2k/include/asm/sic_regs.h b/arch/e2k/include/asm/sic_regs.h index cd6fd3d..a602d2a 100644 --- a/arch/e2k/include/asm/sic_regs.h +++ b/arch/e2k/include/asm/sic_regs.h @@ -189,22 +189,27 @@ /* MC */ -#define SIC_MAX_MC_COUNT 4 +#define SIC_MAX_MC_COUNT E16C_SIC_MC_COUNT #define SIC_MC_COUNT (machine.sic_mc_count) +#define SIC_MC_BASE 0x400 +#define SIC_MC_SIZE (machine.sic_mc_size) + +#define SIC_mc_ecc 0x440 #define SIC_mc0_ecc 0x400 #define SIC_mc1_ecc (machine.sic_mc1_ecc) #define SIC_mc2_ecc 0x480 #define SIC_mc3_ecc 0x4c0 -#define SIC_MC_BASE SIC_mc0_ecc -#define SIC_MC_SIZE (IS_MACHINE_E2S ? 0xa4 : \ - (IS_MACHINE_E8C ? 0xe4 : 0xf4)) +#define SIC_mc_ch 0x400 +#define SIC_mc_status 0x44c -/* PHY */ -#define SIC_PHY_BASE (IS_MACHINE_E8C2 ? 0x4000 : 0x1000) -#define SIC_PHY_SIZE (IS_MACHINE_E2S ? 0x0c00 : \ - (IS_MACHINE_E8C ? 0x1000 : 0x4000)) +/* HMU */ +#define SIC_hmu_mic 0xd00 +#define SIC_hmu0_int 0xd40 +#define SIC_hmu1_int 0xd70 +#define SIC_hmu2_int 0xda0 +#define SIC_hmu3_int 0xdd0 /* IPCC */ #define SIC_IPCC_LINKS_COUNT 3 @@ -260,6 +265,28 @@ typedef union { u32 word; } freq_core_mon_t; +/* PMC_FREQ_CORE_0_CTRL fields: */ +typedef union { + struct { + u32 enable : 1; + u32 mode : 3; + u32 progr_divF : 6; + u32 progr_divF_max : 6; + u32 decr_dsbl : 1; + u32 pin_en : 1; + u32 clk_en : 1; + u32 log_en : 1; + u32 sleep_c2 : 1; + u32 w_trap : 1; + u32 ev_term : 1; + u32 mon_Fmax : 1; + u32 divF_curr : 6; + u32 bfs_bypass : 1; + u32 rmwen : 1; + }; + u32 word; +} freq_core_ctrl_t; + /* PMC_SYS_MON_1 fields: */ typedef union { struct { @@ -427,6 +454,9 @@ typedef union { #define BC_MM_REG_SIZE (BC_MM_REG_END - BC_MM_REG_BASE) #define BC_MM_REG_NUM (BC_MM_REG_SIZE / 4) +#define EFUSE_RAM_ADDR 0x0cc0 +#define EFUSE_RAM_DATA 0x0cc4 + #ifndef __ASSEMBLY__ /* * Read/Write RT_LCFGj Regs diff --git a/arch/e2k/include/asm/signal.h b/arch/e2k/include/asm/signal.h index af8a43f..c45cbca 100644 --- a/arch/e2k/include/asm/signal.h +++ b/arch/e2k/include/asm/signal.h @@ -128,7 +128,7 @@ struct signal_stack; extern void free_signal_stack(struct signal_stack *signal_stack); extern struct signal_stack_context __user *pop_signal_stack(void); extern struct signal_stack_context __user *get_signal_stack(void); -extern int setup_signal_stack(struct pt_regs *regs); +extern int setup_signal_stack(struct pt_regs *regs, bool is_signal); #define GET_SIG_RESTORE_STACK(ti, sbr, usd_lo, usd_hi) \ do { \ diff --git a/arch/e2k/include/asm/smp.h b/arch/e2k/include/asm/smp.h index 104b905..5985733 100644 --- a/arch/e2k/include/asm/smp.h +++ b/arch/e2k/include/asm/smp.h @@ -59,16 +59,10 @@ extern unsigned long smp_invalidate_needed; extern int pic_mode; extern cpumask_t callin_go; -extern void smp_alloc_memory(void); extern void e2k_start_secondary(int cpuid); extern void start_secondary_resume(int cpuid, int cpu); extern void wait_for_startup(int cpuid, int hotplug); -extern void smp_flush_tlb(void); -extern void smp_message_irq(int cpl, void *dev_id, struct pt_regs *regs); extern void smp_send_reschedule(int cpu); -extern void smp_invalidate_rcv(void); /* Process an NMI */ -extern void (*mtrr_hook) (void); -extern void zap_low_mappings (void); extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); extern void smp_send_refresh(void); diff --git a/arch/e2k/include/asm/sparsemem.h b/arch/e2k/include/asm/sparsemem.h index 83e05dc..63144ee 100644 --- a/arch/e2k/include/asm/sparsemem.h +++ b/arch/e2k/include/asm/sparsemem.h @@ -4,7 +4,7 @@ #ifdef CONFIG_SPARSEMEM # define SECTION_SIZE_BITS 28 -# define MAX_PHYSMEM_BITS 40 +# define MAX_PHYSMEM_BITS CONFIG_E2K_PA_BITS #endif /* CONFIG_SPARSEMEM */ #endif /* _ASM_E2K_SPARSEMEM_H */ diff --git a/arch/e2k/include/asm/string.h b/arch/e2k/include/asm/string.h index fdb12c2..df5262c 100644 --- a/arch/e2k/include/asm/string.h +++ b/arch/e2k/include/asm/string.h @@ -432,7 +432,13 @@ static inline void native_tagged_memcpy_8(void *__restrict dst, * * All parameters must be 8-bytes aligned. */ -#if defined(CONFIG_PARAVIRT_GUEST) +#ifdef CONFIG_BOOT_E2K +#define tagged_memcpy_8(dst, src, n) \ +({ \ + native_tagged_memcpy_8(dst, src, n, \ + __alignof(*(dst)), __alignof(*(src))); \ +}) +#elif defined(CONFIG_PARAVIRT_GUEST) #include #elif defined(CONFIG_KVM_GUEST_KERNEL) #include diff --git a/arch/e2k/include/asm/sync_pg_tables.h b/arch/e2k/include/asm/sync_pg_tables.h new file mode 100644 index 0000000..08dfc51 --- /dev/null +++ b/arch/e2k/include/asm/sync_pg_tables.h @@ -0,0 +1,18 @@ +/* Functions to sync shadow page tables with guest page tables + * without flushing tlb. Used only by guest kernels + * + * Copyright 2021 Andrey Alekhin (alekhin_amcst.ru) + */ +#ifndef _E2K_SYNC_PG_TABLES_H +#define _E2K_SYNC_PG_TABLES_H + +#if defined(CONFIG_KVM_GUEST_KERNEL) + +#include + +#define sync_addr_range kvm_sync_addr_range +#else +#define sync_addr_range +#endif /* !CONFIG_KVM_GUEST_KERNEL */ + +#endif diff --git a/arch/e2k/include/asm/syscalls.h b/arch/e2k/include/asm/syscalls.h index fc75f09..a473a87 100644 --- a/arch/e2k/include/asm/syscalls.h +++ b/arch/e2k/include/asm/syscalls.h @@ -39,10 +39,6 @@ extern long sys_stat64(const char __user *filename, extern long sys_fstat64(unsigned long fd, struct stat64 __user *statbuf); extern long sys_lstat64(const char __user *filename, struct stat64 __user *statbuf); -#ifdef CONFIG_MAC_ -extern int sys_macctl(register int request, register void *data, - register int size); -#endif extern asmlinkage long sys_set_backtrace(unsigned long *__user buf, size_t count, size_t skip, unsigned long flags); @@ -83,8 +79,13 @@ extern long protected_sys_rt_sigaction_ex(int sig, const size_t sigsetsize); extern long protected_sys_mq_notify(const long a1, const unsigned long __user a2); -extern long protected_sys_timer_create(const long a1, - const unsigned long __user a2, const unsigned long __user a3); +extern long protected_sys_timer_create(const long a1, /* clockid */ + const unsigned long __user a2, /* sevp */ + const unsigned long __user a3, /* timerid */ + const unsigned long unused4, + const unsigned long unused5, + const unsigned long unused6, + const struct pt_regs *regs); extern long protected_sys_rt_sigtimedwait(const unsigned long __user a1, const unsigned long __user a2, const unsigned long __user a3, const unsigned long a4); @@ -229,6 +230,13 @@ extern long protected_sys_sendmsg(const unsigned long sockfd, const unsigned long unused5, const unsigned long unused6, const struct pt_regs *regs); +extern long protected_sys_sendmmsg(const unsigned long sockfd, + const unsigned long __user msgvec, + const unsigned long vlen, + const unsigned long flags, + const unsigned long unused5, + const unsigned long unused6, + const struct pt_regs *regs); extern long protected_sys_recvmsg(const unsigned long socket, const unsigned long __user message, const unsigned long flags, @@ -236,6 +244,13 @@ extern long protected_sys_recvmsg(const unsigned long socket, const unsigned long unused5, const unsigned long unused6, const struct pt_regs *regs); +extern long protected_sys_recvmmsg(const unsigned long socket, + const unsigned long __user message, + const unsigned long vlen, + const unsigned long flags, + const unsigned long __user timeout, + const unsigned long unused6, + const struct pt_regs *regs); extern long protected_sys_olduselib(const unsigned long __user a1, /* library */ const unsigned long __user a2); /* umdd */ /* NB> 'olduselib' is obsolete syscall; unsupported in CPU ISET V6 */ @@ -313,13 +328,13 @@ extern long protected_sys_prctl(const int option, /* a1 */ const unsigned long arg5, const unsigned long unused6, const struct pt_regs *regs); -extern long protected_sys_ioctl(const int fd, /* a1 */ - const unsigned long request,/* a2 */ - void *argp, /* a3 */ - const unsigned long unused4, - const unsigned long unused5, - const unsigned long unused6, - const struct pt_regs *regs); +extern long protected_sys_ioctl(const int fd, /* a1 */ + const unsigned long request, /* a2 */ + const unsigned long __user argp, /* a3 */ + const unsigned long unused4, + const unsigned long unused5, + const unsigned long unused6, + const struct pt_regs *regs); extern long protected_sys_epoll_ctl(const unsigned long epfd, /* a1 */ const unsigned long op, /* a2 */ const unsigned long fd, /* a3 */ @@ -348,6 +363,48 @@ extern long protected_sys_pselect6(const long nfds, /* a1 */ const unsigned long timeout, /* a5 */ const unsigned long sigmask, /* a6 */ const struct pt_regs *regs); +extern long protected_sys_rt_sigqueueinfo(const long tgid, /* a1 */ + const long sig, /* a2 */ + const unsigned long __user uinfo, /* a3 */ + const unsigned long unused4, + const unsigned long unused5, + const unsigned long unused6, + const struct pt_regs *regs); +extern long protected_sys_rt_tgsigqueueinfo(const long tgid, /* a1 */ + const long tid, /* a2 */ + const long sig, /* a3 */ + const unsigned long __user uinfo, /* a4 */ + const unsigned long unused5, + const unsigned long unused6, + const struct pt_regs *regs); +extern long protected_sys_pidfd_send_signal(const long pidfd, /* a1 */ + const long sig, /* a2 */ + const unsigned long __user info, /* a3 */ + unsigned long flags, /* a4 */ + const unsigned long unused5, + const unsigned long unused6, + const struct pt_regs *regs); +extern long protected_sys_waitid(const long which, /* a1 */ + const long pid, /* a2 */ + const unsigned long __user *infop, /* a3 */ + const long options, /* a4 */ + const unsigned long __user *ru, /* a5 */ + const unsigned long unused6, + const struct pt_regs *regs); +extern long protected_sys_io_uring_register(const unsigned long fd, /* a1 */ + const unsigned long opcode, /* a2 */ + const unsigned long __user arg, /* a3 */ + const unsigned long nr_args, /* a4 */ + const unsigned long unused5, + const unsigned long unused6, + const struct pt_regs *regs); +extern long protected_sys_kexec_load(const unsigned long entry, /* a1 */ + const unsigned long nr_segments, /* a2 */ + const unsigned long __user segments, /* a3 */ + const unsigned long flags, /* a4 */ + const unsigned long unused5, + const unsigned long unused6, + const struct pt_regs *regs); extern int arch_init_pm_sc_debug_mode(const int debug_mask); diff --git a/arch/e2k/include/asm/system.h b/arch/e2k/include/asm/system.h index d5e0b3a..3fc61f5 100644 --- a/arch/e2k/include/asm/system.h +++ b/arch/e2k/include/asm/system.h @@ -635,7 +635,7 @@ extern void * __e2k_read_kernel_return_address(int n); : \ __e2k_read_kernel_return_address(n); }) -#if CONFIG_CPU_ISET < 5 +#ifndef CONFIG_CPU_HW_CLEAR_RF typedef void (*clear_rf_t)(void); extern const clear_rf_t clear_rf_fn[]; @@ -643,7 +643,7 @@ static __always_inline void clear_rf_kernel_except_current(u64 num_q) { clear_rf_fn[num_q](); } -#endif /* CONFIG_CPU_ISET < 5 */ +#endif #define SWITCH_TO_KERNEL_UPSR(upsr_reg, irq_en, nmirq_dis) \ NATIVE_SWITCH_TO_KERNEL_UPSR(upsr_reg, irq_en, nmirq_dis) diff --git a/arch/e2k/include/asm/thread_info.h b/arch/e2k/include/asm/thread_info.h index 3a734fc..5681b19 100644 --- a/arch/e2k/include/asm/thread_info.h +++ b/arch/e2k/include/asm/thread_info.h @@ -9,8 +9,9 @@ #ifdef __KERNEL__ #ifndef __ASSEMBLY__ -#include #include +#include +#include #include #include @@ -232,13 +233,8 @@ typedef struct thread_info { #define TIF_PSEUDOTHREAD 24 /* the thread is pseudo only to run */ /* on VIRQ VCPU as starter of VIRQ */ /* handler */ -#define TIF_HOST_AT_VCPU_MODE 25 /* the host thread is switching to */ - /* VCPU running mode and wait for */ - /* interception (trap on PV mode) */ #define TIF_VIRQS_ACTIVE 26 /* the thread is ready to inject */ /* VIRQS interrupt */ -#define TIF_VIRQ_HANDLER 27 /* the thread is VIRQ handler and */ - /* should run with max priorety */ #define TIF_LIGHT_HYPERCALL 28 /* hypervisor is executing light */ /* hypercall */ #define TIF_GENERIC_HYPERCALL 29 /* hypervisor is executing generic */ @@ -266,9 +262,7 @@ typedef struct thread_info { #define _TIF_VIRTUALIZED_GUEST (1 << TIF_VIRTUALIZED_GUEST) #define _TIF_PARAVIRT_GUEST (1 << TIF_PARAVIRT_GUEST) #define _TIF_PSEUDOTHREAD (1 << TIF_PSEUDOTHREAD) -#define _TIF_HOST_AT_VCPU_MODE (1 << TIF_HOST_AT_VCPU_MODE) #define _TIF_VIRQS_ACTIVE (1 << TIF_VIRQS_ACTIVE) -#define _TIF_VIRQ_HANDLER (1 << TIF_VIRQ_HANDLER) #define _TIF_LIGHT_HYPERCALL (1 << TIF_LIGHT_HYPERCALL) #define _TIF_GENERIC_HYPERCALL (1 << TIF_GENERIC_HYPERCALL) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) @@ -298,18 +292,18 @@ typedef struct thread_info { * have to worry about atomic accesses. */ #define TS_DELAYED_SIG_HANDLING 0x00000001 -#define TS_KEEP_PAGES_VALID 0x00000004 -#define TS_MMAP_PRIVILEGED 0x00000010 -#define TS_MMAP_DONTEXPAND 0x00000020 -#define TS_MMAP_DONTCOPY 0x00000040 -#define TS_KERNEL_SYSCALL 0x00000100 -#define TS_USER_EXECVE 0x00001000 -#define TS_MMAP_PS 0x00010000 -#define TS_MMAP_PCS 0x00020000 -#define TS_MMAP_NOHUGEPAGE 0x00040000 -#define TS_MMAP_SIGNAL_STACK 0x00080000 -#define TS_SINGLESTEP_KERNEL 0x00100000 -#define TS_SINGLESTEP_USER 0x00200000 +#define TS_KEEP_PAGES_VALID 0x00000002 +#define TS_MMAP_PRIVILEGED 0x00000004 +#define TS_MMAP_PS 0x00000008 +#define TS_MMAP_PCS 0x00000010 +#define TS_MMAP_SIGNAL_STACK 0x00000020 +#define TS_KERNEL_SYSCALL 0x00000040 +#define TS_USER_EXECVE 0x00000080 +#define TS_SINGLESTEP_KERNEL 0x00000100 +#define TS_SINGLESTEP_USER 0x00000200 +/* the host thread is switching to VCPU running mode + * and wait for interception (trap on PV mode) */ +#define TS_HOST_AT_VCPU_MODE 0x00001000 #define THREAD_SIZE KERNEL_STACKS_SIZE @@ -343,12 +337,22 @@ static inline unsigned long test_ti_status_flag(struct thread_info *ti, return ti->status & flag; } +static inline unsigned long test_and_clear_ti_status_flag( + struct thread_info *ti, int flag) +{ + typeof(ti->status) status = ti->status; + ti->status = status & ~flag; + return status & flag; +} + #define set_ts_flag(flag) \ set_ti_status_flag(current_thread_info(), flag) #define clear_ts_flag(flag) \ clear_ti_status_flag(current_thread_info(), flag) #define test_ts_flag(flag) \ test_ti_status_flag(current_thread_info(), flag) +#define test_and_clear_ts_flag(flag) \ + test_and_clear_ti_status_flag(current_thread_info(), flag) #define native_current_thread_info() current_thread_info() #define boot_current_thread_info() BOOT_READ_CURRENT_REG() diff --git a/arch/e2k/include/asm/trap_table.S.h b/arch/e2k/include/asm/trap_table.S.h index 2281f7a..ea59739 100644 --- a/arch/e2k/include/asm/trap_table.S.h +++ b/arch/e2k/include/asm/trap_table.S.h @@ -29,10 +29,6 @@ * This is needed to flush SLT before trying to load anything. */ #define SWITCH_HW_STACKS_FROM_USER(...) \ - { \ - /* Disable load/store generations */ \ - crp; \ - } \ { \ /* Wait for FPU exceptions before switching stacks */ \ wait all_e = 1; \ @@ -43,6 +39,18 @@ rrd %psp.hi, GCURTASK; \ stgdq,sm %qg18, 0, TSK_TI_G_MY_CPU_OFFSET; \ cmpesb,1 0, 0, %pred0; \ + /* Do restore %rpr (it's clobbered by "crp" below) */ \ + cmpesb,3 0, 0, %pred1; \ + } \ + { \ + /* 'crp' instruction also clears %rpr besides the generations \ + * table, so make sure we preserve %rpr value. */ \ + rrd %rpr.lo, GCPUOFFSET; \ + } \ + { \ + rrd %rpr.hi, GCPUID; \ + /* Disable load/store generations */ \ + crp; \ } \ SWITCH_HW_STACKS(TSK_TI_, ##__VA_ARGS__) @@ -50,6 +58,9 @@ * This assumes that GVCPUSTATE points to current_thread_info() * and %psp.hi has been read into GCURTASK * + * %pred0 - set to "true" if PSP/PCSP should be switched. + * %pred1 - set to "true" if RPR should be restored. + * * Does the following: * * 1) Saves global registers either to 'thread_info.tmp_k_gregs' or to @@ -65,14 +76,18 @@ */ #define SWITCH_HW_STACKS(PREFIX, ...) \ { \ - ldd,0 GVCPUSTATE, TI_K_PSP_LO, GCPUOFFSET; \ - ldd,2 GVCPUSTATE, TI_K_PCSP_LO, GCPUID; \ + rwd GCPUOFFSET, %rpr.lo ? %pred1; \ + ldd,2 GVCPUSTATE, TI_K_PSP_LO, GCPUOFFSET; \ __VA_ARGS__ \ } \ + { \ + rwd GCPUID, %rpr.hi ? %pred1; \ + ldd,2 GVCPUSTATE, TI_K_PCSP_LO, GCPUID; \ + } \ { \ rrd %psp.lo, GCURTASK ? %pred0; \ stgdd,2 GCURTASK, 0, TSK_TI_TMP_U_PSP_HI ? %pred0; \ - SMP_ONLY(ldw,5 GVCPUSTATE, TSK_TI_CPU_DELTA, GCPUID ? ~ %pred0;) \ + SMP_ONLY(ldgdw,5 0, TSK_TI_CPU_DELTA, GCPUID ? ~ %pred0;) \ } \ { \ rrd %pcsp.hi, GCURTASK ? %pred0; \ @@ -82,7 +97,6 @@ ibranch trap_handler_switched_stacks ? ~ %pred0; \ } \ { \ - nop 1; /* ldd -> use */ \ rrd %pcsp.lo, GCURTASK; \ stgdd,2 GCURTASK, 0, TSK_TI_TMP_U_PCSP_HI; \ } \ diff --git a/arch/e2k/include/asm/trap_table.h b/arch/e2k/include/asm/trap_table.h index 78cdea0..707460c 100644 --- a/arch/e2k/include/asm/trap_table.h +++ b/arch/e2k/include/asm/trap_table.h @@ -35,12 +35,29 @@ static inline bool is_gdb_breakpoint_trap(struct pt_regs *regs) { u64 *instr = (u64 *)GET_IP_CR0_HI(regs->crs.cr0_hi); + u64 sylab; - return (*instr & GDB_BREAKPOINT_STUB_MASK) == GDB_BREAKPOINT_STUB; + host_get_user(sylab, instr, regs); + return (sylab & GDB_BREAKPOINT_STUB_MASK) == GDB_BREAKPOINT_STUB; } extern void kernel_stack_overflow(unsigned int overflows); +static inline void native_clear_fork_child_pt_regs(struct pt_regs *childregs) +{ + childregs->sys_rval = 0; + /* + * Remove all pointers to parent's data stack + * (these are not needed anyway for system calls) + */ + childregs->trap = NULL; + childregs->aau_context = NULL; +#ifdef CONFIG_KERNEL_TIMES_ACCOUNT + childregs->scall_times = NULL; +#endif + childregs->next = NULL; +} + static inline unsigned int native_is_kernel_data_stack_bounds(bool trap_on_kernel, e2k_usd_lo_t usd_lo) { @@ -63,23 +80,14 @@ native_correct_trap_return_ip(struct pt_regs *regs, unsigned long return_ip) } regs->crs.cr0_hi.CR0_hi_IP = return_ip; } -static inline void -native_handle_deferred_traps_in_syscall(struct pt_regs *regs) -{ - /* none deferred traps in system call */ -} -static inline bool -native_have_deferred_traps(struct pt_regs *regs) -{ - return false; /* none deferred traps in system call */ -} -static inline void +static inline int native_do_aau_page_fault(struct pt_regs *const regs, e2k_addr_t address, const tc_cond_t condition, const tc_mask_t mask, const unsigned int aa_no) { (void)do_page_fault(regs, address, condition, mask, 0); + return 0; } extern long native_ttable_entry1(int sys_num, ...); @@ -160,17 +168,17 @@ extern const system_call_func sys_call_table_deprecated[NR_syscalls]; #define FILL_HARDWARE_STACKS() NATIVE_FILL_HARDWARE_STACKS() +static inline void clear_fork_child_pt_regs(struct pt_regs *childregs) +{ + native_clear_fork_child_pt_regs(childregs); +} + static inline void correct_trap_return_ip(struct pt_regs *regs, unsigned long return_ip) { native_correct_trap_return_ip(regs, return_ip); } static inline void -handle_deferred_traps_in_syscall(struct pt_regs *regs) -{ - native_handle_deferred_traps_in_syscall(regs); -} -static inline void stack_bounds_trap_enable(void) { native_stack_bounds_trap_enable(); @@ -203,12 +211,12 @@ kvm_mmio_page_fault(struct pt_regs *regs, trap_cellar_t *tcellar) #define instr_page_fault(__regs, __ftype, __async) \ native_do_instr_page_fault(__regs, __ftype, __async) -static inline void +static inline int do_aau_page_fault(struct pt_regs *const regs, e2k_addr_t address, const tc_cond_t condition, const tc_mask_t mask, const unsigned int aa_no) { - native_do_aau_page_fault(regs, address, condition, mask, aa_no); + return native_do_aau_page_fault(regs, address, condition, mask, aa_no); } static inline unsigned int @@ -250,7 +258,7 @@ static inline void init_pt_regs_for_syscall(struct pt_regs *regs) regs->aau_context = NULL; #endif - regs->flags = 0; + AW(regs->flags) = 0; init_guest_syscalls_handling(regs); } #endif diff --git a/arch/e2k/include/asm/traps.h b/arch/e2k/include/asm/traps.h index e8ef0fd..fa06f70 100644 --- a/arch/e2k/include/asm/traps.h +++ b/arch/e2k/include/asm/traps.h @@ -34,6 +34,9 @@ do { \ force_sig_fault(signo, code, addr, trapno); \ } while (0) +extern int pf_on_page_boundary(unsigned long address, tc_cond_t cond); +extern bool is_spurious_qp_store(bool store, unsigned long address, + int fmt, tc_mask_t mask, unsigned long *pf_address); extern void parse_TIR_registers(struct pt_regs *regs, u64 exceptions); extern void do_aau_fault(int aa_field, struct pt_regs *regs); extern int handle_proc_stack_bounds(struct e2k_stacks *stacks, diff --git a/arch/e2k/include/asm/uaccess.h b/arch/e2k/include/asm/uaccess.h index 4383af2..3e9c960 100644 --- a/arch/e2k/include/asm/uaccess.h +++ b/arch/e2k/include/asm/uaccess.h @@ -119,9 +119,15 @@ struct exception_table_entry might_fault(); \ __TRY_USR_PFAULT +#pragma unknown_control_flow(set_usr_pfault_jump) +static __always_inline void set_usr_pfault_jump(void) +{ + SAVE_CURRENT_ADDR(¤t_thread_info()->usr_pfault_jump); +} + #define __TRY_USR_PFAULT \ - unsigned long _usr_pfault_jmp = current_thread_info()->usr_pfault_jump;\ - SAVE_CURRENT_ADDR(¤t_thread_info()->usr_pfault_jump); \ + unsigned long _usr_pfault_jmp = current_thread_info()->usr_pfault_jump; \ + set_usr_pfault_jump(); \ if (likely(current_thread_info()->usr_pfault_jump)) { #define CATCH_USR_PFAULT \ diff --git a/arch/e2k/include/uapi/asm/siginfo.h b/arch/e2k/include/uapi/asm/siginfo.h index ddc8b3f..f73f205 100644 --- a/arch/e2k/include/uapi/asm/siginfo.h +++ b/arch/e2k/include/uapi/asm/siginfo.h @@ -6,7 +6,7 @@ #define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) #define __ARCH_SI_TRAPNO -#define __ARCH_SI_BAND_T int +#define __ARCH_SI_BAND_T long #include diff --git a/arch/e2k/include/uapi/asm/unistd.h b/arch/e2k/include/uapi/asm/unistd.h index 0b0ea39..5dafd19 100644 --- a/arch/e2k/include/uapi/asm/unistd.h +++ b/arch/e2k/include/uapi/asm/unistd.h @@ -431,9 +431,7 @@ #define __NR_copy_file_range 394 #define __NR_preadv2 395 #define __NR_pwritev2 396 -#define __NR_pkey_mprotect 397 -#define __NR_pkey_alloc 398 -#define __NR_pkey_free 399 +/* free (unused) entries - reserve 397 - 399 */ #define __NR_name_to_handle_at 400 #define __NR_open_by_handle_at 401 #define __NR_statx 402 diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index 3f6fdde..defed62 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -614,7 +614,6 @@ acpi_status acpi_remove_pm_notifier(struct acpi_device *adev); bool acpi_pm_device_can_wakeup(struct device *dev); int acpi_pm_device_sleep_state(struct device *, int *, int); int acpi_pm_set_device_wakeup(struct device *dev, bool enable); -int acpi_pm_set_bridge_wakeup(struct device *dev, bool enable); #else static inline void acpi_pm_wakeup_event(struct device *dev) { @@ -645,10 +644,6 @@ static inline int acpi_pm_set_device_wakeup(struct device *dev, bool enable) { return -ENODEV; } -static inline int acpi_pm_set_bridge_wakeup(struct device *dev, bool enable) -{ - return -ENODEV; -} #endif #ifdef CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index 559f8c6..90a0630 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -1162,8 +1162,17 @@ static inline bool arch_has_pfn_modify_check(void) #endif /* !__ASSEMBLY__ */ -#ifndef io_remap_pfn_range -#define io_remap_pfn_range remap_pfn_range +#if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) +#ifdef CONFIG_PHYS_ADDR_T_64BIT +/* + * ZSMALLOC needs to know the highest PFN on 32-bit architectures + * with physical address space extension, but falls back to + * BITS_PER_LONG otherwise. + */ +#error Missing MAX_POSSIBLE_PHYSMEM_BITS definition +#else +#define MAX_POSSIBLE_PHYSMEM_BITS 32 +#endif #endif #ifndef has_transparent_hugepage @@ -1174,6 +1183,16 @@ static inline bool arch_has_pfn_modify_check(void) #endif #endif +#ifndef p4d_offset_lockless +#define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address) +#endif +#ifndef pud_offset_lockless +#define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address) +#endif +#ifndef pmd_offset_lockless +#define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address) +#endif + /* * On some architectures it depends on the mm if the p4d/pud or pmd * layer of the page table hierarchy is folded or not. diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index d7616d0..130f16c 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -340,6 +340,7 @@ */ #ifndef RO_AFTER_INIT_DATA #define RO_AFTER_INIT_DATA \ + . = ALIGN(8); \ __start_ro_after_init = .; \ *(.data..ro_after_init) \ JUMP_TABLE_DATA \ @@ -519,7 +520,10 @@ */ #define TEXT_TEXT \ ALIGN_FUNCTION(); \ - *(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \ + *(.text.hot .text.hot.*) \ + *(TEXT_MAIN .text.fixup) \ + *(.text.unlikely .text.unlikely.*) \ + *(.text.unknown .text.unknown.*) \ *(.text..refcount) \ *(.ref.text) \ MEM_KEEP(init.text*) \ @@ -598,7 +602,7 @@ #define BTF \ .BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \ __start_BTF = .; \ - *(.BTF) \ + KEEP(*(.BTF)) \ __stop_BTF = .; \ } #else diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index 864849e..c1a8d4a 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -135,6 +135,7 @@ struct af_alg_async_req { * SG? * @enc: Cryptographic operation to be performed when * recvmsg is invoked. + * @init: True if metadata has been sent. * @len: Length of memory allocated for this data structure. */ struct af_alg_ctx { @@ -151,6 +152,7 @@ struct af_alg_ctx { bool more; bool merge; bool enc; + bool init; unsigned int len; }; @@ -226,7 +228,7 @@ unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset); void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, size_t dst_offset); void af_alg_wmem_wakeup(struct sock *sk); -int af_alg_wait_for_data(struct sock *sk, unsigned flags); +int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min); int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, unsigned int ivsize); ssize_t af_alg_sendpage(struct socket *sock, struct page *page, diff --git a/include/linux/acpi.h b/include/linux/acpi.h index ce29a01..dd61703 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -837,6 +837,13 @@ static inline int acpi_device_modalias(struct device *dev, return -ENODEV; } +static inline struct platform_device * +acpi_create_platform_device(struct acpi_device *adev, + struct property_entry *properties) +{ + return NULL; +} + static inline bool acpi_dma_supported(struct acpi_device *adev) { return false; diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index 080012a..157e4a6 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -76,6 +76,8 @@ ARM_SMCCC_SMC_32, \ 0, 0x7fff) +#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED 1 + #ifndef __ASSEMBLY__ #include diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index b40fc63..a345d9f 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -44,7 +44,13 @@ struct linux_binprm { * exec has happened. Used to sanitize execution environment * and to set AT_SECURE auxv for glibc. */ - secureexec:1; + secureexec:1, + /* + * Set by flush_old_exec, when exec_mmap has been called. + * This is past the point of no return, when the + * exec_update_mutex has been taken. + */ + called_exec_mmap:1; #ifdef __alpha__ unsigned int taso:1; #endif diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h index 4bbb5f1..4c0224f 100644 --- a/include/linux/bitfield.h +++ b/include/linux/bitfield.h @@ -64,7 +64,7 @@ */ #define FIELD_FIT(_mask, _val) \ ({ \ - __BF_FIELD_CHECK(_mask, 0ULL, _val, "FIELD_FIT: "); \ + __BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_FIT: "); \ !((((typeof(_mask))_val) << __bf_shf(_mask)) & ~(_mask)); \ }) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 26a6d58..81c7ea8 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -342,6 +342,7 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u16 stack_depth; /* max. stack depth used by this function */ + bool has_tail_call; }; /* single container for all structs diff --git a/include/linux/build_bug.h b/include/linux/build_bug.h index 0fe5426..1fb55c0 100644 --- a/include/linux/build_bug.h +++ b/include/linux/build_bug.h @@ -77,4 +77,9 @@ #define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr) #define __static_assert(expr, msg, ...) _Static_assert(expr, msg) +#ifdef __GENKSYMS__ +/* genksyms gets confused by _Static_assert */ +#define _Static_assert(expr, ...) +#endif + #endif /* _LINUX_BUILD_BUG_H */ diff --git a/include/linux/bvec.h b/include/linux/bvec.h index a032f01..d7a628e 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -110,11 +110,18 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, return true; } +static inline void bvec_iter_skip_zero_bvec(struct bvec_iter *iter) +{ + iter->bi_bvec_done = 0; + iter->bi_idx++; +} + #define for_each_bvec(bvl, bio_vec, iter, start) \ for (iter = (start); \ (iter).bi_size && \ ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \ - bvec_iter_advance((bio_vec), &(iter), (bvl).bv_len)) + (bvl).bv_len ? (void)bvec_iter_advance((bio_vec), &(iter), \ + (bvl).bv_len) : bvec_iter_skip_zero_bvec(&(iter))) /* for iterating one bio from start to end */ #define BVEC_ITER_ALL_INIT (struct bvec_iter) \ diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index a954def..0783b0c 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -61,21 +61,17 @@ static inline void can_skb_set_owner(struct sk_buff *skb, struct sock *sk) */ static inline struct sk_buff *can_create_echo_skb(struct sk_buff *skb) { - if (skb_shared(skb)) { - struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); + struct sk_buff *nskb; - if (likely(nskb)) { - can_skb_set_owner(nskb, skb->sk); - consume_skb(skb); - return nskb; - } else { - kfree_skb(skb); - return NULL; - } + nskb = skb_clone(skb, GFP_ATOMIC); + if (unlikely(!nskb)) { + kfree_skb(skb); + return NULL; } - /* we can assume to have an unshared skb with proper owner */ - return skb; + can_skb_set_owner(nskb, skb->sk); + consume_skb(skb); + return nskb; } #endif /* !_CAN_SKB_H */ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 333a669..0e06df2 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -36,9 +36,3 @@ __has_builtin(__builtin_sub_overflow) #define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1 #endif - -/* The following are for compatibility with GCC, from compiler-gcc.h, - * and may be redefined here because they should not be shared with other - * compilers, like ICC. - */ -#define barrier() __asm__ __volatile__("" : : : "memory") diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index 77db11a..88bb4bd 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h @@ -16,25 +16,6 @@ # error Sorry, your compiler is too old - please upgrade it. #endif -/* Optimization barrier */ -/* The "volatile" is due to gcc bugs */ - -#define barrier() __asm__ __volatile__("": : :"memory") -/* - * This version is i.e. to prevent dead stores elimination on @ptr - * where gcc and llvm may behave differently when otherwise using - * normal barrier(): while gcc behavior gets along with a normal - * barrier(), llvm needs an explicit input variable to be assumed - * clobbered. The issue is as follows: while the inline asm might - * access any memory it wants, the compiler could have fit all of - * @ptr into memory registers instead, and since @ptr never escaped - * from that, it proved that the inline asm wasn't touching any of - * it. This version works well with both compilers, i.e. we're telling - * the compiler that the inline asm absolutely may see the contents - * of @ptr. See also: https://llvm.org/bugs/show_bug.cgi?id=15495 - */ -#define barrier_data(ptr) __asm__ __volatile__("": :"r"(ptr) :"memory") - /* * This macro obfuscates arithmetic on a variable address so that gcc * shouldn't recognize the original var, and make assumptions about it. @@ -179,10 +160,3 @@ #else #define __diag_GCC_8(s) #endif - -#if defined(CONFIG_MCST) && defined(__LCC__) - /* lcc bug #121410 workaround */ -#define __no_fgcse -#else -#define __no_fgcse __attribute__((optimize("-fno-gcse"))) -#endif diff --git a/include/linux/compiler.h b/include/linux/compiler.h index d8f0050..223f9e2 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -99,11 +99,25 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, /* Optimization barrier */ #ifndef barrier -# define barrier() __memory_barrier() +/* The "volatile" is due to gcc bugs */ +# define barrier() __asm__ __volatile__("": : :"memory") #endif #ifndef barrier_data -# define barrier_data(ptr) barrier() +/* + * This version is i.e. to prevent dead stores elimination on @ptr + * where gcc and llvm may behave differently when otherwise using + * normal barrier(): while gcc behavior gets along with a normal + * barrier(), llvm needs an explicit input variable to be assumed + * clobbered. The issue is as follows: while the inline asm might + * access any memory it wants, the compiler could have fit all of + * @ptr into memory registers instead, and since @ptr never escaped + * from that, it proved that the inline asm wasn't touching any of + * it. This version works well with both compilers, i.e. we're telling + * the compiler that the inline asm absolutely may see the contents + * of @ptr. See also: https://llvm.org/bugs/show_bug.cgi?id=15495 + */ +# define barrier_data(ptr) __asm__ __volatile__("": :"r"(ptr) :"memory") #endif /* workaround for GCC PR82365 if needed */ diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 72393a8..7743363 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -212,10 +212,6 @@ struct ftrace_likely_data { #define asm_inline asm #endif -#ifndef __no_fgcse -# define __no_fgcse -#endif - /* Are two types/vars the same type (ignoring qualifiers)? */ #define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b)) diff --git a/include/linux/dax.h b/include/linux/dax.h index 9bd8528..72a7f03 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -56,6 +56,8 @@ static inline void set_dax_synchronous(struct dax_device *dax_dev) { __set_dax_synchronous(dax_dev); } +bool dax_supported(struct dax_device *dax_dev, struct block_device *bdev, + int blocksize, sector_t start, sector_t len); /* * Check if given mapping is supported by the file / underlying device. */ @@ -102,6 +104,12 @@ static inline bool dax_synchronous(struct dax_device *dax_dev) static inline void set_dax_synchronous(struct dax_device *dax_dev) { } +static inline bool dax_supported(struct dax_device *dax_dev, + struct block_device *bdev, int blocksize, sector_t start, + sector_t len) +{ + return false; +} static inline bool daxdev_mapping_supported(struct vm_area_struct *vma, struct dax_device *dax_dev) { @@ -197,14 +205,23 @@ static inline void dax_unlock_page(struct page *page, dax_entry_t cookie) } #endif +#if IS_ENABLED(CONFIG_DAX) int dax_read_lock(void); void dax_read_unlock(int id); +#else +static inline int dax_read_lock(void) +{ + return 0; +} + +static inline void dax_read_unlock(int id) +{ +} +#endif /* CONFIG_DAX */ bool dax_alive(struct dax_device *dax_dev); void *dax_get_private(struct dax_device *dax_dev); long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn); -bool dax_supported(struct dax_device *dax_dev, struct block_device *bdev, - int blocksize, sector_t start, sector_t len); size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i); size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, diff --git a/include/linux/debugfs.h b/include/linux/debugfs.h index 58424eb..798f0b9 100644 --- a/include/linux/debugfs.h +++ b/include/linux/debugfs.h @@ -54,6 +54,8 @@ static const struct file_operations __fops = { \ .llseek = no_llseek, \ } +typedef struct vfsmount *(*debugfs_automount_t)(struct dentry *, void *); + #if defined(CONFIG_DEBUG_FS) struct dentry *debugfs_lookup(const char *name, struct dentry *parent); @@ -75,7 +77,6 @@ struct dentry *debugfs_create_dir(const char *name, struct dentry *parent); struct dentry *debugfs_create_symlink(const char *name, struct dentry *parent, const char *dest); -typedef struct vfsmount *(*debugfs_automount_t)(struct dentry *, void *); struct dentry *debugfs_create_automount(const char *name, struct dentry *parent, debugfs_automount_t f, @@ -203,7 +204,7 @@ static inline struct dentry *debugfs_create_symlink(const char *name, static inline struct dentry *debugfs_create_automount(const char *name, struct dentry *parent, - struct vfsmount *(*f)(void *), + debugfs_automount_t f, void *data) { return ERR_PTR(-ENODEV); diff --git a/include/linux/dm-bufio.h b/include/linux/dm-bufio.h index 3c8b7d2..45ba37a 100644 --- a/include/linux/dm-bufio.h +++ b/include/linux/dm-bufio.h @@ -138,6 +138,7 @@ void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned n); unsigned dm_bufio_get_block_size(struct dm_bufio_client *c); sector_t dm_bufio_get_device_size(struct dm_bufio_client *c); +struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c); sector_t dm_bufio_get_block_number(struct dm_buffer *b); void *dm_bufio_get_block_data(struct dm_buffer *b); void *dm_bufio_get_aux_data(struct dm_buffer *b); diff --git a/include/linux/efi.h b/include/linux/efi.h index d87acf6..13ed2c6 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1039,7 +1039,11 @@ extern void *efi_get_pal_addr (void); extern void efi_map_pal_code (void); extern void efi_memmap_walk (efi_freemem_callback_t callback, void *arg); extern void efi_gettimeofday (struct timespec64 *ts); +#ifdef CONFIG_EFI extern void efi_enter_virtual_mode (void); /* switch EFI to virtual mode, if possible */ +#else +static inline void efi_enter_virtual_mode (void) {} +#endif #ifdef CONFIG_X86 extern efi_status_t efi_query_variable_store(u32 attributes, unsigned long size, diff --git a/include/linux/fb.h b/include/linux/fb.h index c858acb..dead4a9 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -400,8 +400,6 @@ struct fb_tile_ops { #define FBINFO_HWACCEL_YPAN 0x2000 /* optional */ #define FBINFO_HWACCEL_YWRAP 0x4000 /* optional */ -#define FBINFO_MISC_USEREVENT 0x10000 /* event request - from userspace */ #define FBINFO_MISC_TILEBLITTING 0x20000 /* use tile blitting */ /* A driver may set this flag to indicate that it does want a set_par to be diff --git a/include/linux/font.h b/include/linux/font.h index 51b91c8..59faa80 100644 --- a/include/linux/font.h +++ b/include/linux/font.h @@ -59,4 +59,17 @@ extern const struct font_desc *get_default_font(int xres, int yres, /* Max. length for the name of a predefined font */ #define MAX_FONT_NAME 32 +/* Extra word getters */ +#define REFCOUNT(fd) (((int *)(fd))[-1]) +#define FNTSIZE(fd) (((int *)(fd))[-2]) +#define FNTCHARCNT(fd) (((int *)(fd))[-3]) +#define FNTSUM(fd) (((int *)(fd))[-4]) + +#define FONT_EXTRA_WORDS 4 + +struct font_data { + unsigned int extra[FONT_EXTRA_WORDS]; + const unsigned char data[]; +} __packed; + #endif /* _VIDEO_FONT_H */ diff --git a/include/linux/fs.h b/include/linux/fs.h index 76cf42d..0c4cf54 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1449,9 +1449,6 @@ struct super_block { struct rw_semaphore s_umount; int s_count; atomic_t s_active; -#ifdef CONFIG_MCST_FS_SECDEL - int overwriting_count; -#endif #ifdef CONFIG_SECURITY void *s_security; #endif @@ -1780,16 +1777,6 @@ typedef int (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64, struct dir_context { filldir_t actor; loff_t pos; -#if defined(CONFIG_MCST_SECURITY_ELMAC) && defined(CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES) -/* previous line is long becouse of community script 'unifdef' - * is not able to handle multi-lines */ - /* Required to keep the original callback - * in the security_iterate_dir hook. */ - const filldir_t saved_actor; - /* Required to get the inode structure from the pathname - * in the security_iterate hook. */ - struct file *file; -#endif }; struct block_device_operations; @@ -2162,6 +2149,10 @@ static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp) * * I_CREATING New object's inode in the middle of setting up. * + * I_SYNC_QUEUED Inode is queued in b_io or b_more_io writeback lists. + * Used to detect that mark_inode_dirty() should not move + * inode between dirty lists. + * * Q: What is the difference between I_WILL_FREE and I_FREEING? */ #define I_DIRTY_SYNC (1 << 0) @@ -2179,11 +2170,11 @@ static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp) #define I_DIO_WAKEUP (1 << __I_DIO_WAKEUP) #define I_LINKABLE (1 << 10) #define I_DIRTY_TIME (1 << 11) -#define __I_DIRTY_TIME_EXPIRED 12 -#define I_DIRTY_TIME_EXPIRED (1 << __I_DIRTY_TIME_EXPIRED) +#define I_DIRTY_TIME_EXPIRED (1 << 12) #define I_WB_SWITCH (1 << 13) #define I_OVL_INUSE (1 << 14) #define I_CREATING (1 << 15) +#define I_SYNC_QUEUED (1 << 17) #define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC) #define I_DIRTY (I_DIRTY_INODE | I_DIRTY_PAGES) @@ -2613,12 +2604,6 @@ static inline bool sb_is_blkdev_sb(struct super_block *sb) { return sb == blockdev_superblock; } -#ifdef CONFIG_MCST_FS_SECDEL -extern int blkdev_zero_blocks(struct super_block *sb, - int (*trim_fs_func)(struct super_block *sb, - struct fstrim_range *range), - sector_t block, sector_t count); -#endif #else static inline void bd_forget(struct inode *inode) {} static inline int sync_blockdev(struct block_device *bdev) { return 0; } diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index f622f74..032e5bc 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -100,6 +100,35 @@ static inline void fscrypt_handle_d_move(struct dentry *dentry) dentry->d_flags &= ~DCACHE_ENCRYPTED_NAME; } +/** + * fscrypt_is_nokey_name() - test whether a dentry is a no-key name + * @dentry: the dentry to check + * + * This returns true if the dentry is a no-key dentry. A no-key dentry is a + * dentry that was created in an encrypted directory that hasn't had its + * encryption key added yet. Such dentries may be either positive or negative. + * + * When a filesystem is asked to create a new filename in an encrypted directory + * and the new filename's dentry is a no-key dentry, it must fail the operation + * with ENOKEY. This includes ->create(), ->mkdir(), ->mknod(), ->symlink(), + * ->rename(), and ->link(). (However, ->rename() and ->link() are already + * handled by fscrypt_prepare_rename() and fscrypt_prepare_link().) + * + * This is necessary because creating a filename requires the directory's + * encryption key, but just checking for the key on the directory inode during + * the final filesystem operation doesn't guarantee that the key was available + * during the preceding dentry lookup. And the key must have already been + * available during the dentry lookup in order for it to have been checked + * whether the filename already exists in the directory and for the new file's + * dentry not to be invalidated due to it incorrectly having the no-key flag. + * + * Return: %true if the dentry is a no-key name + */ +static inline bool fscrypt_is_nokey_name(const struct dentry *dentry) +{ + return dentry->d_flags & DCACHE_ENCRYPTED_NAME; +} + /* crypto.c */ extern void fscrypt_enqueue_decrypt_work(struct work_struct *); extern struct fscrypt_ctx *fscrypt_get_ctx(gfp_t); @@ -290,6 +319,11 @@ static inline void fscrypt_handle_d_move(struct dentry *dentry) { } +static inline bool fscrypt_is_nokey_name(const struct dentry *dentry) +{ + return false; +} + /* crypto.c */ static inline void fscrypt_enqueue_decrypt_work(struct work_struct *work) { diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 825f7db..48886ae 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -419,7 +419,7 @@ static inline void free_part_info(struct hd_struct *part) kfree(part->info); } -void update_io_ticks(struct hd_struct *part, unsigned long now); +void update_io_ticks(struct hd_struct *part, unsigned long now, bool end); /* block/genhd.c */ extern void device_add_disk(struct device *parent, struct gendisk *disk, diff --git a/include/linux/hid.h b/include/linux/hid.h index 875f711..c7044a1 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -959,34 +959,49 @@ static inline void hid_device_io_stop(struct hid_device *hid) { * @max: maximal valid usage->code to consider later (out parameter) * @type: input event type (EV_KEY, EV_REL, ...) * @c: code which corresponds to this usage and type + * + * The value pointed to by @bit will be set to NULL if either @type is + * an unhandled event type, or if @c is out of range for @type. This + * can be used as an error condition. */ static inline void hid_map_usage(struct hid_input *hidinput, struct hid_usage *usage, unsigned long **bit, int *max, - __u8 type, __u16 c) + __u8 type, unsigned int c) { struct input_dev *input = hidinput->input; - - usage->type = type; - usage->code = c; + unsigned long *bmap = NULL; + unsigned int limit = 0; switch (type) { case EV_ABS: - *bit = input->absbit; - *max = ABS_MAX; + bmap = input->absbit; + limit = ABS_MAX; break; case EV_REL: - *bit = input->relbit; - *max = REL_MAX; + bmap = input->relbit; + limit = REL_MAX; break; case EV_KEY: - *bit = input->keybit; - *max = KEY_MAX; + bmap = input->keybit; + limit = KEY_MAX; break; case EV_LED: - *bit = input->ledbit; - *max = LED_MAX; + bmap = input->ledbit; + limit = LED_MAX; break; } + + if (unlikely(c > limit || !bmap)) { + pr_warn_ratelimited("%s: Invalid code %d type %d\n", + input->name, c, type); + *bit = NULL; + return; + } + + usage->type = type; + usage->code = c; + *max = limit; + *bit = bmap; } /** @@ -1000,7 +1015,8 @@ static inline void hid_map_usage_clear(struct hid_input *hidinput, __u8 type, __u16 c) { hid_map_usage(hidinput, usage, bit, max, type, c); - clear_bit(c, *bit); + if (*bit) + clear_bit(usage->code, *bit); } /** diff --git a/include/linux/hil_mlc.h b/include/linux/hil_mlc.h index 774f7d3..369221f 100644 --- a/include/linux/hil_mlc.h +++ b/include/linux/hil_mlc.h @@ -103,7 +103,7 @@ struct hilse_node { /* Methods for back-end drivers, e.g. hp_sdc_mlc */ typedef int (hil_mlc_cts) (hil_mlc *mlc); -typedef void (hil_mlc_out) (hil_mlc *mlc); +typedef int (hil_mlc_out) (hil_mlc *mlc); typedef int (hil_mlc_in) (hil_mlc *mlc, suseconds_t timeout); struct hil_mlc_devinfo { diff --git a/include/linux/i2c-algo-pca.h b/include/linux/i2c-algo-pca.h index d030717..7c522fd 100644 --- a/include/linux/i2c-algo-pca.h +++ b/include/linux/i2c-algo-pca.h @@ -53,6 +53,20 @@ #define I2C_PCA_CON_SI 0x08 /* Serial Interrupt */ #define I2C_PCA_CON_CR 0x07 /* Clock Rate (MASK) */ +/** + * struct pca_i2c_bus_settings - The configured PCA i2c bus settings + * @mode: Configured i2c bus mode + * @tlow: Configured SCL LOW period + * @thi: Configured SCL HIGH period + * @clock_freq: The configured clock frequency + */ +struct pca_i2c_bus_settings { + int mode; + int tlow; + int thi; + int clock_freq; +}; + struct i2c_algo_pca_data { void *data; /* private low level data */ void (*write_byte) (void *data, int reg, int val); @@ -64,6 +78,7 @@ struct i2c_algo_pca_data { * For PCA9665, use the frequency you want here. */ unsigned int i2c_clock; unsigned int chip; + struct pca_i2c_bus_settings bus_settings; }; int i2c_pca_add_bus(struct i2c_adapter *); diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 1e5dad8..6b559d2 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -359,8 +359,8 @@ enum { #define QI_DEV_EIOTLB_ADDR(a) ((u64)(a) & VTD_PAGE_MASK) #define QI_DEV_EIOTLB_SIZE (((u64)1) << 11) -#define QI_DEV_EIOTLB_GLOB(g) ((u64)g) -#define QI_DEV_EIOTLB_PASID(p) (((u64)p) << 32) +#define QI_DEV_EIOTLB_GLOB(g) ((u64)(g) & 0x1) +#define QI_DEV_EIOTLB_PASID(p) ((u64)((p) & 0xfffff) << 32) #define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16) #define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4) #define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | \ @@ -706,7 +706,7 @@ extern int iommu_calculate_agaw(struct intel_iommu *iommu); extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); extern int dmar_disabled; extern int intel_iommu_enabled; -extern int intel_iommu_tboot_noforce; +extern int intel_iommu_gfx_mapped; #else static inline int iommu_calculate_agaw(struct intel_iommu *iommu) { diff --git a/include/linux/irq.h b/include/linux/irq.h index f8755e5..e9e69c5 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -211,6 +211,8 @@ struct irq_data { * IRQD_CAN_RESERVE - Can use reservation mode * IRQD_MSI_NOMASK_QUIRK - Non-maskable MSI quirk for affinity change * required + * IRQD_AFFINITY_ON_ACTIVATE - Affinity is set on activation. Don't call + * irq_chip::irq_set_affinity() when deactivated. */ enum { IRQD_TRIGGER_MASK = 0xf, @@ -234,6 +236,7 @@ enum { IRQD_DEFAULT_TRIGGER_SET = (1 << 25), IRQD_CAN_RESERVE = (1 << 26), IRQD_MSI_NOMASK_QUIRK = (1 << 27), + IRQD_AFFINITY_ON_ACTIVATE = (1 << 29), }; #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors) @@ -408,6 +411,16 @@ static inline bool irqd_msi_nomask_quirk(struct irq_data *d) return __irqd_to_state(d) & IRQD_MSI_NOMASK_QUIRK; } +static inline void irqd_set_affinity_on_activate(struct irq_data *d) +{ + __irqd_to_state(d) |= IRQD_AFFINITY_ON_ACTIVATE; +} + +static inline bool irqd_affinity_on_activate(struct irq_data *d) +{ + return __irqd_to_state(d) & IRQD_AFFINITY_ON_ACTIVATE; +} + #undef __irqd_to_state static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d) diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index e85f714..824d7a1 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -382,11 +382,19 @@ extern void irq_domain_associate_many(struct irq_domain *domain, extern void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq); -extern unsigned int irq_create_mapping(struct irq_domain *host, - irq_hw_number_t hwirq); +extern unsigned int irq_create_mapping_affinity(struct irq_domain *host, + irq_hw_number_t hwirq, + const struct irq_affinity_desc *affinity); extern unsigned int irq_create_fwspec_mapping(struct irq_fwspec *fwspec); extern void irq_dispose_mapping(unsigned int virq); +static inline unsigned int irq_create_mapping(struct irq_domain *host, + irq_hw_number_t hwirq) +{ + return irq_create_mapping_affinity(host, hwirq, NULL); +} + + /** * irq_linear_revmap() - Find a linux irq from a hw irq number. * @domain: domain owning this hardware interrupt diff --git a/include/linux/kdev_t.h b/include/linux/kdev_t.h index 85b5151..4856706 100644 --- a/include/linux/kdev_t.h +++ b/include/linux/kdev_t.h @@ -21,61 +21,61 @@ }) /* acceptable for old filesystems */ -static inline bool old_valid_dev(dev_t dev) +static __always_inline bool old_valid_dev(dev_t dev) { return MAJOR(dev) < 256 && MINOR(dev) < 256; } -static inline u16 old_encode_dev(dev_t dev) +static __always_inline u16 old_encode_dev(dev_t dev) { return (MAJOR(dev) << 8) | MINOR(dev); } -static inline dev_t old_decode_dev(u16 val) +static __always_inline dev_t old_decode_dev(u16 val) { return MKDEV((val >> 8) & 255, val & 255); } -static inline u32 new_encode_dev(dev_t dev) +static __always_inline u32 new_encode_dev(dev_t dev) { unsigned major = MAJOR(dev); unsigned minor = MINOR(dev); return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12); } -static inline dev_t new_decode_dev(u32 dev) +static __always_inline dev_t new_decode_dev(u32 dev) { unsigned major = (dev & 0xfff00) >> 8; unsigned minor = (dev & 0xff) | ((dev >> 12) & 0xfff00); return MKDEV(major, minor); } -static inline u64 huge_encode_dev(dev_t dev) +static __always_inline u64 huge_encode_dev(dev_t dev) { return new_encode_dev(dev); } -static inline dev_t huge_decode_dev(u64 dev) +static __always_inline dev_t huge_decode_dev(u64 dev) { return new_decode_dev(dev); } -static inline int sysv_valid_dev(dev_t dev) +static __always_inline int sysv_valid_dev(dev_t dev) { return MAJOR(dev) < (1<<14) && MINOR(dev) < (1<<18); } -static inline u32 sysv_encode_dev(dev_t dev) +static __always_inline u32 sysv_encode_dev(dev_t dev) { return MINOR(dev) | (MAJOR(dev) << 18); } -static inline unsigned sysv_major(u32 dev) +static __always_inline unsigned sysv_major(u32 dev) { return (dev >> 18) & 0x3fff; } -static inline unsigned sysv_minor(u32 dev) +static __always_inline unsigned sysv_minor(u32 dev) { return dev & 0x3ffff; } diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index bc45ea1..c941b73 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -15,6 +15,7 @@ extern int __khugepaged_enter(struct mm_struct *mm); extern void __khugepaged_exit(struct mm_struct *mm); extern int khugepaged_enter_vma_merge(struct vm_area_struct *vma, unsigned long vm_flags); +extern void khugepaged_min_free_kbytes_update(void); #ifdef CONFIG_SHMEM extern void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr); #else @@ -85,6 +86,10 @@ static inline void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) { } + +static inline void khugepaged_min_free_kbytes_update(void) +{ +} #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* _LINUX_KHUGEPAGED_H */ diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 645fd40..a604888 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -369,6 +369,8 @@ void unregister_kretprobes(struct kretprobe **rps, int num); void kprobe_flush_task(struct task_struct *tk); void recycle_rp_inst(struct kretprobe_instance *ri, struct hlist_head *head); +void kprobe_free_init_mem(void); + int disable_kprobe(struct kprobe *kp); int enable_kprobe(struct kprobe *kp); @@ -426,6 +428,9 @@ static inline void unregister_kretprobes(struct kretprobe **rps, int num) static inline void kprobe_flush_task(struct task_struct *tk) { } +static inline void kprobe_free_init_mem(void) +{ +} static inline int disable_kprobe(struct kprobe *kp) { return -ENOSYS; diff --git a/include/linux/libata.h b/include/linux/libata.h index d914429..539237b 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -222,7 +222,8 @@ enum { ATA_FLAG_NO_DIPM = (1 << 23), /* host not happy with DIPM */ ATA_FLAG_SAS_HOST = (1 << 24), /* SAS host */ #ifdef CONFIG_MCST - ATA_FLAG_IOHUB2_REV2 = (1 << 25), /* iohub2 rev2 bugs */ + ATA_FLAG_IOHUB2_REV2 = (1 << 25), /* iohub2 rev2 bugs */ + ATA_FLAG_E2C3_REV0 = (1 << 26), /* E2C3 rev0 bugs */ #endif /* bits 24:31 of ap->flags are reserved for LLD specific flags */ @@ -428,8 +429,9 @@ enum { ATA_HORKAGE_NO_DMA_LOG = (1 << 23), /* don't use DMA for log read */ ATA_HORKAGE_NOTRIM = (1 << 24), /* don't use TRIM */ ATA_HORKAGE_MAX_SEC_1024 = (1 << 25), /* Limit max sects to 1024 */ + ATA_HORKAGE_MAX_TRIM_128M = (1 << 26), /* Limit max trim size to 128M */ #ifdef CONFIG_MCST - ATA_HORKAGE_FIX_ERROR_ON_WRITE = (1 << 26), + ATA_HORKAGE_FIX_ERROR_ON_WRITE = (1 << 27), #endif /* DMA mask for user DMA control: User visible values; DO NOT renumber */ @@ -493,6 +495,7 @@ enum hsm_task_states { }; enum ata_completion_errors { + AC_ERR_OK = 0, /* no error */ AC_ERR_DEV = (1 << 0), /* device reported error */ AC_ERR_HSM = (1 << 1), /* host state machine violation */ AC_ERR_TIMEOUT = (1 << 2), /* timeout */ @@ -902,9 +905,9 @@ struct ata_port_operations { /* * Command execution */ - int (*qc_defer)(struct ata_queued_cmd *qc); - int (*check_atapi_dma)(struct ata_queued_cmd *qc); - void (*qc_prep)(struct ata_queued_cmd *qc); + int (*qc_defer)(struct ata_queued_cmd *qc); + int (*check_atapi_dma)(struct ata_queued_cmd *qc); + enum ata_completion_errors (*qc_prep)(struct ata_queued_cmd *qc); unsigned int (*qc_issue)(struct ata_queued_cmd *qc); bool (*qc_fill_rtf)(struct ata_queued_cmd *qc); @@ -1172,7 +1175,7 @@ extern int ata_xfer_mode2shift(unsigned long xfer_mode); extern const char *ata_mode_string(unsigned long xfer_mask); extern unsigned long ata_id_xfermask(const u16 *id); extern int ata_std_qc_defer(struct ata_queued_cmd *qc); -extern void ata_noop_qc_prep(struct ata_queued_cmd *qc); +extern enum ata_completion_errors ata_noop_qc_prep(struct ata_queued_cmd *qc); extern void ata_sg_init(struct ata_queued_cmd *qc, struct scatterlist *sg, unsigned int n_elem); extern unsigned int ata_dev_classify(const struct ata_taskfile *tf); @@ -1906,9 +1909,9 @@ extern const struct ata_port_operations ata_bmdma_port_ops; .sg_tablesize = LIBATA_MAX_PRD, \ .dma_boundary = ATA_DMA_BOUNDARY -extern void ata_bmdma_qc_prep(struct ata_queued_cmd *qc); +extern enum ata_completion_errors ata_bmdma_qc_prep(struct ata_queued_cmd *qc); extern unsigned int ata_bmdma_qc_issue(struct ata_queued_cmd *qc); -extern void ata_bmdma_dumb_qc_prep(struct ata_queued_cmd *qc); +extern enum ata_completion_errors ata_bmdma_dumb_qc_prep(struct ata_queued_cmd *qc); extern unsigned int ata_bmdma_port_intr(struct ata_port *ap, struct ata_queued_cmd *qc); extern irqreturn_t ata_bmdma_interrupt(int irq, void *dev_instance); diff --git a/include/linux/linkage.h b/include/linux/linkage.h index 7e02078..f3ae8f3 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -75,32 +75,58 @@ #ifdef __ASSEMBLY__ +/* SYM_T_FUNC -- type used by assembler to mark functions */ +#ifndef SYM_T_FUNC +#define SYM_T_FUNC STT_FUNC +#endif + +/* SYM_T_OBJECT -- type used by assembler to mark data */ +#ifndef SYM_T_OBJECT +#define SYM_T_OBJECT STT_OBJECT +#endif + +/* SYM_T_NONE -- type used by assembler to mark entries of unknown type */ +#ifndef SYM_T_NONE +#define SYM_T_NONE STT_NOTYPE +#endif + +/* SYM_A_* -- align the symbol? */ +#define SYM_A_ALIGN ALIGN +#define SYM_A_NONE /* nothing */ + +/* SYM_L_* -- linkage of symbols */ +#define SYM_L_GLOBAL(name) .globl name +#define SYM_L_WEAK(name) .weak name +#define SYM_L_LOCAL(name) /* nothing */ + #ifndef LINKER_SCRIPT #define ALIGN __ALIGN #define ALIGN_STR __ALIGN_STR +/* === DEPRECATED annotations === */ + #ifndef GLOBAL +/* deprecated, use SYM_DATA*, SYM_ENTRY, or similar */ #define GLOBAL(name) \ .globl name ASM_NL \ name: #endif #ifndef ENTRY +/* deprecated, use SYM_FUNC_START */ #define ENTRY(name) \ - .globl name ASM_NL \ - ALIGN ASM_NL \ - name: + SYM_FUNC_START(name) #endif #endif /* LINKER_SCRIPT */ #ifndef WEAK +/* deprecated, use SYM_FUNC_START_WEAK* */ #define WEAK(name) \ - .weak name ASM_NL \ - ALIGN ASM_NL \ - name: + SYM_FUNC_START_WEAK(name) #endif #ifndef END +/* deprecated, use SYM_FUNC_END, SYM_DATA_END, or SYM_END */ #define END(name) \ .size name, .-name #endif @@ -110,11 +136,214 @@ * static analysis tools such as stack depth analyzer. */ #ifndef ENDPROC +/* deprecated, use SYM_FUNC_END */ #define ENDPROC(name) \ - .type name, @function ASM_NL \ - END(name) + SYM_FUNC_END(name) #endif +/* === generic annotations === */ + +/* SYM_ENTRY -- use only if you have to for non-paired symbols */ +#ifndef SYM_ENTRY +#define SYM_ENTRY(name, linkage, align...) \ + linkage(name) ASM_NL \ + align ASM_NL \ + name: #endif +/* SYM_START -- use only if you have to */ +#ifndef SYM_START +#define SYM_START(name, linkage, align...) \ + SYM_ENTRY(name, linkage, align) #endif + +/* SYM_END -- use only if you have to */ +#ifndef SYM_END +#define SYM_END(name, sym_type) \ + .type name sym_type ASM_NL \ + .size name, .-name +#endif + +/* === code annotations === */ + +/* + * FUNC -- C-like functions (proper stack frame etc.) + * CODE -- non-C code (e.g. irq handlers with different, special stack etc.) + * + * Objtool validates stack for FUNC, but not for CODE. + * Objtool generates debug info for both FUNC & CODE, but needs special + * annotations for each CODE's start (to describe the actual stack frame). + * + * ALIAS -- does not generate debug info -- the aliased function will + */ + +/* SYM_INNER_LABEL_ALIGN -- only for labels in the middle of code */ +#ifndef SYM_INNER_LABEL_ALIGN +#define SYM_INNER_LABEL_ALIGN(name, linkage) \ + .type name SYM_T_NONE ASM_NL \ + SYM_ENTRY(name, linkage, SYM_A_ALIGN) +#endif + +/* SYM_INNER_LABEL -- only for labels in the middle of code */ +#ifndef SYM_INNER_LABEL +#define SYM_INNER_LABEL(name, linkage) \ + .type name SYM_T_NONE ASM_NL \ + SYM_ENTRY(name, linkage, SYM_A_NONE) +#endif + +/* + * SYM_FUNC_START_LOCAL_ALIAS -- use where there are two local names for one + * function + */ +#ifndef SYM_FUNC_START_LOCAL_ALIAS +#define SYM_FUNC_START_LOCAL_ALIAS(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) +#endif + +/* + * SYM_FUNC_START_ALIAS -- use where there are two global names for one + * function + */ +#ifndef SYM_FUNC_START_ALIAS +#define SYM_FUNC_START_ALIAS(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) +#endif + +/* SYM_FUNC_START -- use for global functions */ +#ifndef SYM_FUNC_START +/* + * The same as SYM_FUNC_START_ALIAS, but we will need to distinguish these two + * later. + */ +#define SYM_FUNC_START(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) +#endif + +/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */ +#ifndef SYM_FUNC_START_NOALIGN +#define SYM_FUNC_START_NOALIGN(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) +#endif + +/* SYM_FUNC_START_LOCAL -- use for local functions */ +#ifndef SYM_FUNC_START_LOCAL +/* the same as SYM_FUNC_START_LOCAL_ALIAS, see comment near SYM_FUNC_START */ +#define SYM_FUNC_START_LOCAL(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) +#endif + +/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */ +#ifndef SYM_FUNC_START_LOCAL_NOALIGN +#define SYM_FUNC_START_LOCAL_NOALIGN(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) +#endif + +/* SYM_FUNC_START_WEAK -- use for weak functions */ +#ifndef SYM_FUNC_START_WEAK +#define SYM_FUNC_START_WEAK(name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) +#endif + +/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */ +#ifndef SYM_FUNC_START_WEAK_NOALIGN +#define SYM_FUNC_START_WEAK_NOALIGN(name) \ + SYM_START(name, SYM_L_WEAK, SYM_A_NONE) +#endif + +/* SYM_FUNC_END_ALIAS -- the end of LOCAL_ALIASed or ALIASed function */ +#ifndef SYM_FUNC_END_ALIAS +#define SYM_FUNC_END_ALIAS(name) \ + SYM_END(name, SYM_T_FUNC) +#endif + +/* + * SYM_FUNC_END -- the end of SYM_FUNC_START_LOCAL, SYM_FUNC_START, + * SYM_FUNC_START_WEAK, ... + */ +#ifndef SYM_FUNC_END +/* the same as SYM_FUNC_END_ALIAS, see comment near SYM_FUNC_START */ +#define SYM_FUNC_END(name) \ + SYM_END(name, SYM_T_FUNC) +#endif + +/* SYM_CODE_START -- use for non-C (special) functions */ +#ifndef SYM_CODE_START +#define SYM_CODE_START(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) +#endif + +/* SYM_CODE_START_NOALIGN -- use for non-C (special) functions, w/o alignment */ +#ifndef SYM_CODE_START_NOALIGN +#define SYM_CODE_START_NOALIGN(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) +#endif + +/* SYM_CODE_START_LOCAL -- use for local non-C (special) functions */ +#ifndef SYM_CODE_START_LOCAL +#define SYM_CODE_START_LOCAL(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) +#endif + +/* + * SYM_CODE_START_LOCAL_NOALIGN -- use for local non-C (special) functions, + * w/o alignment + */ +#ifndef SYM_CODE_START_LOCAL_NOALIGN +#define SYM_CODE_START_LOCAL_NOALIGN(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) +#endif + +/* SYM_CODE_END -- the end of SYM_CODE_START_LOCAL, SYM_CODE_START, ... */ +#ifndef SYM_CODE_END +#define SYM_CODE_END(name) \ + SYM_END(name, SYM_T_NONE) +#endif + +/* === data annotations === */ + +/* SYM_DATA_START -- global data symbol */ +#ifndef SYM_DATA_START +#define SYM_DATA_START(name) \ + SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) +#endif + +/* SYM_DATA_START -- local data symbol */ +#ifndef SYM_DATA_START_LOCAL +#define SYM_DATA_START_LOCAL(name) \ + SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) +#endif + +/* SYM_DATA_END -- the end of SYM_DATA_START symbol */ +#ifndef SYM_DATA_END +#define SYM_DATA_END(name) \ + SYM_END(name, SYM_T_OBJECT) +#endif + +/* SYM_DATA_END_LABEL -- the labeled end of SYM_DATA_START symbol */ +#ifndef SYM_DATA_END_LABEL +#define SYM_DATA_END_LABEL(name, linkage, label) \ + linkage(label) ASM_NL \ + .type label SYM_T_OBJECT ASM_NL \ + label: \ + SYM_END(name, SYM_T_OBJECT) +#endif + +/* SYM_DATA -- start+end wrapper around simple global data */ +#ifndef SYM_DATA +#define SYM_DATA(name, data...) \ + SYM_DATA_START(name) ASM_NL \ + data ASM_NL \ + SYM_DATA_END(name) +#endif + +/* SYM_DATA_LOCAL -- start+end wrapper around simple local data */ +#ifndef SYM_DATA_LOCAL +#define SYM_DATA_LOCAL(name, data...) \ + SYM_DATA_START_LOCAL(name) ASM_NL \ + data ASM_NL \ + SYM_DATA_END(name) +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* _LINUX_LINKAGE_H */ diff --git a/include/linux/log2.h b/include/linux/log2.h index 83a4a3c..c619ec6 100644 --- a/include/linux/log2.h +++ b/include/linux/log2.h @@ -173,7 +173,7 @@ unsigned long __rounddown_pow_of_two(unsigned long n) #define roundup_pow_of_two(n) \ ( \ __builtin_constant_p(n) ? ( \ - (n == 1) ? 1 : \ + ((n) == 1) ? 1 : \ (1UL << (ilog2((n) - 1) + 1)) \ ) : \ __roundup_pow_of_two(n) \ diff --git a/include/linux/lsm_audit.h b/include/linux/lsm_audit.h index 2495693..915330a 100644 --- a/include/linux/lsm_audit.h +++ b/include/linux/lsm_audit.h @@ -99,9 +99,6 @@ struct common_audit_data { #ifdef CONFIG_SECURITY_SMACK struct smack_audit_data *smack_audit_data; #endif -#ifdef CONFIG_MCST_SECURITY_ELMAC - struct elmac_audit_data *elmac_audit_data; -#endif #ifdef CONFIG_SECURITY_SELINUX struct selinux_audit_data *selinux_audit_data; #endif diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 88b08a4..9ed67dd 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -1807,13 +1807,6 @@ union security_list_options { void (*audit_rule_free)(void *lsmrule); #endif /* CONFIG_AUDIT */ -#ifdef CONFIG_MCST_SECURITY_ELMAC -#ifdef CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES - int (*inode_pre_permission)(struct inode *inode, int mask); - int (*iterate_dir)(struct file *file, struct dir_context *ctx, - bool shared); -#endif /* CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES */ -#endif /* CONFIG_MCST_SECURITY_ELMAC */ #ifdef CONFIG_BPF_SYSCALL int (*bpf)(int cmd, union bpf_attr *attr, @@ -2058,12 +2051,6 @@ struct security_hook_heads { struct hlist_head audit_rule_match; struct hlist_head audit_rule_free; #endif /* CONFIG_AUDIT */ -#ifdef CONFIG_MCST_SECURITY_ELMAC -#ifdef CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES - struct hlist_head inode_pre_permission; - struct hlist_head iterate_dir; -#endif /* CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES */ -#endif /* CONFIG_MCST_SECURITY_ELMAC */ #ifdef CONFIG_BPF_SYSCALL struct hlist_head bpf; struct hlist_head bpf_map; diff --git a/include/linux/mac/mac_kernel.h b/include/linux/mac/mac_kernel.h index 5c3fdcb..0d7257f 100644 --- a/include/linux/mac/mac_kernel.h +++ b/include/linux/mac/mac_kernel.h @@ -8,68 +8,6 @@ #ifdef __KERNEL__ #include -#ifdef CONFIG_MAC_ - -/* - * Mandatory Access Control (MAC) enabled - */ - -/* kernel notation for identifier of MAC subject */ -typedef mac_subject_id_t kmac_subject_id_t; - -/* kernel notation for identifier of MAC security label */ -typedef mac_label_id_t kmac_label_id_t; - -/* kernel notation for MAC access mode to an object */ -typedef mac_mode_t kmac_mode_t; - -/* kernel notation for identifier of MAC user */ -typedef mac_user_id_t kmac_user_id_t; - -/* kernel notation of network address for MAC network address object */ -typedef mac_addr_t kmac_addr_t; - -/* kernel notation of network port number for MAC network port object */ -typedef mac_port_t kmac_port_t; - -/* kernel notation of network protocol number for MAC network port object */ -typedef mac_proto_t kmac_proto_t; - -/* - * Declaration of interface activation flag variable for MAC. - * The variable is defined in uts/common/os/main.c file. - * If the variable has zero value no kernel's interface function - * for MAC must be called. This must be checked by all code before - * any call to the functions. - */ -extern int mac_active; -extern int mac_attached; -extern unsigned int mac_test_inode; -/* - * The pointer setup to a function for checking MAC rights for user- - * owned object in kernel. - */ -extern int (*kmac_access_userp)(kmac_subject_id_t, kmac_user_id_t, int); - -/* - * The pointer setup to a function for checking MAC rights for network - * address in kernel. - */ -extern int (*kmac_access_addrp)(kmac_subject_id_t, kmac_addr_t, int); - -/* - * The pointer setup to a function for checking MAC rights for network - * port number in kernel. - */ -extern int (*kmac_access_portp)(kmac_subject_id_t, kmac_port_t, - kmac_proto_t, int); - -extern int (*sys_macctl_real)(register int request, register void *data, - register int size); -extern asmlinkage int sys_macctl(register int request, register void *data, - register int size); - -#endif /* CONFIG_MAC_ */ #endif /* _KERNEL */ #endif /* __LINUX_MAC_MAC_KERNEL_H */ diff --git a/include/linux/memstick.h b/include/linux/memstick.h index 216a713..1198ea3 100644 --- a/include/linux/memstick.h +++ b/include/linux/memstick.h @@ -281,6 +281,7 @@ struct memstick_host { struct memstick_dev *card; unsigned int retries; + bool removing; /* Notify the host that some requests are pending. */ void (*request)(struct memstick_host *host); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 8978296..2b65ffb 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -299,6 +299,7 @@ struct mlx5_cmd { struct semaphore sem; struct semaphore pages_sem; int mode; + u16 allowed_opcode; struct mlx5_cmd_work_ent *ent_arr[MLX5_MAX_COMMANDS]; struct dma_pool *pool; struct mlx5_cmd_debug dbg; @@ -769,6 +770,8 @@ struct mlx5_cmd_work_ent { u64 ts2; u16 op; bool polling; + /* Track the max comp handlers */ + refcount_t refcnt; }; struct mlx5_pas { @@ -888,10 +891,15 @@ mlx5_frag_buf_get_idx_last_contig_stride(struct mlx5_frag_buf_ctrl *fbc, u32 ix) return min_t(u32, last_frag_stride_idx - fbc->strides_offset, fbc->sz_m1); } +enum { + CMD_ALLOWED_OPCODE_ALL, +}; + int mlx5_cmd_init(struct mlx5_core_dev *dev); void mlx5_cmd_cleanup(struct mlx5_core_dev *dev); void mlx5_cmd_use_events(struct mlx5_core_dev *dev); void mlx5_cmd_use_polling(struct mlx5_core_dev *dev); +void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode); struct mlx5_async_ctx { struct mlx5_core_dev *dev; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index aba5607..75e5a7f 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1139,6 +1139,11 @@ enum mlx5_fc_bulk_alloc_bitmask { #define MLX5_FC_BULK_NUM_FCS(fc_enum) (MLX5_FC_BULK_SIZE_FACTOR * (fc_enum)) +enum { + MLX5_STEERING_FORMAT_CONNECTX_5 = 0, + MLX5_STEERING_FORMAT_CONNECTX_6DX = 1, +}; + struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_0[0x30]; u8 vhca_id[0x10]; @@ -1419,7 +1424,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 general_obj_types[0x40]; - u8 reserved_at_440[0x20]; + u8 reserved_at_440[0x4]; + u8 steering_format_version[0x4]; + u8 create_qp_start_hint[0x18]; u8 reserved_at_460[0x3]; u8 log_max_uctx[0x5]; diff --git a/include/linux/mm.h b/include/linux/mm.h index 7470fc9..e5ae4fd 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2224,7 +2224,7 @@ static inline void zero_resv_unavail(void) {} extern void set_dma_reserve(unsigned long new_dma_reserve); extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long, - enum memmap_context, struct vmem_altmap *); + enum meminit_context, struct vmem_altmap *); extern void setup_per_zone_wmarks(void); extern int __meminit init_per_zone_wmark_min(void); extern void mem_init(void); @@ -2588,6 +2588,15 @@ static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, return VM_FAULT_NOPAGE; } +#ifndef io_remap_pfn_range +static inline int io_remap_pfn_range(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, + unsigned long size, pgprot_t prot) +{ + return remap_pfn_range(vma, addr, pfn, size, pgprot_decrypted(prot)); +} +#endif + static inline vm_fault_t vmf_error(int err) { if (err == -ENOMEM) diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index e459b38..cf3780a 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -226,7 +226,7 @@ struct mmc_queue_req; * MMC Physical partitions */ struct mmc_part { - unsigned int size; /* partition size (in bytes) */ + u64 size; /* partition size (in bytes) */ unsigned int part_cfg; /* partition type */ char name[MAX_MMC_PART_NAME_LEN]; bool force_ro; /* to make boot parts RO by default */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 85804ba..a90aba3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -822,10 +822,15 @@ bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned int alloc_flags); bool zone_watermark_ok_safe(struct zone *z, unsigned int order, unsigned long mark, int classzone_idx); -enum memmap_context { - MEMMAP_EARLY, - MEMMAP_HOTPLUG, +/* + * Memory initialization context, use to differentiate memory added by + * the platform statically or via memory hotplug interface. + */ +enum meminit_context { + MEMINIT_EARLY, + MEMINIT_HOTPLUG, }; + extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn, unsigned long size); diff --git a/include/linux/mtd/pfow.h b/include/linux/mtd/pfow.h index 122f343..c65d7a3 100644 --- a/include/linux/mtd/pfow.h +++ b/include/linux/mtd/pfow.h @@ -128,7 +128,7 @@ static inline void print_drs_error(unsigned dsr) if (!(dsr & DSR_AVAILABLE)) printk(KERN_NOTICE"DSR.15: (0) Device not Available\n"); - if (prog_status & 0x03) + if ((prog_status & 0x03) == 0x03) printk(KERN_NOTICE"DSR.9,8: (11) Attempt to program invalid " "half with 41h command\n"); else if (prog_status & 0x02) diff --git a/include/linux/net.h b/include/linux/net.h index 9cafb5f..47dd797 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -288,6 +289,21 @@ do { \ #define net_get_random_once_wait(buf, nbytes) \ get_random_once_wait((buf), (nbytes)) +/* + * E.g. XFS meta- & log-data is in slab pages, or bcache meta + * data pages, or other high order pages allocated by + * __get_free_pages() without __GFP_COMP, which have a page_count + * of 0 and/or have PageSlab() set. We cannot use send_page for + * those, as that does get_page(); put_page(); and would cause + * either a VM_BUG directly, or __page_cache_release a page that + * would actually still be referenced by someone, leading to some + * obscure delayed Oops somewhere else. + */ +static inline bool sendpage_ok(struct page *page) +{ + return !PageSlab(page) && page_count(page) >= 1; +} + int kernel_sendmsg(struct socket *sock, struct msghdr *msg, struct kvec *vec, size_t num, size_t len); int kernel_sendmsg_locked(struct sock *sk, struct msghdr *msg, diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index 77ebb61..4c0e653 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -316,7 +316,7 @@ NF_HOOK_LIST(uint8_t pf, unsigned int hook, struct net *net, struct sock *sk, INIT_LIST_HEAD(&sublist); list_for_each_entry_safe(skb, next, head, list) { - list_del(&skb->list); + skb_list_del_init(skb); if (nf_hook(pf, hook, net, sk, skb, in, out, okfn) == 1) list_add_tail(&skb->list, &sublist); } diff --git a/include/linux/netfilter/nf_conntrack_sctp.h b/include/linux/netfilter/nf_conntrack_sctp.h index 9a33f17..625f491 100644 --- a/include/linux/netfilter/nf_conntrack_sctp.h +++ b/include/linux/netfilter/nf_conntrack_sctp.h @@ -9,6 +9,8 @@ struct ip_ct_sctp { enum sctp_conntrack state; __be32 vtag[IP_CT_DIR_MAX]; + u8 last_dir; + u8 flags; }; #endif /* _NF_CONNTRACK_SCTP_H */ diff --git a/include/linux/netfilter/nfnetlink.h b/include/linux/netfilter/nfnetlink.h index 851425c..f6267e2 100644 --- a/include/linux/netfilter/nfnetlink.h +++ b/include/linux/netfilter/nfnetlink.h @@ -24,6 +24,12 @@ struct nfnl_callback { const u_int16_t attr_count; /* number of nlattr's */ }; +enum nfnl_abort_action { + NFNL_ABORT_NONE = 0, + NFNL_ABORT_AUTOLOAD, + NFNL_ABORT_VALIDATE, +}; + struct nfnetlink_subsystem { const char *name; __u8 subsys_id; /* nfnetlink subsystem ID */ @@ -31,7 +37,8 @@ struct nfnetlink_subsystem { const struct nfnl_callback *cb; /* callback for individual types */ struct module *owner; int (*commit)(struct net *net, struct sk_buff *skb); - int (*abort)(struct net *net, struct sk_buff *skb, bool autoload); + int (*abort)(struct net *net, struct sk_buff *skb, + enum nfnl_abort_action action); void (*cleanup)(struct net *net); bool (*valid_genid)(struct net *net, u32 genid); }; @@ -43,8 +50,7 @@ int nfnetlink_has_listeners(struct net *net, unsigned int group); int nfnetlink_send(struct sk_buff *skb, struct net *net, u32 portid, unsigned int group, int echo, gfp_t flags); int nfnetlink_set_err(struct net *net, u32 portid, u32 group, int error); -int nfnetlink_unicast(struct sk_buff *skb, struct net *net, u32 portid, - int flags); +int nfnetlink_unicast(struct sk_buff *skb, struct net *net, u32 portid); static inline u16 nfnl_msg_type(u8 subsys, u8 msg_type) { diff --git a/include/linux/netfilter/x_tables.h b/include/linux/netfilter/x_tables.h index 1b261c5..f5c21b7 100644 --- a/include/linux/netfilter/x_tables.h +++ b/include/linux/netfilter/x_tables.h @@ -227,7 +227,7 @@ struct xt_table { unsigned int valid_hooks; /* Man behind the curtain... */ - struct xt_table_info *private; + struct xt_table_info __rcu *private; /* Set this to THIS_MODULE if you are a module, otherwise NULL */ struct module *me; @@ -448,6 +448,9 @@ xt_get_per_cpu_counter(struct xt_counters *cnt, unsigned int cpu) struct nf_hook_ops *xt_hook_ops_alloc(const struct xt_table *, nf_hookfn *); +struct xt_table_info +*xt_table_get_private_protected(const struct xt_table *table); + #ifdef CONFIG_COMPAT #include diff --git a/include/linux/netfilter_ipv4.h b/include/linux/netfilter_ipv4.h index 082e2c4..5b70ca8 100644 --- a/include/linux/netfilter_ipv4.h +++ b/include/linux/netfilter_ipv4.h @@ -16,7 +16,7 @@ struct ip_rt_info { u_int32_t mark; }; -int ip_route_me_harder(struct net *net, struct sk_buff *skb, unsigned addr_type); +int ip_route_me_harder(struct net *net, struct sock *sk, struct sk_buff *skb, unsigned addr_type); struct nf_queue_entry; diff --git a/include/linux/netfilter_ipv6.h b/include/linux/netfilter_ipv6.h index aac42c2..48314ad 100644 --- a/include/linux/netfilter_ipv6.h +++ b/include/linux/netfilter_ipv6.h @@ -42,7 +42,7 @@ struct nf_ipv6_ops { #if IS_MODULE(CONFIG_IPV6) int (*chk_addr)(struct net *net, const struct in6_addr *addr, const struct net_device *dev, int strict); - int (*route_me_harder)(struct net *net, struct sk_buff *skb); + int (*route_me_harder)(struct net *net, struct sock *sk, struct sk_buff *skb); int (*dev_get_saddr)(struct net *net, const struct net_device *dev, const struct in6_addr *daddr, unsigned int srcprefs, struct in6_addr *saddr); @@ -58,7 +58,6 @@ struct nf_ipv6_ops { int (*output)(struct net *, struct sock *, struct sk_buff *)); int (*reroute)(struct sk_buff *skb, const struct nf_queue_entry *entry); #if IS_MODULE(CONFIG_IPV6) - int (*br_defrag)(struct net *net, struct sk_buff *skb, u32 user); int (*br_fragment)(struct net *net, struct sock *sk, struct sk_buff *skb, struct nf_bridge_frag_data *data, @@ -117,23 +116,6 @@ static inline int nf_ip6_route(struct net *net, struct dst_entry **dst, #include -static inline int nf_ipv6_br_defrag(struct net *net, struct sk_buff *skb, - u32 user) -{ -#if IS_MODULE(CONFIG_IPV6) - const struct nf_ipv6_ops *v6_ops = nf_get_ipv6_ops(); - - if (!v6_ops) - return 1; - - return v6_ops->br_defrag(net, skb, user); -#elif IS_BUILTIN(CONFIG_IPV6) - return nf_ct_frag6_gather(net, skb, user); -#else - return 1; -#endif -} - int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, struct nf_bridge_frag_data *data, int (*output)(struct net *, struct sock *sk, @@ -161,9 +143,9 @@ static inline int nf_br_ip6_fragment(struct net *net, struct sock *sk, #endif } -int ip6_route_me_harder(struct net *net, struct sk_buff *skb); +int ip6_route_me_harder(struct net *net, struct sock *sk, struct sk_buff *skb); -static inline int nf_ip6_route_me_harder(struct net *net, struct sk_buff *skb) +static inline int nf_ip6_route_me_harder(struct net *net, struct sock *sk, struct sk_buff *skb) { #if IS_MODULE(CONFIG_IPV6) const struct nf_ipv6_ops *v6_ops = nf_get_ipv6_ops(); @@ -171,9 +153,9 @@ static inline int nf_ip6_route_me_harder(struct net *net, struct sk_buff *skb) if (!v6_ops) return -EHOSTUNREACH; - return v6_ops->route_me_harder(net, skb); + return v6_ops->route_me_harder(net, sk, skb); #elif IS_BUILTIN(CONFIG_IPV6) - return ip6_route_me_harder(net, skb); + return ip6_route_me_harder(net, sk, skb); #else return -EHOSTUNREACH; #endif diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index 0bbd587..7e9419d 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -142,6 +142,8 @@ extern void nfs_unlock_and_release_request(struct nfs_page *); extern int nfs_page_group_lock(struct nfs_page *); extern void nfs_page_group_unlock(struct nfs_page *); extern bool nfs_page_group_sync_on_bit(struct nfs_page *, unsigned int); +extern int nfs_page_set_headlock(struct nfs_page *req); +extern void nfs_page_clear_headlock(struct nfs_page *req); extern bool nfs_async_iocounter_wait(struct rpc_task *, struct nfs_lock_context *); /* diff --git a/include/linux/node.h b/include/linux/node.h index 4866f32..014ba3a 100644 --- a/include/linux/node.h +++ b/include/linux/node.h @@ -99,11 +99,13 @@ extern struct node *node_devices[]; typedef void (*node_registration_func_t)(struct node *); #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_NUMA) -extern int link_mem_sections(int nid, unsigned long start_pfn, - unsigned long end_pfn); +int link_mem_sections(int nid, unsigned long start_pfn, + unsigned long end_pfn, + enum meminit_context context); #else static inline int link_mem_sections(int nid, unsigned long start_pfn, - unsigned long end_pfn) + unsigned long end_pfn, + enum meminit_context context) { return 0; } @@ -128,7 +130,8 @@ static inline int register_one_node(int nid) if (error) return error; /* link memory sections under this node */ - error = link_mem_sections(nid, start_pfn, end_pfn); + error = link_mem_sections(nid, start_pfn, end_pfn, + MEMINIT_EARLY); } return error; diff --git a/include/linux/of.h b/include/linux/of.h index 28f7005..3fcbb53 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -1297,6 +1297,7 @@ static inline int of_get_available_child_count(const struct device_node *np) #define _OF_DECLARE(table, name, compat, fn, fn_type) \ static const struct of_device_id __of_table_##name \ __used __section(__##table##_of_table) \ + __aligned(__alignof__(struct of_device_id)) \ = { .compatible = compat, \ .data = (fn == (fn_type)NULL) ? fn : fn } #else diff --git a/include/linux/oom.h b/include/linux/oom.h index 0dc25c8..956dc52 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -55,6 +55,7 @@ struct oom_control { }; extern struct mutex oom_lock; +extern struct mutex oom_adj_mutex; static inline void set_current_oom_origin(void) { diff --git a/include/linux/overflow.h b/include/linux/overflow.h index 6590450..50c93ca 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -3,6 +3,7 @@ #define __LINUX_OVERFLOW_H #include +#include /* * In the fallback code below, we need to compute the minimum and diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h index 8679ccd..3468794 100644 --- a/include/linux/page_owner.h +++ b/include/linux/page_owner.h @@ -11,7 +11,7 @@ extern struct page_ext_operations page_owner_ops; extern void __reset_page_owner(struct page *page, unsigned int order); extern void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask); -extern void __split_page_owner(struct page *page, unsigned int order); +extern void __split_page_owner(struct page *page, unsigned int nr); extern void __copy_page_owner(struct page *oldpage, struct page *newpage); extern void __set_page_owner_migrate_reason(struct page *page, int reason); extern void __dump_page_owner(struct page *page); @@ -31,10 +31,10 @@ static inline void set_page_owner(struct page *page, __set_page_owner(page, order, gfp_mask); } -static inline void split_page_owner(struct page *page, unsigned int order) +static inline void split_page_owner(struct page *page, unsigned int nr) { if (static_branch_unlikely(&page_owner_inited)) - __split_page_owner(page, order); + __split_page_owner(page, nr); } static inline void copy_page_owner(struct page *oldpage, struct page *newpage) { diff --git a/include/linux/pci.h b/include/linux/pci.h index 94df131..663a3b0 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -427,6 +427,7 @@ struct pci_dev { unsigned int is_probed:1; /* Device probing in progress */ unsigned int link_active_reporting:1;/* Device capable of reporting link active */ unsigned int no_vf_scan:1; /* Don't scan for VFs after IOV enablement */ + unsigned int no_command_memory:1; /* No PCI_COMMAND_MEMORY */ pci_dev_flags_t dev_flags; atomic_t enable_cnt; /* pci_enable_device has been called */ @@ -1226,7 +1227,6 @@ int pci_enable_rom(struct pci_dev *pdev); void pci_disable_rom(struct pci_dev *pdev); void __iomem __must_check *pci_map_rom(struct pci_dev *pdev, size_t *size); void pci_unmap_rom(struct pci_dev *pdev, void __iomem *rom); -void __iomem __must_check *pci_platform_rom(struct pci_dev *pdev, size_t *size); /* Power management related routines */ int pci_save_state(struct pci_dev *dev); diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index b2ef2c5..057d7dc 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -3146,6 +3146,7 @@ #define PCI_DEVICE_ID_MCST_KMNG 0x8006 #define PCI_DEVICE_ID_MCST_PS2 0x8004 #define PCI_DEVICE_ID_MCST_GPIO 0x8005 + #define PCI_DEVICE_ID_MCST_OHCI 0x8008 #define PCI_DEVICE_ID_MCST_MGA2 0x8009 #define PCI_DEVICE_ID_MCST_HDA 0x800a @@ -3166,7 +3167,6 @@ #define PCI_DEVICE_ID_MCST_EHCI 0x801e #define PCI_DEVICE_ID_MCST_MGEX 0x8022 -#define PCI_VENDOR_ID_MCST 0x1fff /* MCST EIOHUB */ #define PCI_DEVICE_ID_MCST_GPIO_MPV_EIOH 0x8025 @@ -3175,14 +3175,18 @@ #define PCI_DEVICE_ID_MCST_I2C_SPI_EPIC 0x8028 #define PCI_DEVICE_ID_MCST_USB_3_0 0x8029 #define PCI_DEVICE_ID_MCST_3D_IMAGINATION_GX6650 0x802a +#define PCI_DEVICE_ID_IMAGINATION_VXE 0x802b +#define PCI_DEVICE_ID_IMAGINATION_VXD 0x802c #define PCI_DEVICE_ID_MCST_VP9_BIGEV2 0x802d #define PCI_DEVICE_ID_MCST_VP9_G2 0x802e -#define PCI_DEVICE_ID_MCST_VP9_G2_R2000P 0x803c #define PCI_DEVICE_ID_MCST_SERIAL 0x802f #define PCI_DEVICE_ID_MCST_MGA25 0x8031 #define PCI_DEVICE_ID_MCST_PCIE_X16 0x8032 #define PCI_DEVICE_ID_MCST_PCIE_X4 0x8033 +#define PCI_DEVICE_ID_MCST_3D_VIVANTE_R2000P 0x8039 #define PCI_DEVICE_ID_MCST_MGA26 0x803a +#define PCI_DEVICE_ID_MCST_VP9_BIGEV2_R2000P 0x803b +#define PCI_DEVICE_ID_MCST_VP9_G2_R2000P 0x803c #endif diff --git a/include/linux/platform_data/dma-dw.h b/include/linux/platform_data/dma-dw.h index f3eaf9e..70078be 100644 --- a/include/linux/platform_data/dma-dw.h +++ b/include/linux/platform_data/dma-dw.h @@ -21,6 +21,7 @@ * @dst_id: dst request line * @m_master: memory master for transfers on allocated channel * @p_master: peripheral master for transfers on allocated channel + * @channels: mask of the channels permitted for allocation (zero value means any) * @hs_polarity:set active low polarity of handshake interface */ struct dw_dma_slave { @@ -29,6 +30,7 @@ struct dw_dma_slave { u8 dst_id; u8 m_master; u8 p_master; + u8 channels; bool hs_polarity; }; diff --git a/include/linux/pm.h b/include/linux/pm.h index 4c441be..c1d21e9 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -598,7 +598,7 @@ struct dev_pm_info { #endif #ifdef CONFIG_PM struct hrtimer suspend_timer; - unsigned long timer_expires; + u64 timer_expires; struct work_struct work; wait_queue_head_t wait_queue; struct wake_irq *wakeirq; diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 22af69d..7145795 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -54,11 +54,10 @@ extern u64 pm_runtime_autosuspend_expiration(struct device *dev); extern void pm_runtime_update_max_time_suspended(struct device *dev, s64 delta_ns); extern void pm_runtime_set_memalloc_noio(struct device *dev, bool enable); -extern void pm_runtime_clean_up_links(struct device *dev); extern void pm_runtime_get_suppliers(struct device *dev); extern void pm_runtime_put_suppliers(struct device *dev); extern void pm_runtime_new_link(struct device *dev); -extern void pm_runtime_drop_link(struct device *dev); +extern void pm_runtime_drop_link(struct device_link *link); static inline void pm_suspend_ignore_children(struct device *dev, bool enable) { @@ -173,11 +172,10 @@ static inline u64 pm_runtime_autosuspend_expiration( struct device *dev) { return 0; } static inline void pm_runtime_set_memalloc_noio(struct device *dev, bool enable){} -static inline void pm_runtime_clean_up_links(struct device *dev) {} static inline void pm_runtime_get_suppliers(struct device *dev) {} static inline void pm_runtime_put_suppliers(struct device *dev) {} static inline void pm_runtime_new_link(struct device *dev) {} -static inline void pm_runtime_drop_link(struct device *dev) {} +static inline void pm_runtime_drop_link(struct device_link *link) {} #endif /* !CONFIG_PM */ @@ -226,6 +224,27 @@ static inline int pm_runtime_get_sync(struct device *dev) return __pm_runtime_resume(dev, RPM_GET_PUT); } +/** + * pm_runtime_resume_and_get - Bump up usage counter of a device and resume it. + * @dev: Target device. + * + * Resume @dev synchronously and if that is successful, increment its runtime + * PM usage counter. Return 0 if the runtime PM usage counter of @dev has been + * incremented or a negative error code otherwise. + */ +static inline int pm_runtime_resume_and_get(struct device *dev) +{ + int ret; + + ret = __pm_runtime_resume(dev, RPM_GET_PUT); + if (ret < 0) { + pm_runtime_put_noidle(dev); + return ret; + } + + return 0; +} + static inline int pm_runtime_put(struct device *dev) { return __pm_runtime_idle(dev, RPM_GET_PUT | RPM_ASYNC); diff --git a/include/linux/prandom.h b/include/linux/prandom.h index aa16e64..cc1e713 100644 --- a/include/linux/prandom.h +++ b/include/linux/prandom.h @@ -16,12 +16,44 @@ void prandom_bytes(void *buf, size_t nbytes); void prandom_seed(u32 seed); void prandom_reseed_late(void); +#if BITS_PER_LONG == 64 +/* + * The core SipHash round function. Each line can be executed in + * parallel given enough CPU resources. + */ +#define PRND_SIPROUND(v0, v1, v2, v3) ( \ + v0 += v1, v1 = rol64(v1, 13), v2 += v3, v3 = rol64(v3, 16), \ + v1 ^= v0, v0 = rol64(v0, 32), v3 ^= v2, \ + v0 += v3, v3 = rol64(v3, 21), v2 += v1, v1 = rol64(v1, 17), \ + v3 ^= v0, v1 ^= v2, v2 = rol64(v2, 32) \ +) + +#define PRND_K0 (0x736f6d6570736575 ^ 0x6c7967656e657261) +#define PRND_K1 (0x646f72616e646f6d ^ 0x7465646279746573) + +#elif BITS_PER_LONG == 32 +/* + * On 32-bit machines, we use HSipHash, a reduced-width version of SipHash. + * This is weaker, but 32-bit machines are not used for high-traffic + * applications, so there is less output for an attacker to analyze. + */ +#define PRND_SIPROUND(v0, v1, v2, v3) ( \ + v0 += v1, v1 = rol32(v1, 5), v2 += v3, v3 = rol32(v3, 8), \ + v1 ^= v0, v0 = rol32(v0, 16), v3 ^= v2, \ + v0 += v3, v3 = rol32(v3, 7), v2 += v1, v1 = rol32(v1, 13), \ + v3 ^= v0, v1 ^= v2, v2 = rol32(v2, 16) \ +) +#define PRND_K0 0x6c796765 +#define PRND_K1 0x74656462 + +#else +#error Unsupported BITS_PER_LONG +#endif + struct rnd_state { __u32 s1, s2, s3, s4; }; -DECLARE_PER_CPU(struct rnd_state, net_rand_state); - u32 prandom_u32_state(struct rnd_state *state); void prandom_bytes_state(struct rnd_state *state, void *buf, size_t nbytes); void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state); diff --git a/include/linux/prefetch.h b/include/linux/prefetch.h index 13eafeb..b83a3f9 100644 --- a/include/linux/prefetch.h +++ b/include/linux/prefetch.h @@ -15,6 +15,7 @@ #include #include +struct page; /* prefetch(x) attempts to pre-emptively get the memory pointed to by address "x" into the CPU L1 cache. @@ -62,4 +63,11 @@ static inline void prefetch_range(void *addr, size_t len) #endif } +static inline void prefetch_page_address(struct page *page) +{ +#if defined(WANT_PAGE_VIRTUAL) || defined(HASHED_PAGE_VIRTUAL) + prefetch(page); +#endif +} + #endif diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index a705aa2..865d02c 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -21,6 +21,7 @@ extern void proc_flush_task(struct task_struct *); extern struct proc_dir_entry *proc_symlink(const char *, struct proc_dir_entry *, const char *); +struct proc_dir_entry *_proc_mkdir(const char *, umode_t, struct proc_dir_entry *, void *, bool); extern struct proc_dir_entry *proc_mkdir(const char *, struct proc_dir_entry *); extern struct proc_dir_entry *proc_mkdir_data(const char *, umode_t, struct proc_dir_entry *, void *); @@ -99,6 +100,11 @@ static inline struct proc_dir_entry *proc_symlink(const char *name, static inline struct proc_dir_entry *proc_mkdir(const char *name, struct proc_dir_entry *parent) {return NULL;} static inline struct proc_dir_entry *proc_create_mount_point(const char *name) { return NULL; } +static inline struct proc_dir_entry *_proc_mkdir(const char *name, umode_t mode, + struct proc_dir_entry *parent, void *data, bool force_lookup) +{ + return NULL; +} static inline struct proc_dir_entry *proc_mkdir_data(const char *name, umode_t mode, struct proc_dir_entry *parent, void *data) { return NULL; } static inline struct proc_dir_entry *proc_mkdir_mode(const char *name, @@ -136,7 +142,7 @@ struct net; static inline struct proc_dir_entry *proc_net_mkdir( struct net *net, const char *name, struct proc_dir_entry *parent) { - return proc_mkdir_data(name, 0, parent, net); + return _proc_mkdir(name, 0, parent, net, true); } struct ns_common; diff --git a/include/linux/qcom-geni-se.h b/include/linux/qcom-geni-se.h index dd46494..5b90eff 100644 --- a/include/linux/qcom-geni-se.h +++ b/include/linux/qcom-geni-se.h @@ -229,6 +229,9 @@ struct geni_se { #define GENI_SE_VERSION_MINOR(ver) ((ver & HW_VER_MINOR_MASK) >> HW_VER_MINOR_SHFT) #define GENI_SE_VERSION_STEP(ver) (ver & HW_VER_STEP_MASK) +/* QUP SE VERSION value for major number 2 and minor number 5 */ +#define QUP_SE_VERSION_2_5 0x20050000 + #if IS_ENABLED(CONFIG_QCOM_GENI_SE) u32 geni_se_get_qup_hw_version(struct geni_se *se); diff --git a/include/linux/qed/qed_if.h b/include/linux/qed/qed_if.h index b5db1ee..65a7355 100644 --- a/include/linux/qed/qed_if.h +++ b/include/linux/qed/qed_if.h @@ -637,6 +637,7 @@ struct qed_dev_info { #define QED_MFW_VERSION_3_OFFSET 24 u32 flash_size; + bool b_arfs_capable; bool b_inter_pf_switch; bool tx_switching; bool rdma_supported; diff --git a/include/linux/rwsem-rt.h b/include/linux/rwsem-rt.h index 2018ff7..3fb092b 100644 --- a/include/linux/rwsem-rt.h +++ b/include/linux/rwsem-rt.h @@ -56,6 +56,7 @@ static inline int rwsem_is_contended(struct rw_semaphore *sem) } extern void __down_read(struct rw_semaphore *sem); +extern int __down_read_interruptible(struct rw_semaphore *sem); extern int __down_read_killable(struct rw_semaphore *sem); extern int __down_read_trylock(struct rw_semaphore *sem); extern void __down_write(struct rw_semaphore *sem); diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index 393f452..d169433 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -131,6 +131,7 @@ static inline int rwsem_is_contended(struct rw_semaphore *sem) * lock for reading */ extern void down_read(struct rw_semaphore *sem); +extern int __must_check down_read_interruptible(struct rw_semaphore *sem); extern int __must_check down_read_killable(struct rw_semaphore *sem); /* @@ -179,6 +180,7 @@ extern void downgrade_write(struct rw_semaphore *sem); * See Documentation/locking/lockdep-design.rst for more details.) */ extern void down_read_nested(struct rw_semaphore *sem, int subclass); +extern int __must_check down_read_killable_nested(struct rw_semaphore *sem, int subclass); extern void down_write_nested(struct rw_semaphore *sem, int subclass); extern int down_write_killable_nested(struct rw_semaphore *sem, int subclass); extern void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest_lock); @@ -199,6 +201,7 @@ extern void down_read_non_owner(struct rw_semaphore *sem); extern void up_read_non_owner(struct rw_semaphore *sem); #else # define down_read_nested(sem, subclass) down_read(sem) +# define down_read_killable_nested(sem, subclass) down_read_killable(sem) # define down_write_nest_lock(sem, nest_lock) down_write(sem) # define down_write_nested(sem, subclass) down_write(sem) # define down_write_killable_nested(sem, subclass) down_write_killable(sem) diff --git a/include/linux/sched/coredump.h b/include/linux/sched/coredump.h index ecdc654..dfd82ea 100644 --- a/include/linux/sched/coredump.h +++ b/include/linux/sched/coredump.h @@ -72,6 +72,7 @@ static inline int get_dumpable(struct mm_struct *mm) #define MMF_DISABLE_THP 24 /* disable THP for all VMAs */ #define MMF_OOM_VICTIM 25 /* mm is the oom victim */ #define MMF_OOM_REAP_QUEUED 26 /* mm was queued for oom_reaper */ +#define MMF_MULTIPROCESS 27 /* mm is shared between processes */ #define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 8805025..baf58f4 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -224,7 +224,15 @@ struct signal_struct { struct mutex cred_guard_mutex; /* guard against foreign influences on * credential calculations - * (notably. ptrace) */ + * (notably. ptrace) + * Deprecated do not use in new code. + * Use exec_update_lock instead. + */ + struct rw_semaphore exec_update_lock; /* Held while task_struct is + * being updated during exec, + * and may have inconsistent + * permissions. + */ } __randomize_layout; /* diff --git a/include/linux/security.h b/include/linux/security.h index 9bec46c..df90399 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -445,13 +445,6 @@ int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen); int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen); int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen); int security_locked_down(enum lockdown_reason what); -#ifdef CONFIG_MCST_SECURITY_ELMAC -#ifdef CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES -int security_inode_pre_permission(struct inode *inode, int mask); -int security_iterate_dir(struct file *file, struct dir_context *ctx, - bool shared); -#endif /* CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES */ -#endif /* CONFIG_MCST_SECURITY_ELMAC */ #else /* CONFIG_SECURITY */ static inline int call_blocking_lsm_notifier(enum lsm_event event, void *data) @@ -859,7 +852,7 @@ static inline int security_inode_killpriv(struct dentry *dentry) static inline int security_inode_getsecurity(struct inode *inode, const char *name, void **buffer, bool alloc) { - return -EOPNOTSUPP; + return cap_inode_getsecurity(inode, name, buffer, alloc); } static inline int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -1279,26 +1272,6 @@ static inline int security_locked_down(enum lockdown_reason what) { return 0; } -#ifdef CONFIG_MCST_SECURITY_ELMAC -#ifdef CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES -static inline int security_inode_pre_permission(struct inode *inode, int mask) -{ - return 0; -} -int security_iterate_dir(struct file *file, struct dir_context *ctx, - bool shared) -{ - int res; - - if (shared) - res = file->f_op->iterate_shared(file, ctx); - else - res = file->f_op->iterate(file, ctx); - - return res; -} -#endif /* CONFIG_MCST_SECURITY_ELMAC_HIDING_FILES */ -#endif /* CONFIG_MCST_SECURITY_ELMAC */ #endif /* CONFIG_SECURITY */ #ifdef CONFIG_SECURITY_NETWORK diff --git a/include/linux/seq_buf.h b/include/linux/seq_buf.h index aa5deb0..7cc9522 100644 --- a/include/linux/seq_buf.h +++ b/include/linux/seq_buf.h @@ -30,7 +30,7 @@ static inline void seq_buf_clear(struct seq_buf *s) } static inline void -seq_buf_init(struct seq_buf *s, unsigned char *buf, unsigned int size) +seq_buf_init(struct seq_buf *s, char *buf, unsigned int size) { s->buffer = buf; s->size = size; diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index bc458c2..e520789 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -253,6 +253,13 @@ static inline void raw_write_seqcount_end(seqcount_t *s) * usual consistency guarantee. It is one wmb cheaper, because we can * collapse the two back-to-back wmb()s. * + * Note that, writes surrounding the barrier should be declared atomic (e.g. + * via WRITE_ONCE): a) to ensure the writes become visible to other threads + * atomically, avoiding compiler optimizations; b) to document which writes are + * meant to propagate to the reader critical section. This is necessary because + * neither writes before and after the barrier are enclosed in a seq-writer + * critical section that would ensure readers are aware of ongoing writes. + * * seqcount_t seq; * bool X = true, Y = false; * @@ -272,11 +279,11 @@ static inline void raw_write_seqcount_end(seqcount_t *s) * * void write(void) * { - * Y = true; + * WRITE_ONCE(Y, true); * * raw_write_seqcount_barrier(seq); * - * X = false; + * WRITE_ONCE(X, false); * } */ static inline void raw_write_seqcount_barrier(seqcount_t *s) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 0581195..f6dcd58 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1817,6 +1817,18 @@ static inline __u32 skb_queue_len(const struct sk_buff_head *list_) return list_->qlen; } +/** + * skb_queue_len_lockless - get queue length + * @list_: list to measure + * + * Return the length of an &sk_buff queue. + * This variant can be used in lockless contexts. + */ +static inline __u32 skb_queue_len_lockless(const struct sk_buff_head *list_) +{ + return READ_ONCE(list_->qlen); +} + /** * __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head * @list: queue to initialize @@ -2028,7 +2040,7 @@ static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) { struct sk_buff *next, *prev; - list->qlen--; + WRITE_ONCE(list->qlen, list->qlen - 1); next = skb->next; prev = skb->prev; skb->next = skb->prev = NULL; @@ -3192,8 +3204,9 @@ static inline int skb_padto(struct sk_buff *skb, unsigned int len) * is untouched. Otherwise it is extended. Returns zero on * success. The skb is freed on error if @free_on_error is true. */ -static inline int __skb_put_padto(struct sk_buff *skb, unsigned int len, - bool free_on_error) +static inline int __must_check __skb_put_padto(struct sk_buff *skb, + unsigned int len, + bool free_on_error) { unsigned int size = skb->len; @@ -3216,7 +3229,7 @@ static inline int __skb_put_padto(struct sk_buff *skb, unsigned int len, * is untouched. Otherwise it is extended. Returns zero on * success. The skb is freed on error. */ -static inline int skb_put_padto(struct sk_buff *skb, unsigned int len) +static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len) { return __skb_put_padto(skb, len, true); } diff --git a/include/linux/slab.h b/include/linux/slab.h index 97fa1f0..db8363d 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -148,9 +148,6 @@ struct mem_cgroup; void __init kmem_cache_init(void); bool slab_is_available(void); -#ifdef CONFIG_MAC_ -typedef struct kmem_cache kmem_cache_t; -#endif extern bool usercopy_fallback; diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index af4f265..acd9130 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -663,6 +663,25 @@ static inline struct spi_controller *spi_alloc_slave(struct device *host, return __spi_alloc_controller(host, size, true); } +struct spi_controller *__devm_spi_alloc_controller(struct device *dev, + unsigned int size, + bool slave); + +static inline struct spi_controller *devm_spi_alloc_master(struct device *dev, + unsigned int size) +{ + return __devm_spi_alloc_controller(dev, size, false); +} + +static inline struct spi_controller *devm_spi_alloc_slave(struct device *dev, + unsigned int size) +{ + if (!IS_ENABLED(CONFIG_SPI_SLAVE)) + return NULL; + + return __devm_spi_alloc_controller(dev, size, true); +} + extern int spi_register_controller(struct spi_controller *ctlr); extern int devm_spi_register_controller(struct device *dev, struct spi_controller *ctlr); diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 26f282e..77589ed 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -154,9 +154,8 @@ struct svc_rdma_send_ctxt { }; /* svc_rdma_backchannel.c */ -extern int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, - __be32 *rdma_resp, - struct xdr_buf *rcvbuf); +extern void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp, + struct svc_rdma_recv_ctxt *rctxt); /* svc_rdma_recvfrom.c */ extern void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma); diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h index d783e15..d7ef5b9 100644 --- a/include/linux/sunrpc/xprt.h +++ b/include/linux/sunrpc/xprt.h @@ -330,6 +330,7 @@ struct xprt_class { struct rpc_xprt * (*setup)(struct xprt_create *); struct module *owner; char name[32]; + const char * netid[]; }; /* diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index e1fc19a..3e9bae5 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -5,6 +5,7 @@ #include #include #include +#include struct device; struct page; diff --git a/include/linux/time64.h b/include/linux/time64.h index 1912548..5eab3f2 100644 --- a/include/linux/time64.h +++ b/include/linux/time64.h @@ -132,6 +132,10 @@ static inline bool timespec64_valid_settod(const struct timespec64 *ts) */ static inline s64 timespec64_to_ns(const struct timespec64 *ts) { + /* Prevent multiplication overflow */ + if ((unsigned long long)ts->tv_sec >= KTIME_SEC_MAX) + return KTIME_MAX; + return ((s64) ts->tv_sec * NSEC_PER_SEC) + ts->tv_nsec; } diff --git a/include/linux/tpm.h b/include/linux/tpm.h index 53c0ea9..77fdc98 100644 --- a/include/linux/tpm.h +++ b/include/linux/tpm.h @@ -93,6 +93,7 @@ struct tpm_space { u8 *context_buf; u32 session_tbl[3]; u8 *session_buf; + u32 buf_size; }; struct tpm_bios_log { diff --git a/include/linux/tpm_eventlog.h b/include/linux/tpm_eventlog.h index eccfd3a..f3caeeb 100644 --- a/include/linux/tpm_eventlog.h +++ b/include/linux/tpm_eventlog.h @@ -211,9 +211,16 @@ static inline int __calc_tpm2_event_size(struct tcg_pcr_event2_head *event, efispecid = (struct tcg_efi_specid_event_head *)event_header->event; - /* Check if event is malformed. */ + /* + * Perform validation of the event in order to identify malformed + * events. This function may be asked to parse arbitrary byte sequences + * immediately following a valid event log. The caller expects this + * function to recognize that the byte sequence is not a valid event + * and to return an event size of 0. + */ if (memcmp(efispecid->signature, TCG_SPECID_SIG, - sizeof(TCG_SPECID_SIG)) || count > efispecid->num_algs) { + sizeof(TCG_SPECID_SIG)) || + !efispecid->num_algs || count != efispecid->num_algs) { size = 0; goto out; } diff --git a/include/linux/trace_seq.h b/include/linux/trace_seq.h index 6609b39..6db2574 100644 --- a/include/linux/trace_seq.h +++ b/include/linux/trace_seq.h @@ -12,7 +12,7 @@ */ struct trace_seq { - unsigned char buffer[PAGE_SIZE]; + char buffer[PAGE_SIZE]; struct seq_buf seq; int full; }; @@ -51,7 +51,7 @@ static inline int trace_seq_used(struct trace_seq *s) * that is about to be written to and then return the result * of that write. */ -static inline unsigned char * +static inline char * trace_seq_buffer_ptr(struct trace_seq *s) { return s->buffer + seq_buf_used(&s->seq); diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 1fb11da..57ce5af 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -362,7 +362,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) static const char *___tp_str __tracepoint_string = str; \ ___tp_str; \ }) -#define __tracepoint_string __attribute__((section("__tracepoint_str"))) +#define __tracepoint_string __attribute__((section("__tracepoint_str"), used)) #else /* * tracepoint_string() is used to save the string address for userspace diff --git a/include/linux/tty.h b/include/linux/tty.h index a99e9b8..eb33d94 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -306,6 +306,10 @@ struct tty_struct { struct termiox *termiox; /* May be NULL for unsupported */ char name[64]; struct pid *pgrp; /* Protected by ctrl lock */ + /* + * Writes protected by both ctrl lock and legacy mutex, readers must use + * at least one of them. + */ struct pid *session; unsigned long flags; int count; diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index c2887ae..10ff4c3 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -253,9 +253,6 @@ struct hc_driver { /* irq handler */ irqreturn_t (*irq) (struct usb_hcd *hcd); -#ifdef CONFIG_USB_IRQ_ON_THREAD - irqreturn_t (*preirq) (struct usb_hcd *hcd); -#endif int flags; #define HCD_MEMORY 0x0001 /* HC regs use memory (else I/O) */ #define HCD_DMA 0x0002 /* HC uses DMA */ diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index 145c38e..6655ce3 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -442,6 +442,7 @@ static inline unsigned int rdo_max_power(u32 rdo) #define PD_T_ERROR_RECOVERY 100 /* minimum 25 is insufficient */ #define PD_T_SRCSWAPSTDBY 625 /* Maximum of 650ms */ #define PD_T_NEWSRC 250 /* Maximum of 275ms */ +#define PD_T_SWAP_SRC_START 20 /* Minimum of 20ms */ #define PD_T_DRP_TRY 100 /* 75 - 150 ms */ #define PD_T_DRP_TRYWAIT 600 /* 400 - 800 ms */ diff --git a/include/linux/usb_usual.h b/include/linux/usb_usual.h index 000a595..a7f7ebd 100644 --- a/include/linux/usb_usual.h +++ b/include/linux/usb_usual.h @@ -84,6 +84,8 @@ /* Cannot handle REPORT_LUNS */ \ US_FLAG(ALWAYS_SYNC, 0x20000000) \ /* lies about caching, so always sync */ \ + US_FLAG(NO_SAME, 0x40000000) \ + /* Cannot handle WRITE_SAME */ \ #define US_FLAG(name, value) US_FL_##name = value , enum { US_DO_ALL_FLAGS }; diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h index 07875cc..b139f76 100644 --- a/include/linux/virtio_vsock.h +++ b/include/linux/virtio_vsock.h @@ -150,7 +150,8 @@ virtio_transport_dgram_enqueue(struct vsock_sock *vsk, void virtio_transport_destruct(struct vsock_sock *vsk); -void virtio_transport_recv_pkt(struct virtio_vsock_pkt *pkt); +void virtio_transport_recv_pkt(struct virtio_transport *t, + struct virtio_vsock_pkt *pkt); void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt); void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt); u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 wanted); diff --git a/include/linux/zsmalloc.h b/include/linux/zsmalloc.h index 2219cce..4807ca4 100644 --- a/include/linux/zsmalloc.h +++ b/include/linux/zsmalloc.h @@ -20,7 +20,6 @@ * zsmalloc mapping modes * * NOTE: These only make a difference when a mapped object spans pages. - * They also have no effect when PGTABLE_MAPPING is selected. */ enum zs_mapmode { ZS_MM_RW, /* normal read-write mapping */ diff --git a/include/media/v4l2-mediabus.h b/include/media/v4l2-mediabus.h index 45f88f0..c072ed1 100644 --- a/include/media/v4l2-mediabus.h +++ b/include/media/v4l2-mediabus.h @@ -78,6 +78,7 @@ * @V4L2_MBUS_CCP2: CCP2 (Compact Camera Port 2) * @V4L2_MBUS_CSI2_DPHY: MIPI CSI-2 serial interface, with D-PHY * @V4L2_MBUS_CSI2_CPHY: MIPI CSI-2 serial interface, with C-PHY + * @V4L2_MBUS_INVALID: invalid bus type (keep as last) */ enum v4l2_mbus_type { V4L2_MBUS_UNKNOWN, @@ -87,6 +88,7 @@ enum v4l2_mbus_type { V4L2_MBUS_CCP2, V4L2_MBUS_CSI2_DPHY, V4L2_MBUS_CSI2_CPHY, + V4L2_MBUS_INVALID, }; /** diff --git a/include/net/act_api.h b/include/net/act_api.h index 59d05fe..05b568b 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -156,8 +156,6 @@ int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index); int tcf_idr_create(struct tc_action_net *tn, u32 index, struct nlattr *est, struct tc_action **a, const struct tc_action_ops *ops, int bind, bool cpustats); -void tcf_idr_insert(struct tc_action_net *tn, struct tc_action *a); - void tcf_idr_cleanup(struct tc_action_net *tn, u32 index); int tcf_idr_check_alloc(struct tc_action_net *tn, u32 *index, struct tc_action **a, int bind); diff --git a/include/net/af_rxrpc.h b/include/net/af_rxrpc.h index ab98894..55b980b 100644 --- a/include/net/af_rxrpc.h +++ b/include/net/af_rxrpc.h @@ -59,7 +59,7 @@ bool rxrpc_kernel_abort_call(struct socket *, struct rxrpc_call *, void rxrpc_kernel_end_call(struct socket *, struct rxrpc_call *); void rxrpc_kernel_get_peer(struct socket *, struct rxrpc_call *, struct sockaddr_rxrpc *); -u32 rxrpc_kernel_get_srtt(struct socket *, struct rxrpc_call *); +bool rxrpc_kernel_get_srtt(struct socket *, struct rxrpc_call *, u32 *); int rxrpc_kernel_charge_accept(struct socket *, rxrpc_notify_rx_t, rxrpc_user_attach_call_t, unsigned long, gfp_t, unsigned int); diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index b689ace..3cd232c 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -1308,16 +1308,34 @@ static inline void hci_auth_cfm(struct hci_conn *conn, __u8 status) conn->security_cfm_cb(conn, status); } -static inline void hci_encrypt_cfm(struct hci_conn *conn, __u8 status, - __u8 encrypt) +static inline void hci_encrypt_cfm(struct hci_conn *conn, __u8 status) { struct hci_cb *cb; + __u8 encrypt; - if (conn->sec_level == BT_SECURITY_SDP) - conn->sec_level = BT_SECURITY_LOW; + if (conn->state == BT_CONFIG) { + if (!status) + conn->state = BT_CONNECTED; - if (conn->pending_sec_level > conn->sec_level) - conn->sec_level = conn->pending_sec_level; + hci_connect_cfm(conn, status); + hci_conn_drop(conn); + return; + } + + if (!test_bit(HCI_CONN_ENCRYPT, &conn->flags)) + encrypt = 0x00; + else if (test_bit(HCI_CONN_AES_CCM, &conn->flags)) + encrypt = 0x02; + else + encrypt = 0x01; + + if (!status) { + if (conn->sec_level == BT_SECURITY_SDP) + conn->sec_level = BT_SECURITY_LOW; + + if (conn->pending_sec_level > conn->sec_level) + conn->sec_level = conn->pending_sec_level; + } mutex_lock(&hci_cb_list_lock); list_for_each_entry(cb, &hci_cb_list, list) { diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 093aede..8efc241 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -623,6 +623,8 @@ struct l2cap_ops { struct sk_buff *(*alloc_skb) (struct l2cap_chan *chan, unsigned long hdr_len, unsigned long len, int nb); + int (*filter) (struct l2cap_chan * chan, + struct sk_buff *skb); }; struct l2cap_conn { diff --git a/include/net/bonding.h b/include/net/bonding.h index 3d56b02..1bee8fd 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -180,6 +180,11 @@ struct slave { struct rtnl_link_stats64 slave_stats; }; +static inline struct slave *to_slave(struct kobject *kobj) +{ + return container_of(kobj, struct slave, kobj); +} + struct bond_up_slave { unsigned int count; struct rcu_head rcu; @@ -743,6 +748,9 @@ extern struct bond_parm_tbl ad_select_tbl[]; /* exported from bond_netlink.c */ extern struct rtnl_link_ops bond_link_ops; +/* exported from bond_sysfs_slave.c */ +extern const struct sysfs_ops slave_sysfs_ops; + static inline void bond_tx_drop(struct net_device *dev, struct sk_buff *skb) { atomic_long_inc(&dev->tx_dropped); diff --git a/include/net/flow.h b/include/net/flow.h index a50fb77..d058e63 100644 --- a/include/net/flow.h +++ b/include/net/flow.h @@ -116,6 +116,7 @@ static inline void flowi4_init_output(struct flowi4 *fl4, int oif, fl4->saddr = saddr; fl4->fl4_dport = dport; fl4->fl4_sport = sport; + fl4->flowi4_multipath_hash = 0; } /* Reset some input parameters after previous lookup */ diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index 8955460..c71eb29 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -309,6 +309,10 @@ int inet_csk_compat_getsockopt(struct sock *sk, int level, int optname, int inet_csk_compat_setsockopt(struct sock *sk, int level, int optname, char __user *optval, unsigned int optlen); +/* update the fast reuse flag when adding a socket */ +void inet_csk_update_fastreuse(struct inet_bind_bucket *tb, + struct sock *sk); + struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu); #define TCP_PINGPONG_THRESH 3 diff --git a/include/net/inet_ecn.h b/include/net/inet_ecn.h index e1eaf17..563457f 100644 --- a/include/net/inet_ecn.h +++ b/include/net/inet_ecn.h @@ -107,7 +107,7 @@ static inline int IP_ECN_set_ect1(struct iphdr *iph) if ((iph->tos & INET_ECN_MASK) != INET_ECN_ECT_0) return 0; - check += (__force u16)htons(0x100); + check += (__force u16)htons(0x1); iph->check = (__force __sum16)(check + (check>=0xFFFF)); iph->tos ^= INET_ECN_MASK; diff --git a/include/net/ip.h b/include/net/ip.h index 4b15cc1..0278d63 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -439,12 +439,18 @@ static inline unsigned int ip_dst_mtu_maybe_forward(const struct dst_entry *dst, bool forwarding) { struct net *net = dev_net(dst->dev); + unsigned int mtu; if (net->ipv4.sysctl_ip_fwd_use_pmtu || ip_mtu_locked(dst) || !forwarding) return dst_mtu(dst); + /* 'forwarding = true' case should always honour route mtu */ + mtu = dst_metric_raw(dst, RTAX_MTU); + if (mtu) + return mtu; + return min(READ_ONCE(dst->dev->mtu), IP_MAX_MTU); } diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index af64560..56deb25 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -472,9 +472,11 @@ static inline void ip_tunnel_info_opts_set(struct ip_tunnel_info *info, const void *from, int len, __be16 flags) { - memcpy(ip_tunnel_info_opts(info), from, len); info->options_len = len; - info->key.tun_flags |= flags; + if (len > 0) { + memcpy(ip_tunnel_info_opts(info), from, len); + info->key.tun_flags |= flags; + } } static inline struct ip_tunnel_info *lwt_tun_info(struct lwtunnel_state *lwtstate) @@ -520,7 +522,6 @@ static inline void ip_tunnel_info_opts_set(struct ip_tunnel_info *info, __be16 flags) { info->options_len = 0; - info->key.tun_flags |= flags; } #endif /* CONFIG_INET */ diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index 078887c..7c37e3c 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -1624,18 +1624,16 @@ static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp) } #endif /* CONFIG_IP_VS_NFCT */ -/* Really using conntrack? */ -static inline bool ip_vs_conn_uses_conntrack(struct ip_vs_conn *cp, - struct sk_buff *skb) +/* Using old conntrack that can not be redirected to another real server? */ +static inline bool ip_vs_conn_uses_old_conntrack(struct ip_vs_conn *cp, + struct sk_buff *skb) { #ifdef CONFIG_IP_VS_NFCT enum ip_conntrack_info ctinfo; struct nf_conn *ct; - if (!(cp->flags & IP_VS_CONN_F_NFCT)) - return false; ct = nf_ct_get(skb, &ctinfo); - if (ct) + if (ct && nf_ct_is_confirmed(ct)) return true; #endif return false; diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 6cdf3a0..4d6291e 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -204,6 +204,7 @@ struct neigh_table { int (*pconstructor)(struct pneigh_entry *); void (*pdestructor)(struct pneigh_entry *); void (*proxy_redo)(struct sk_buff *skb); + int (*is_multicast)(const void *pkey); bool (*allow_add)(const struct net_device *dev, struct netlink_ext_ack *extack); char *id; diff --git a/include/net/netfilter/nf_log.h b/include/net/netfilter/nf_log.h index 0d39208..716db4a 100644 --- a/include/net/netfilter/nf_log.h +++ b/include/net/netfilter/nf_log.h @@ -108,6 +108,7 @@ int nf_log_dump_tcp_header(struct nf_log_buf *m, const struct sk_buff *skb, unsigned int logflags); void nf_log_dump_sk_uid_gid(struct net *net, struct nf_log_buf *m, struct sock *sk); +void nf_log_dump_vlan(struct nf_log_buf *m, const struct sk_buff *skb); void nf_log_dump_packet_common(struct nf_log_buf *m, u_int8_t pf, unsigned int hooknum, const struct sk_buff *skb, const struct net_device *in, diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 2d0275f..f694f08 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -143,6 +143,8 @@ static inline u64 nft_reg_load64(u32 *sreg) static inline void nft_data_copy(u32 *dst, const struct nft_data *src, unsigned int len) { + if (len % NFT_REG32_SIZE) + dst[len / NFT_REG32_SIZE] = 0; memcpy(dst, src, len); } @@ -870,6 +872,12 @@ static inline struct nft_expr *nft_expr_last(const struct nft_rule *rule) return (struct nft_expr *)&rule->data[rule->dlen]; } +static inline bool nft_expr_more(const struct nft_rule *rule, + const struct nft_expr *expr) +{ + return expr != nft_expr_last(rule) && expr->ops; +} + static inline struct nft_userdata *nft_userdata(const struct nft_rule *rule) { return (void *)&rule->data[rule->dlen]; @@ -1454,4 +1462,10 @@ void nft_chain_filter_fini(void); void __init nft_chain_route_init(void); void nft_chain_route_fini(void); + +void nf_tables_trans_destroy_flush_work(void); + +int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result); +__be64 nf_jiffies64_to_msecs(u64 input); + #endif /* _NET_NF_TABLES_H */ diff --git a/include/net/netfilter/nf_tables_offload.h b/include/net/netfilter/nf_tables_offload.h index 03cf585..d0bb9e3 100644 --- a/include/net/netfilter/nf_tables_offload.h +++ b/include/net/netfilter/nf_tables_offload.h @@ -37,6 +37,7 @@ void nft_offload_update_dependency(struct nft_offload_ctx *ctx, struct nft_flow_key { struct flow_dissector_key_basic basic; + struct flow_dissector_key_control control; union { struct flow_dissector_key_ipv4_addrs ipv4; struct flow_dissector_key_ipv6_addrs ipv6; @@ -61,6 +62,9 @@ struct nft_flow_rule { #define NFT_OFFLOAD_F_ACTION (1 << 0) +void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow, + enum flow_dissector_key_id addr_type); + struct nft_rule; struct nft_flow_rule *nft_flow_rule_create(struct net *net, const struct nft_rule *rule); void nft_flow_rule_destroy(struct nft_flow_rule *flow); diff --git a/include/net/red.h b/include/net/red.h index 9665582..e21e7fd 100644 --- a/include/net/red.h +++ b/include/net/red.h @@ -168,12 +168,14 @@ static inline void red_set_vars(struct red_vars *v) v->qcount = -1; } -static inline bool red_check_params(u32 qth_min, u32 qth_max, u8 Wlog) +static inline bool red_check_params(u32 qth_min, u32 qth_max, u8 Wlog, u8 Scell_log) { if (fls(qth_min) + Wlog > 32) return false; if (fls(qth_max) + Wlog > 32) return false; + if (Scell_log >= 32) + return false; if (qth_max < qth_min) return false; return true; diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index 2b6f3f1..3e8f87a 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -224,12 +224,14 @@ struct sctp_sock { data_ready_signalled:1; atomic_t pd_mode; + + /* Fields after this point will be skipped on copies, like on accept + * and peeloff operations + */ + /* Receive to here while partial delivery is in effect. */ struct sk_buff_head pd_lobby; - /* These must be the last fields, as they will skipped on copies, - * like on accept and peeloff operations - */ struct list_head auto_asconf_list; int do_auto_asconf; }; diff --git a/include/net/sock.h b/include/net/sock.h index 85a94bc..b2b374b 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -854,6 +854,8 @@ static inline int sk_memalloc_socks(void) { return static_branch_unlikely(&memalloc_socks_key); } + +void __receive_sock(struct file *file); #else static inline int sk_memalloc_socks(void) @@ -861,6 +863,8 @@ static inline int sk_memalloc_socks(void) return 0; } +static inline void __receive_sock(struct file *file) +{ } #endif static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask) @@ -910,11 +914,11 @@ static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb) skb_dst_force(skb); if (!sk->sk_backlog.tail) - sk->sk_backlog.head = skb; + WRITE_ONCE(sk->sk_backlog.head, skb); else sk->sk_backlog.tail->next = skb; - sk->sk_backlog.tail = skb; + WRITE_ONCE(sk->sk_backlog.tail, skb); skb->next = NULL; } diff --git a/include/net/tcp.h b/include/net/tcp.h index 7cf1b49..3771792 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1650,6 +1650,8 @@ void tcp_fastopen_destroy_cipher(struct sock *sk); void tcp_fastopen_ctx_destroy(struct net *net); int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, void *primary_key, void *backup_key); +int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk, + u64 *key); void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, struct request_sock *req, diff --git a/include/net/tls.h b/include/net/tls.h index 0a065bd..697df45 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -221,6 +221,12 @@ enum tls_context_flags { * to be atomic. */ TLS_TX_SYNC_SCHED = 1, + /* tls_dev_del was called for the RX side, device state was released, + * but tls_ctx->netdev might still be kept, because TX-side driver + * resources might not be released yet. Used to prevent the second + * tls_dev_del call in tls_device_down if it happens simultaneously. + */ + TLS_RX_DEV_CLOSED = 2, }; struct cipher_context { diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 12aa6e1..c00b9ae 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -1773,21 +1773,17 @@ static inline unsigned int xfrm_replay_state_esn_len(struct xfrm_replay_state_es static inline int xfrm_replay_clone(struct xfrm_state *x, struct xfrm_state *orig) { - x->replay_esn = kzalloc(xfrm_replay_state_esn_len(orig->replay_esn), + + x->replay_esn = kmemdup(orig->replay_esn, + xfrm_replay_state_esn_len(orig->replay_esn), GFP_KERNEL); if (!x->replay_esn) return -ENOMEM; - - x->replay_esn->bmp_len = orig->replay_esn->bmp_len; - x->replay_esn->replay_window = orig->replay_esn->replay_window; - - x->preplay_esn = kmemdup(x->replay_esn, - xfrm_replay_state_esn_len(x->replay_esn), + x->preplay_esn = kmemdup(orig->preplay_esn, + xfrm_replay_state_esn_len(orig->preplay_esn), GFP_KERNEL); - if (!x->preplay_esn) { - kfree(x->replay_esn); + if (!x->preplay_esn) return -ENOMEM; - } return 0; } diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index a91b2af..8e94279 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -95,10 +95,11 @@ static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offs size_t length) { return -EINVAL; } -static inline int ib_umem_find_best_pgsz(struct ib_umem *umem, - unsigned long pgsz_bitmap, - unsigned long virt) { - return -EINVAL; +static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, + unsigned long pgsz_bitmap, + unsigned long virt) +{ + return 0; } #endif /* CONFIG_INFINIBAND_USER_MEM */ diff --git a/include/scsi/libiscsi.h b/include/scsi/libiscsi.h index c25fb86..b3bbd10 100644 --- a/include/scsi/libiscsi.h +++ b/include/scsi/libiscsi.h @@ -132,6 +132,9 @@ struct iscsi_task { void *dd_data; /* driver/transport data */ }; +/* invalid scsi_task pointer */ +#define INVALID_SCSI_TASK (struct iscsi_task *)-1l + static inline int iscsi_task_has_unsol_data(struct iscsi_task *task) { return task->unsol_r2t.data_length > task->unsol_r2t.sent; diff --git a/include/scsi/scsi_common.h b/include/scsi/scsi_common.h index 731ac09..5b567b4 100644 --- a/include/scsi/scsi_common.h +++ b/include/scsi/scsi_common.h @@ -25,6 +25,13 @@ scsi_command_size(const unsigned char *cmnd) scsi_varlen_cdb_length(cmnd) : COMMAND_SIZE(cmnd[0]); } +static inline unsigned char +scsi_command_control(const unsigned char *cmnd) +{ + return (cmnd[0] == VARIABLE_LENGTH_CMD) ? + cmnd[1] : cmnd[COMMAND_SIZE(cmnd[0]) - 1]; +} + /* Returns a human-readable name for the device */ extern const char *scsi_device_type(unsigned type); diff --git a/include/soc/nps/common.h b/include/soc/nps/common.h index 9b1d43d..8c18dc6 100644 --- a/include/soc/nps/common.h +++ b/include/soc/nps/common.h @@ -45,6 +45,12 @@ #define CTOP_INST_MOV2B_FLIP_R3_B1_B2_INST 0x5B60 #define CTOP_INST_MOV2B_FLIP_R3_B1_B2_LIMM 0x00010422 +#ifndef AUX_IENABLE +#define AUX_IENABLE 0x40c +#endif + +#define CTOP_AUX_IACK (0xFFFFF800 + 0x088) + #ifndef __ASSEMBLY__ /* In order to increase compilation test coverage */ diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h index 3083c9e..a5e2f8b 100644 --- a/include/sound/hda_codec.h +++ b/include/sound/hda_codec.h @@ -257,6 +257,7 @@ struct hda_codec { unsigned int force_pin_prefix:1; /* Add location prefix */ unsigned int link_down_at_suspend:1; /* link down at runtime suspend */ unsigned int relaxed_resume:1; /* don't resume forcibly for jack */ + unsigned int forced_resume:1; /* forced resume for jack */ #ifdef CONFIG_PM unsigned long power_on_acct; @@ -497,6 +498,11 @@ void snd_hda_update_power_acct(struct hda_codec *codec); static inline void snd_hda_set_power_save(struct hda_bus *bus, int delay) {} #endif +static inline bool hda_codec_need_resume(struct hda_codec *codec) +{ + return !codec->relaxed_resume && codec->jacktbl.used; +} + #ifdef CONFIG_SND_HDA_PATCH_LOADER /* * patch firmware diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 75ae189..94a3adb 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -1159,25 +1159,27 @@ DEFINE_EVENT(btrfs__reserved_extent, btrfs_reserved_extent_free, TRACE_EVENT(find_free_extent, - TP_PROTO(const struct btrfs_fs_info *fs_info, u64 num_bytes, + TP_PROTO(const struct btrfs_root *root, u64 num_bytes, u64 empty_size, u64 data), - TP_ARGS(fs_info, num_bytes, empty_size, data), + TP_ARGS(root, num_bytes, empty_size, data), TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) __field( u64, num_bytes ) __field( u64, empty_size ) __field( u64, data ) ), - TP_fast_assign_btrfs(fs_info, + TP_fast_assign_btrfs(root->fs_info, + __entry->root_objectid = root->root_key.objectid; __entry->num_bytes = num_bytes; __entry->empty_size = empty_size; __entry->data = data; ), TP_printk_btrfs("root=%llu(%s) len=%llu empty_size=%llu flags=%llu(%s)", - show_root_type(BTRFS_EXTENT_TREE_OBJECTID), + show_root_type(__entry->root_objectid), __entry->num_bytes, __entry->empty_size, __entry->data, __print_flags((unsigned long)__entry->data, "|", BTRFS_GROUP_FLAGS)) diff --git a/include/trace/events/sctp.h b/include/trace/events/sctp.h index 7475c7b..d4aac34 100644 --- a/include/trace/events/sctp.h +++ b/include/trace/events/sctp.h @@ -75,15 +75,6 @@ TRACE_EVENT(sctp_probe, __entry->pathmtu = asoc->pathmtu; __entry->rwnd = asoc->peer.rwnd; __entry->unack_data = asoc->unack_data; - - if (trace_sctp_probe_path_enabled()) { - struct sctp_transport *sp; - - list_for_each_entry(sp, &asoc->peer.transport_addr_list, - transports) { - trace_sctp_probe_path(sp, asoc); - } - } ), TP_printk("asoc=%#llx mark=%#x bind_port=%d peer_port=%d pathmtu=%d " diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h index ffa3c51..f16e9fb 100644 --- a/include/trace/events/sunrpc.h +++ b/include/trace/events/sunrpc.h @@ -165,6 +165,7 @@ DECLARE_EVENT_CLASS(rpc_task_running, DEFINE_RPC_RUNNING_EVENT(begin); DEFINE_RPC_RUNNING_EVENT(run_action); DEFINE_RPC_RUNNING_EVENT(complete); +DEFINE_RPC_RUNNING_EVENT(end); DECLARE_EVENT_CLASS(rpc_task_queued, @@ -356,10 +357,10 @@ TRACE_EVENT(rpc_xdr_overflow, __field(size_t, tail_len) __field(unsigned int, page_len) __field(unsigned int, len) - __string(progname, - xdr->rqst->rq_task->tk_client->cl_program->name) - __string(procedure, - xdr->rqst->rq_task->tk_msg.rpc_proc->p_name) + __string(progname, xdr->rqst ? + xdr->rqst->rq_task->tk_client->cl_program->name : "unknown") + __string(procedure, xdr->rqst ? + xdr->rqst->rq_task->tk_msg.rpc_proc->p_name : "unknown") ), TP_fast_assign( diff --git a/include/trace/events/target.h b/include/trace/events/target.h index 914a872..e87a371 100644 --- a/include/trace/events/target.h +++ b/include/trace/events/target.h @@ -140,6 +140,7 @@ TRACE_EVENT(target_sequencer_start, __field( unsigned int, opcode ) __field( unsigned int, data_length ) __field( unsigned int, task_attribute ) + __field( unsigned char, control ) __array( unsigned char, cdb, TCM_MAX_COMMAND_SIZE ) __string( initiator, cmd->se_sess->se_node_acl->initiatorname ) ), @@ -149,6 +150,7 @@ TRACE_EVENT(target_sequencer_start, __entry->opcode = cmd->t_task_cdb[0]; __entry->data_length = cmd->data_length; __entry->task_attribute = cmd->sam_task_attr; + __entry->control = scsi_command_control(cmd->t_task_cdb); memcpy(__entry->cdb, cmd->t_task_cdb, TCM_MAX_COMMAND_SIZE); __assign_str(initiator, cmd->se_sess->se_node_acl->initiatorname); ), @@ -158,9 +160,7 @@ TRACE_EVENT(target_sequencer_start, show_opcode_name(__entry->opcode), __entry->data_length, __print_hex(__entry->cdb, 16), show_task_attribute_name(__entry->task_attribute), - scsi_command_size(__entry->cdb) <= 16 ? - __entry->cdb[scsi_command_size(__entry->cdb) - 1] : - __entry->cdb[1] + __entry->control ) ); @@ -175,6 +175,7 @@ TRACE_EVENT(target_cmd_complete, __field( unsigned int, opcode ) __field( unsigned int, data_length ) __field( unsigned int, task_attribute ) + __field( unsigned char, control ) __field( unsigned char, scsi_status ) __field( unsigned char, sense_length ) __array( unsigned char, cdb, TCM_MAX_COMMAND_SIZE ) @@ -187,6 +188,7 @@ TRACE_EVENT(target_cmd_complete, __entry->opcode = cmd->t_task_cdb[0]; __entry->data_length = cmd->data_length; __entry->task_attribute = cmd->sam_task_attr; + __entry->control = scsi_command_control(cmd->t_task_cdb); __entry->scsi_status = cmd->scsi_status; __entry->sense_length = cmd->scsi_status == SAM_STAT_CHECK_CONDITION ? min(18, ((u8 *) cmd->sense_buffer)[SPC_ADD_SENSE_LEN_OFFSET] + 8) : 0; @@ -203,9 +205,7 @@ TRACE_EVENT(target_cmd_complete, show_opcode_name(__entry->opcode), __entry->data_length, __print_hex(__entry->cdb, 16), show_task_attribute_name(__entry->task_attribute), - scsi_command_size(__entry->cdb) <= 16 ? - __entry->cdb[scsi_command_size(__entry->cdb) - 1] : - __entry->cdb[1] + __entry->control ) ); diff --git a/include/trace/events/writeback.h b/include/trace/events/writeback.h index 6628255..a8af22e 100644 --- a/include/trace/events/writeback.h +++ b/include/trace/events/writeback.h @@ -192,7 +192,7 @@ TRACE_EVENT(inode_foreign_history, ), TP_fast_assign( - strncpy(__entry->name, bdi_dev_name(inode_to_bdi(inode)), 32); + strscpy_pad(__entry->name, bdi_dev_name(inode_to_bdi(inode)), 32); __entry->ino = inode->i_ino; __entry->cgroup_ino = __trace_wbc_assign_cgroup(wbc); __entry->history = history; @@ -221,7 +221,7 @@ TRACE_EVENT(inode_switch_wbs, ), TP_fast_assign( - strncpy(__entry->name, bdi_dev_name(old_wb->bdi), 32); + strscpy_pad(__entry->name, bdi_dev_name(old_wb->bdi), 32); __entry->ino = inode->i_ino; __entry->old_cgroup_ino = __trace_wb_assign_cgroup(old_wb); __entry->new_cgroup_ino = __trace_wb_assign_cgroup(new_wb); @@ -254,7 +254,7 @@ TRACE_EVENT(track_foreign_dirty, struct address_space *mapping = page_mapping(page); struct inode *inode = mapping ? mapping->host : NULL; - strncpy(__entry->name, bdi_dev_name(wb->bdi), 32); + strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32); __entry->bdi_id = wb->bdi->id; __entry->ino = inode ? inode->i_ino : 0; __entry->memcg_id = wb->memcg_css->id; @@ -287,7 +287,7 @@ TRACE_EVENT(flush_foreign, ), TP_fast_assign( - strncpy(__entry->name, bdi_dev_name(wb->bdi), 32); + strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32); __entry->cgroup_ino = __trace_wb_assign_cgroup(wb); __entry->frn_bdi_id = frn_bdi_id; __entry->frn_memcg_id = frn_memcg_id; @@ -499,8 +499,9 @@ DEFINE_WBC_EVENT(wbc_writepage); TRACE_EVENT(writeback_queue_io, TP_PROTO(struct bdi_writeback *wb, struct wb_writeback_work *work, + unsigned long dirtied_before, int moved), - TP_ARGS(wb, work, moved), + TP_ARGS(wb, work, dirtied_before, moved), TP_STRUCT__entry( __array(char, name, 32) __field(unsigned long, older) @@ -510,19 +511,17 @@ TRACE_EVENT(writeback_queue_io, __field(unsigned int, cgroup_ino) ), TP_fast_assign( - unsigned long *older_than_this = work->older_than_this; strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32); - __entry->older = older_than_this ? *older_than_this : 0; - __entry->age = older_than_this ? - (jiffies - *older_than_this) * 1000 / HZ : -1; + __entry->older = dirtied_before; + __entry->age = (jiffies - dirtied_before) * 1000 / HZ; __entry->moved = moved; __entry->reason = work->reason; __entry->cgroup_ino = __trace_wb_assign_cgroup(wb); ), TP_printk("bdi %s: older=%lu age=%ld enqueue=%d reason=%s cgroup_ino=%u", __entry->name, - __entry->older, /* older_than_this in jiffies */ - __entry->age, /* older_than_this in relative milliseconds */ + __entry->older, /* dirtied_before in jiffies */ + __entry->age, /* dirtied_before in relative milliseconds */ __entry->moved, __print_symbolic(__entry->reason, WB_WORK_REASON), __entry->cgroup_ino diff --git a/include/uapi/linux/android/binder.h b/include/uapi/linux/android/binder.h index 2832134..7317808 100644 --- a/include/uapi/linux/android/binder.h +++ b/include/uapi/linux/android/binder.h @@ -248,6 +248,7 @@ enum transaction_flags { TF_ROOT_OBJECT = 0x04, /* contents are the component's root object */ TF_STATUS_CODE = 0x08, /* contents are a 32-bit status code */ TF_ACCEPT_FDS = 0x10, /* allow replies with file descriptors */ + TF_CLEAR_BUF = 0x20, /* clear buffer on txn complete */ }; struct binder_transaction_data { diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 77c6be9..8649422 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1294,8 +1294,8 @@ union bpf_attr { * Return * The return value depends on the result of the test, and can be: * - * * 0, if the *skb* task belongs to the cgroup2. - * * 1, if the *skb* task does not belong to the cgroup2. + * * 0, if current task belongs to the cgroup2. + * * 1, if current task does not belong to the cgroup2. * * A negative error code, if an error occurred. * * int bpf_skb_change_tail(struct sk_buff *skb, u32 len, u64 flags) diff --git a/include/uapi/linux/btrfs_tree.h b/include/uapi/linux/btrfs_tree.h index b65c7ee..035e59e 100644 --- a/include/uapi/linux/btrfs_tree.h +++ b/include/uapi/linux/btrfs_tree.h @@ -4,6 +4,11 @@ #include #include +#ifdef __KERNEL__ +#include +#else +#include +#endif /* * This header contains the structure definitions and constants used @@ -650,6 +655,15 @@ struct btrfs_root_item { __le64 reserved[8]; /* for future */ } __attribute__ ((__packed__)); +/* + * Btrfs root item used to be smaller than current size. The old format ends + * at where member generation_v2 is. + */ +static inline __u32 btrfs_legacy_root_item_size(void) +{ + return offsetof(struct btrfs_root_item, generation_v2); +} + /* * this is used for both forward and backward root refs */ diff --git a/include/uapi/linux/const.h b/include/uapi/linux/const.h index 5ed721a..af2a44c 100644 --- a/include/uapi/linux/const.h +++ b/include/uapi/linux/const.h @@ -28,4 +28,9 @@ #define _BITUL(x) (_UL(1) << (x)) #define _BITULL(x) (_ULL(1) << (x)) +#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (typeof(x))(a) - 1) +#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask)) + +#define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) + #endif /* _UAPI_LINUX_CONST_H */ diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h index 8938b76..7857aa4 100644 --- a/include/uapi/linux/ethtool.h +++ b/include/uapi/linux/ethtool.h @@ -14,7 +14,7 @@ #ifndef _UAPI_LINUX_ETHTOOL_H #define _UAPI_LINUX_ETHTOOL_H -#include +#include #include #include diff --git a/include/uapi/linux/fscrypt.h b/include/uapi/linux/fscrypt.h index 39ccfe9..b14f436 100644 --- a/include/uapi/linux/fscrypt.h +++ b/include/uapi/linux/fscrypt.h @@ -17,7 +17,6 @@ #define FSCRYPT_POLICY_FLAGS_PAD_32 0x03 #define FSCRYPT_POLICY_FLAGS_PAD_MASK 0x03 #define FSCRYPT_POLICY_FLAG_DIRECT_KEY 0x04 -#define FSCRYPT_POLICY_FLAGS_VALID 0x07 /* Encryption algorithms */ #define FSCRYPT_MODE_AES_256_XTS 1 @@ -25,7 +24,7 @@ #define FSCRYPT_MODE_AES_128_CBC 5 #define FSCRYPT_MODE_AES_128_CTS 6 #define FSCRYPT_MODE_ADIANTUM 9 -#define __FSCRYPT_MODE_MAX 9 +/* If adding a mode number > 9, update FSCRYPT_MODE_MAX in fscrypt_private.h */ /* * Legacy policy version; ad-hoc KDF and no key verification. @@ -162,7 +161,7 @@ struct fscrypt_get_key_status_arg { #define FS_POLICY_FLAGS_PAD_32 FSCRYPT_POLICY_FLAGS_PAD_32 #define FS_POLICY_FLAGS_PAD_MASK FSCRYPT_POLICY_FLAGS_PAD_MASK #define FS_POLICY_FLAG_DIRECT_KEY FSCRYPT_POLICY_FLAG_DIRECT_KEY -#define FS_POLICY_FLAGS_VALID FSCRYPT_POLICY_FLAGS_VALID +#define FS_POLICY_FLAGS_VALID 0x07 /* contains old flags only */ #define FS_ENCRYPTION_MODE_INVALID 0 /* never used */ #define FS_ENCRYPTION_MODE_AES_256_XTS FSCRYPT_MODE_AES_256_XTS #define FS_ENCRYPTION_MODE_AES_256_GCM 2 /* never used */ diff --git a/include/uapi/linux/if_alg.h b/include/uapi/linux/if_alg.h index bc2bcde..7690507 100644 --- a/include/uapi/linux/if_alg.h +++ b/include/uapi/linux/if_alg.h @@ -24,6 +24,22 @@ struct sockaddr_alg { __u8 salg_name[64]; }; +/* + * Linux v4.12 and later removed the 64-byte limit on salg_name[]; it's now an + * arbitrary-length field. We had to keep the original struct above for source + * compatibility with existing userspace programs, though. Use the new struct + * below if support for very long algorithm names is needed. To do this, + * allocate 'sizeof(struct sockaddr_alg_new) + strlen(algname) + 1' bytes, and + * copy algname (including the null terminator) into salg_name. + */ +struct sockaddr_alg_new { + __u16 salg_family; + __u8 salg_type[14]; + __u32 salg_feat; + __u32 salg_mask; + __u8 salg_name[]; +}; + struct af_alg_iv { __u32 ivlen; __u8 iv[0]; diff --git a/include/uapi/linux/kernel.h b/include/uapi/linux/kernel.h index 0ff8f74..fadf2db 100644 --- a/include/uapi/linux/kernel.h +++ b/include/uapi/linux/kernel.h @@ -3,13 +3,6 @@ #define _UAPI_LINUX_KERNEL_H #include - -/* - * 'kernel.h' contains some often-used function prototypes etc - */ -#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (typeof(x))(a) - 1) -#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask)) - -#define __KERNEL_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#include #endif /* _UAPI_LINUX_KERNEL_H */ diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 94e3b82..879b438 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -775,9 +775,10 @@ struct kvm_ppc_resize_hpt { #define KVM_VM_PPC_HV 1 #define KVM_VM_PPC_PR 2 -/* on MIPS, 0 forces trap & emulate, 1 forces VZ ASE */ -#define KVM_VM_MIPS_TE 0 +/* on MIPS, 0 indicates auto, 1 forces VZ ASE, 2 forces trap & emulate */ +#define KVM_VM_MIPS_AUTO 0 #define KVM_VM_MIPS_VZ 1 +#define KVM_VM_MIPS_TE 2 #define KVM_S390_SIE_PAGE_OFFSET 1 diff --git a/include/uapi/linux/lightnvm.h b/include/uapi/linux/lightnvm.h index f9a1be7..ead2e72 100644 --- a/include/uapi/linux/lightnvm.h +++ b/include/uapi/linux/lightnvm.h @@ -21,7 +21,7 @@ #define _UAPI_LINUX_LIGHTNVM_H #ifdef __KERNEL__ -#include +#include #include #else /* __KERNEL__ */ #include diff --git a/include/uapi/linux/mroute6.h b/include/uapi/linux/mroute6.h index c36177a..a1fd617 100644 --- a/include/uapi/linux/mroute6.h +++ b/include/uapi/linux/mroute6.h @@ -2,7 +2,7 @@ #ifndef _UAPI__LINUX_MROUTE6_H #define _UAPI__LINUX_MROUTE6_H -#include +#include #include #include #include /* For struct sockaddr_in6. */ diff --git a/include/uapi/linux/netfilter/nf_tables.h b/include/uapi/linux/netfilter/nf_tables.h index ed8881a..0a99540 100644 --- a/include/uapi/linux/netfilter/nf_tables.h +++ b/include/uapi/linux/netfilter/nf_tables.h @@ -132,7 +132,7 @@ enum nf_tables_msg_types { * @NFTA_LIST_ELEM: list element (NLA_NESTED) */ enum nft_list_attributes { - NFTA_LIST_UNPEC, + NFTA_LIST_UNSPEC, NFTA_LIST_ELEM, __NFTA_LIST_MAX }; diff --git a/include/uapi/linux/netfilter/x_tables.h b/include/uapi/linux/netfilter/x_tables.h index a8283f7..b8c6bb2 100644 --- a/include/uapi/linux/netfilter/x_tables.h +++ b/include/uapi/linux/netfilter/x_tables.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI_X_TABLES_H #define _UAPI_X_TABLES_H -#include +#include #include #define XT_FUNCTION_MAXNAMELEN 30 diff --git a/include/uapi/linux/netlink.h b/include/uapi/linux/netlink.h index 0a4d733..622c78c 100644 --- a/include/uapi/linux/netlink.h +++ b/include/uapi/linux/netlink.h @@ -2,7 +2,7 @@ #ifndef _UAPI__LINUX_NETLINK_H #define _UAPI__LINUX_NETLINK_H -#include +#include #include /* for __kernel_sa_family_t */ #include diff --git a/include/uapi/linux/nfs4.h b/include/uapi/linux/nfs4.h index 8572930..54a7852 100644 --- a/include/uapi/linux/nfs4.h +++ b/include/uapi/linux/nfs4.h @@ -136,6 +136,8 @@ #define EXCHGID4_FLAG_UPD_CONFIRMED_REC_A 0x40000000 #define EXCHGID4_FLAG_CONFIRMED_R 0x80000000 + +#define EXCHGID4_FLAG_SUPP_FENCE_OPS 0x00000004 /* * Since the validity of these bits depends on whether * they're set in the argument or response, have separate @@ -143,6 +145,7 @@ */ #define EXCHGID4_FLAG_MASK_A 0x40070103 #define EXCHGID4_FLAG_MASK_R 0x80070103 +#define EXCHGID4_2_FLAG_MASK_R 0x80070107 #define SEQ4_STATUS_CB_PATH_DOWN 0x00000001 #define SEQ4_STATUS_CB_GSS_CONTEXTS_EXPIRING 0x00000002 diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index bb7b271..ceccd98 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -1131,7 +1131,7 @@ union perf_mem_data_src { #define PERF_MEM_SNOOPX_FWD 0x01 /* forward */ /* 1 free */ -#define PERF_MEM_SNOOPX_SHIFT 37 +#define PERF_MEM_SNOOPX_SHIFT 38 /* locked instruction */ #define PERF_MEM_LOCK_NA 0x01 /* not available */ diff --git a/include/uapi/linux/ptrace.h b/include/uapi/linux/ptrace.h index a71b6e3..83ee45f 100644 --- a/include/uapi/linux/ptrace.h +++ b/include/uapi/linux/ptrace.h @@ -81,7 +81,8 @@ struct seccomp_metadata { struct ptrace_syscall_info { __u8 op; /* PTRACE_SYSCALL_INFO_* */ - __u32 arch __attribute__((__aligned__(sizeof(__u32)))); + __u8 pad[3]; + __u32 arch; __u64 instruction_pointer; __u64 stack_pointer; union { diff --git a/include/uapi/linux/seccomp.h b/include/uapi/linux/seccomp.h index 90734aa..b5f901a 100644 --- a/include/uapi/linux/seccomp.h +++ b/include/uapi/linux/seccomp.h @@ -93,5 +93,6 @@ struct seccomp_notif_resp { #define SECCOMP_IOCTL_NOTIF_RECV SECCOMP_IOWR(0, struct seccomp_notif) #define SECCOMP_IOCTL_NOTIF_SEND SECCOMP_IOWR(1, \ struct seccomp_notif_resp) -#define SECCOMP_IOCTL_NOTIF_ID_VALID SECCOMP_IOR(2, __u64) +#define SECCOMP_IOCTL_NOTIF_ID_VALID SECCOMP_IOW(2, __u64) + #endif /* _UAPI_LINUX_SECCOMP_H */ diff --git a/include/uapi/linux/sysctl.h b/include/uapi/linux/sysctl.h index 2216dd1..df9b463 100644 --- a/include/uapi/linux/sysctl.h +++ b/include/uapi/linux/sysctl.h @@ -23,7 +23,7 @@ #ifndef _UAPI_LINUX_SYSCTL_H #define _UAPI_LINUX_SYSCTL_H -#include +#include #include #include diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 9e843a1..cabc931 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -748,6 +748,21 @@ struct vfio_iommu_type1_info_cap_iova_range { struct vfio_iova_range iova_ranges[]; }; +/* + * The DMA available capability allows to report the current number of + * simultaneously outstanding DMA mappings that are allowed. + * + * The structure below defines version 1 of this capability. + * + * avail: specifies the current number of outstanding DMA mappings allowed. + */ +#define VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL 3 + +struct vfio_iommu_type1_info_dma_avail { + struct vfio_info_cap_header header; + __u32 avail; +}; + #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12) /** diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index 530638d..3210b3c 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -371,9 +371,9 @@ enum v4l2_hsv_encoding { enum v4l2_quantization { /* - * The default for R'G'B' quantization is always full range, except - * for the BT2020 colorspace. For Y'CbCr the quantization is always - * limited range, except for COLORSPACE_JPEG: this is full range. + * The default for R'G'B' quantization is always full range. + * For Y'CbCr the quantization is always limited range, except + * for COLORSPACE_JPEG: this is full range. */ V4L2_QUANTIZATION_DEFAULT = 0, V4L2_QUANTIZATION_FULL_RANGE = 1, @@ -382,14 +382,13 @@ enum v4l2_quantization { /* * Determine how QUANTIZATION_DEFAULT should map to a proper quantization. - * This depends on whether the image is RGB or not, the colorspace and the - * Y'CbCr encoding. + * This depends on whether the image is RGB or not, the colorspace. + * The Y'CbCr encoding is not used anymore, but is still there for backwards + * compatibility. */ #define V4L2_MAP_QUANTIZATION_DEFAULT(is_rgb_or_hsv, colsp, ycbcr_enc) \ - (((is_rgb_or_hsv) && (colsp) == V4L2_COLORSPACE_BT2020) ? \ - V4L2_QUANTIZATION_LIM_RANGE : \ - (((is_rgb_or_hsv) || (colsp) == V4L2_COLORSPACE_JPEG) ? \ - V4L2_QUANTIZATION_FULL_RANGE : V4L2_QUANTIZATION_LIM_RANGE)) + (((is_rgb_or_hsv) || (colsp) == V4L2_COLORSPACE_JPEG) ? \ + V4L2_QUANTIZATION_FULL_RANGE : V4L2_QUANTIZATION_LIM_RANGE) /* * Deprecated names for opRGB colorspace (IEC 61966-2-5) diff --git a/include/uapi/linux/wireless.h b/include/uapi/linux/wireless.h index a2c006a..24f3371 100644 --- a/include/uapi/linux/wireless.h +++ b/include/uapi/linux/wireless.h @@ -74,7 +74,11 @@ #include /* for "struct sockaddr" et al */ #include /* for IFNAMSIZ and co... */ -#include /* for offsetof */ +#ifdef __KERNEL__ +# include /* for offsetof */ +#else +# include /* for offsetof */ +#endif /***************************** VERSION *****************************/ /* diff --git a/include/uapi/linux/xattr.h b/include/uapi/linux/xattr.h index 290db8b..c1395b5 100644 --- a/include/uapi/linux/xattr.h +++ b/include/uapi/linux/xattr.h @@ -66,13 +66,6 @@ #define XATTR_NAME_SMACKTRANSMUTE XATTR_SECURITY_PREFIX XATTR_SMACK_TRANSMUTE #define XATTR_NAME_SMACKMMAP XATTR_SECURITY_PREFIX XATTR_SMACK_MMAP -#define XATTR_ELMAC_SUFFIX "ELMAC64" -#define XATTR_ELMAC_IPIN "ELMAC64IPIN" -#define XATTR_ELMAC_IPOUT "ELMAC64IPOUT" -#define XATTR_NAME_ELMAC XATTR_SECURITY_PREFIX XATTR_ELMAC_SUFFIX -#define XATTR_NAME_ELMAC_IPIN XATTR_SECURITY_PREFIX XATTR_ELMAC_IPIN -#define XATTR_NAME_ELMAC_IPOUT XATTR_SECURITY_PREFIX XATTR_ELMAC_IPOUT - #define XATTR_APPARMOR_SUFFIX "apparmor" #define XATTR_NAME_APPARMOR XATTR_SECURITY_PREFIX XATTR_APPARMOR_SUFFIX diff --git a/include/uapi/sound/skl-tplg-interface.h b/include/uapi/sound/skl-tplg-interface.h index 9eee32f..a93c0de 100644 --- a/include/uapi/sound/skl-tplg-interface.h +++ b/include/uapi/sound/skl-tplg-interface.h @@ -18,6 +18,8 @@ */ #define SKL_CONTROL_TYPE_BYTE_TLV 0x100 #define SKL_CONTROL_TYPE_MIC_SELECT 0x102 +#define SKL_CONTROL_TYPE_MULTI_IO_SELECT 0x103 +#define SKL_CONTROL_TYPE_MULTI_IO_SELECT_DMIC 0x104 #define HDA_SST_CFG_MAX 900 /* size of copier cfg*/ #define MAX_IN_QUEUE 8 diff --git a/include/xen/events.h b/include/xen/events.h index c0e6a05..3195230 100644 --- a/include/xen/events.h +++ b/include/xen/events.h @@ -14,11 +14,16 @@ unsigned xen_evtchn_nr_channels(void); -int bind_evtchn_to_irq(unsigned int evtchn); -int bind_evtchn_to_irqhandler(unsigned int evtchn, +int bind_evtchn_to_irq(evtchn_port_t evtchn); +int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn); +int bind_evtchn_to_irqhandler(evtchn_port_t evtchn, irq_handler_t handler, unsigned long irqflags, const char *devname, void *dev_id); +int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn, + irq_handler_t handler, + unsigned long irqflags, const char *devname, + void *dev_id); int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu); int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu, irq_handler_t handler, @@ -31,13 +36,21 @@ int bind_ipi_to_irqhandler(enum ipi_vector ipi, const char *devname, void *dev_id); int bind_interdomain_evtchn_to_irq(unsigned int remote_domain, - unsigned int remote_port); + evtchn_port_t remote_port); +int bind_interdomain_evtchn_to_irq_lateeoi(unsigned int remote_domain, + evtchn_port_t remote_port); int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain, - unsigned int remote_port, + evtchn_port_t remote_port, irq_handler_t handler, unsigned long irqflags, const char *devname, void *dev_id); +int bind_interdomain_evtchn_to_irqhandler_lateeoi(unsigned int remote_domain, + evtchn_port_t remote_port, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, + void *dev_id); /* * Common unbind function for all event sources. Takes IRQ to unbind from. @@ -46,6 +59,14 @@ int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain, */ void unbind_from_irqhandler(unsigned int irq, void *dev_id); +/* + * Send late EOI for an IRQ bound to an event channel via one of the *_lateeoi + * functions above. + */ +void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags); +/* Signal an event was spurious, i.e. there was no action resulting from it. */ +#define XEN_EOI_FLAG_SPURIOUS 0x00000001 + #define XEN_IRQ_PRIORITY_MAX EVTCHN_FIFO_PRIORITY_MAX #define XEN_IRQ_PRIORITY_DEFAULT EVTCHN_FIFO_PRIORITY_DEFAULT #define XEN_IRQ_PRIORITY_MIN EVTCHN_FIFO_PRIORITY_MIN diff --git a/include/xen/xenbus.h b/include/xen/xenbus.h index 869c816..eba01ab 100644 --- a/include/xen/xenbus.h +++ b/include/xen/xenbus.h @@ -59,6 +59,15 @@ struct xenbus_watch /* Path being watched. */ const char *node; + unsigned int nr_pending; + + /* + * Called just before enqueing new event while a spinlock is held. + * The event will be discarded if this callback returns false. + */ + bool (*will_handle)(struct xenbus_watch *, + const char *path, const char *token); + /* Callback (executed in a process context with no locks held). */ void (*callback)(struct xenbus_watch *, const char *path, const char *token); @@ -192,10 +201,14 @@ void xenbus_probe(struct work_struct *); int xenbus_watch_path(struct xenbus_device *dev, const char *path, struct xenbus_watch *watch, + bool (*will_handle)(struct xenbus_watch *, + const char *, const char *), void (*callback)(struct xenbus_watch *, const char *, const char *)); -__printf(4, 5) +__printf(5, 6) int xenbus_watch_pathfmt(struct xenbus_device *dev, struct xenbus_watch *watch, + bool (*will_handle)(struct xenbus_watch *, + const char *, const char *), void (*callback)(struct xenbus_watch *, const char *, const char *), const char *pathfmt, ...);