1355 lines
36 KiB
C
1355 lines
36 KiB
C
/*
|
|
* User address space access functions.
|
|
* The non-inlined parts of asm-metag/uaccess.h are here.
|
|
*
|
|
* Copyright (C) 2006, Imagination Technologies.
|
|
* Copyright (C) 2000, Axis Communications AB.
|
|
*
|
|
* Written by Hans-Peter Nilsson.
|
|
* Pieces used from memcpy, originally by Kenny Ranerup long time ago.
|
|
* Modified for Meta by Will Newton.
|
|
*/
|
|
|
|
#include <linux/export.h>
|
|
#include <linux/uaccess.h>
|
|
#include <asm/cache.h> /* def of L1_CACHE_BYTES */
|
|
|
|
#define USE_RAPF
|
|
#define RAPF_MIN_BUF_SIZE (3*L1_CACHE_BYTES)
|
|
|
|
|
|
/* The "double write" in this code is because the Meta will not fault
|
|
* immediately unless the memory pipe is forced to by e.g. a data stall or
|
|
* another memory op. The second write should be discarded by the write
|
|
* combiner so should have virtually no cost.
|
|
*/
|
|
|
|
#define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
asm volatile ( \
|
|
COPY \
|
|
"1:\n" \
|
|
" .section .fixup,\"ax\"\n" \
|
|
" MOV D1Ar1,#0\n" \
|
|
FIXUP \
|
|
" MOVT D1Ar1,#HI(1b)\n" \
|
|
" JUMP D1Ar1,#LO(1b)\n" \
|
|
" .previous\n" \
|
|
" .section __ex_table,\"a\"\n" \
|
|
TENTRY \
|
|
" .previous\n" \
|
|
: "=r" (to), "=r" (from), "=r" (ret) \
|
|
: "0" (to), "1" (from), "2" (ret) \
|
|
: "D1Ar1", "memory")
|
|
|
|
|
|
#define __asm_copy_to_user_1(to, from, ret) \
|
|
__asm_copy_user_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"2: SETB [%0++],D1Ar1\n", \
|
|
"3: ADD %2,%2,#1\n", \
|
|
" .long 2b,3b\n")
|
|
|
|
#define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_user_cont(to, from, ret, \
|
|
" GETW D1Ar1,[%1++]\n" \
|
|
" SETW [%0],D1Ar1\n" \
|
|
"2: SETW [%0++],D1Ar1\n" COPY, \
|
|
"3: ADD %2,%2,#2\n" FIXUP, \
|
|
" .long 2b,3b\n" TENTRY)
|
|
|
|
#define __asm_copy_to_user_2(to, from, ret) \
|
|
__asm_copy_to_user_2x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_to_user_3(to, from, ret) \
|
|
__asm_copy_to_user_2x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"4: SETB [%0++],D1Ar1\n", \
|
|
"5: ADD %2,%2,#1\n", \
|
|
" .long 4b,5b\n")
|
|
|
|
#define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_user_cont(to, from, ret, \
|
|
" GETD D1Ar1,[%1++]\n" \
|
|
" SETD [%0],D1Ar1\n" \
|
|
"2: SETD [%0++],D1Ar1\n" COPY, \
|
|
"3: ADD %2,%2,#4\n" FIXUP, \
|
|
" .long 2b,3b\n" TENTRY)
|
|
|
|
#define __asm_copy_to_user_4(to, from, ret) \
|
|
__asm_copy_to_user_4x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_to_user_5(to, from, ret) \
|
|
__asm_copy_to_user_4x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"4: SETB [%0++],D1Ar1\n", \
|
|
"5: ADD %2,%2,#1\n", \
|
|
" .long 4b,5b\n")
|
|
|
|
#define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_to_user_4x_cont(to, from, ret, \
|
|
" GETW D1Ar1,[%1++]\n" \
|
|
" SETW [%0],D1Ar1\n" \
|
|
"4: SETW [%0++],D1Ar1\n" COPY, \
|
|
"5: ADD %2,%2,#2\n" FIXUP, \
|
|
" .long 4b,5b\n" TENTRY)
|
|
|
|
#define __asm_copy_to_user_6(to, from, ret) \
|
|
__asm_copy_to_user_6x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_to_user_7(to, from, ret) \
|
|
__asm_copy_to_user_6x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"6: SETB [%0++],D1Ar1\n", \
|
|
"7: ADD %2,%2,#1\n", \
|
|
" .long 6b,7b\n")
|
|
|
|
#define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_to_user_4x_cont(to, from, ret, \
|
|
" GETD D1Ar1,[%1++]\n" \
|
|
" SETD [%0],D1Ar1\n" \
|
|
"4: SETD [%0++],D1Ar1\n" COPY, \
|
|
"5: ADD %2,%2,#4\n" FIXUP, \
|
|
" .long 4b,5b\n" TENTRY)
|
|
|
|
#define __asm_copy_to_user_8(to, from, ret) \
|
|
__asm_copy_to_user_8x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_to_user_9(to, from, ret) \
|
|
__asm_copy_to_user_8x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"6: SETB [%0++],D1Ar1\n", \
|
|
"7: ADD %2,%2,#1\n", \
|
|
" .long 6b,7b\n")
|
|
|
|
#define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_to_user_8x_cont(to, from, ret, \
|
|
" GETW D1Ar1,[%1++]\n" \
|
|
" SETW [%0],D1Ar1\n" \
|
|
"6: SETW [%0++],D1Ar1\n" COPY, \
|
|
"7: ADD %2,%2,#2\n" FIXUP, \
|
|
" .long 6b,7b\n" TENTRY)
|
|
|
|
#define __asm_copy_to_user_10(to, from, ret) \
|
|
__asm_copy_to_user_10x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_to_user_11(to, from, ret) \
|
|
__asm_copy_to_user_10x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"8: SETB [%0++],D1Ar1\n", \
|
|
"9: ADD %2,%2,#1\n", \
|
|
" .long 8b,9b\n")
|
|
|
|
#define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_to_user_8x_cont(to, from, ret, \
|
|
" GETD D1Ar1,[%1++]\n" \
|
|
" SETD [%0],D1Ar1\n" \
|
|
"6: SETD [%0++],D1Ar1\n" COPY, \
|
|
"7: ADD %2,%2,#4\n" FIXUP, \
|
|
" .long 6b,7b\n" TENTRY)
|
|
#define __asm_copy_to_user_12(to, from, ret) \
|
|
__asm_copy_to_user_12x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_to_user_13(to, from, ret) \
|
|
__asm_copy_to_user_12x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"8: SETB [%0++],D1Ar1\n", \
|
|
"9: ADD %2,%2,#1\n", \
|
|
" .long 8b,9b\n")
|
|
|
|
#define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_to_user_12x_cont(to, from, ret, \
|
|
" GETW D1Ar1,[%1++]\n" \
|
|
" SETW [%0],D1Ar1\n" \
|
|
"8: SETW [%0++],D1Ar1\n" COPY, \
|
|
"9: ADD %2,%2,#2\n" FIXUP, \
|
|
" .long 8b,9b\n" TENTRY)
|
|
|
|
#define __asm_copy_to_user_14(to, from, ret) \
|
|
__asm_copy_to_user_14x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_to_user_15(to, from, ret) \
|
|
__asm_copy_to_user_14x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"10: SETB [%0++],D1Ar1\n", \
|
|
"11: ADD %2,%2,#1\n", \
|
|
" .long 10b,11b\n")
|
|
|
|
#define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_to_user_12x_cont(to, from, ret, \
|
|
" GETD D1Ar1,[%1++]\n" \
|
|
" SETD [%0],D1Ar1\n" \
|
|
"8: SETD [%0++],D1Ar1\n" COPY, \
|
|
"9: ADD %2,%2,#4\n" FIXUP, \
|
|
" .long 8b,9b\n" TENTRY)
|
|
|
|
#define __asm_copy_to_user_16(to, from, ret) \
|
|
__asm_copy_to_user_16x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_to_user_8x64(to, from, ret) \
|
|
asm volatile ( \
|
|
" GETL D0Ar2,D1Ar1,[%1++]\n" \
|
|
" SETL [%0],D0Ar2,D1Ar1\n" \
|
|
"2: SETL [%0++],D0Ar2,D1Ar1\n" \
|
|
"1:\n" \
|
|
" .section .fixup,\"ax\"\n" \
|
|
"3: ADD %2,%2,#8\n" \
|
|
" MOVT D0Ar2,#HI(1b)\n" \
|
|
" JUMP D0Ar2,#LO(1b)\n" \
|
|
" .previous\n" \
|
|
" .section __ex_table,\"a\"\n" \
|
|
" .long 2b,3b\n" \
|
|
" .previous\n" \
|
|
: "=r" (to), "=r" (from), "=r" (ret) \
|
|
: "0" (to), "1" (from), "2" (ret) \
|
|
: "D1Ar1", "D0Ar2", "memory")
|
|
|
|
/*
|
|
* optimized copying loop using RAPF when 64 bit aligned
|
|
*
|
|
* n will be automatically decremented inside the loop
|
|
* ret will be left intact. if error occurs we will rewind
|
|
* so that the original non optimized code will fill up
|
|
* this value correctly.
|
|
*
|
|
* on fault:
|
|
* > n will hold total number of uncopied bytes
|
|
*
|
|
* > {'to','from'} will be rewind back so that
|
|
* the non-optimized code will do the proper fix up
|
|
*
|
|
* DCACHE drops the cacheline which helps in reducing cache
|
|
* pollution.
|
|
*
|
|
* We introduce an extra SETL at the end of the loop to
|
|
* ensure we don't fall off the loop before we catch all
|
|
* erros.
|
|
*
|
|
* NOTICE:
|
|
* LSM_STEP in TXSTATUS must be cleared in fix up code.
|
|
* since we're using M{S,G}ETL, a fault might happen at
|
|
* any address in the middle of M{S,G}ETL causing
|
|
* the value of LSM_STEP to be incorrect which can
|
|
* cause subsequent use of M{S,G}ET{L,D} to go wrong.
|
|
* ie: if LSM_STEP was 1 when a fault occurs, the
|
|
* next call to M{S,G}ET{L,D} will skip the first
|
|
* copy/getting as it think that the first 1 has already
|
|
* been done.
|
|
*
|
|
*/
|
|
#define __asm_copy_user_64bit_rapf_loop( \
|
|
to, from, ret, n, id, FIXUP) \
|
|
asm volatile ( \
|
|
".balign 8\n" \
|
|
"MOV RAPF, %1\n" \
|
|
"MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"MOV D0Ar6, #0\n" \
|
|
"LSR D1Ar5, %3, #6\n" \
|
|
"SUB TXRPT, D1Ar5, #2\n" \
|
|
"MOV RAPF, %1\n" \
|
|
"$Lloop"id":\n" \
|
|
"ADD RAPF, %1, #64\n" \
|
|
"21:\n" \
|
|
"MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"22:\n" \
|
|
"MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #32\n" \
|
|
"23:\n" \
|
|
"MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"24:\n" \
|
|
"MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #32\n" \
|
|
"DCACHE [%1+#-64], D0Ar6\n" \
|
|
"BR $Lloop"id"\n" \
|
|
\
|
|
"MOV RAPF, %1\n" \
|
|
"25:\n" \
|
|
"MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"26:\n" \
|
|
"MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #32\n" \
|
|
"27:\n" \
|
|
"MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"28:\n" \
|
|
"MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %0, %0, #8\n" \
|
|
"29:\n" \
|
|
"SETL [%0++], D0.7, D1.7\n" \
|
|
"SUB %3, %3, #32\n" \
|
|
"1:" \
|
|
"DCACHE [%1+#-64], D0Ar6\n" \
|
|
"GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \
|
|
"GETL D0FrT, D1RtP, [A0StP+#-32]\n" \
|
|
"GETL D0.5, D1.5, [A0StP+#-24]\n" \
|
|
"GETL D0.6, D1.6, [A0StP+#-16]\n" \
|
|
"GETL D0.7, D1.7, [A0StP+#-8]\n" \
|
|
"SUB A0StP, A0StP, #40\n" \
|
|
" .section .fixup,\"ax\"\n" \
|
|
"4:\n" \
|
|
" ADD %0, %0, #8\n" \
|
|
"3:\n" \
|
|
" MOV D0Ar2, TXSTATUS\n" \
|
|
" MOV D1Ar1, TXSTATUS\n" \
|
|
" AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \
|
|
" MOV TXSTATUS, D1Ar1\n" \
|
|
FIXUP \
|
|
" MOVT D0Ar2,#HI(1b)\n" \
|
|
" JUMP D0Ar2,#LO(1b)\n" \
|
|
" .previous\n" \
|
|
" .section __ex_table,\"a\"\n" \
|
|
" .long 21b,3b\n" \
|
|
" .long 22b,3b\n" \
|
|
" .long 23b,3b\n" \
|
|
" .long 24b,3b\n" \
|
|
" .long 25b,3b\n" \
|
|
" .long 26b,3b\n" \
|
|
" .long 27b,3b\n" \
|
|
" .long 28b,3b\n" \
|
|
" .long 29b,4b\n" \
|
|
" .previous\n" \
|
|
: "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \
|
|
: "0" (to), "1" (from), "2" (ret), "3" (n) \
|
|
: "D1Ar1", "D0Ar2", "memory")
|
|
|
|
/* rewind 'to' and 'from' pointers when a fault occurs
|
|
*
|
|
* Rationale:
|
|
* A fault always occurs on writing to user buffer. A fault
|
|
* is at a single address, so we need to rewind by only 4
|
|
* bytes.
|
|
* Since we do a complete read from kernel buffer before
|
|
* writing, we need to rewind it also. The amount to be
|
|
* rewind equals the number of faulty writes in MSETD
|
|
* which is: [4 - (LSM_STEP-1)]*8
|
|
* LSM_STEP is bits 10:8 in TXSTATUS which is already read
|
|
* and stored in D0Ar2
|
|
*
|
|
* NOTE: If a fault occurs at the last operation in M{G,S}ETL
|
|
* LSM_STEP will be 0. ie: we do 4 writes in our case, if
|
|
* a fault happens at the 4th write, LSM_STEP will be 0
|
|
* instead of 4. The code copes with that.
|
|
*
|
|
* n is updated by the number of successful writes, which is:
|
|
* n = n - (LSM_STEP-1)*8
|
|
*/
|
|
#define __asm_copy_to_user_64bit_rapf_loop(to, from, ret, n, id)\
|
|
__asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \
|
|
"LSR D0Ar2, D0Ar2, #8\n" \
|
|
"AND D0Ar2, D0Ar2, #0x7\n" \
|
|
"ADDZ D0Ar2, D0Ar2, #4\n" \
|
|
"SUB D0Ar2, D0Ar2, #1\n" \
|
|
"MOV D1Ar1, #4\n" \
|
|
"SUB D0Ar2, D1Ar1, D0Ar2\n" \
|
|
"LSL D0Ar2, D0Ar2, #3\n" \
|
|
"LSL D1Ar1, D1Ar1, #3\n" \
|
|
"SUB D1Ar1, D1Ar1, D0Ar2\n" \
|
|
"SUB %0, %0, #8\n" \
|
|
"SUB %1, %1,D0Ar2\n" \
|
|
"SUB %3, %3, D1Ar1\n")
|
|
|
|
/*
|
|
* optimized copying loop using RAPF when 32 bit aligned
|
|
*
|
|
* n will be automatically decremented inside the loop
|
|
* ret will be left intact. if error occurs we will rewind
|
|
* so that the original non optimized code will fill up
|
|
* this value correctly.
|
|
*
|
|
* on fault:
|
|
* > n will hold total number of uncopied bytes
|
|
*
|
|
* > {'to','from'} will be rewind back so that
|
|
* the non-optimized code will do the proper fix up
|
|
*
|
|
* DCACHE drops the cacheline which helps in reducing cache
|
|
* pollution.
|
|
*
|
|
* We introduce an extra SETD at the end of the loop to
|
|
* ensure we don't fall off the loop before we catch all
|
|
* erros.
|
|
*
|
|
* NOTICE:
|
|
* LSM_STEP in TXSTATUS must be cleared in fix up code.
|
|
* since we're using M{S,G}ETL, a fault might happen at
|
|
* any address in the middle of M{S,G}ETL causing
|
|
* the value of LSM_STEP to be incorrect which can
|
|
* cause subsequent use of M{S,G}ET{L,D} to go wrong.
|
|
* ie: if LSM_STEP was 1 when a fault occurs, the
|
|
* next call to M{S,G}ET{L,D} will skip the first
|
|
* copy/getting as it think that the first 1 has already
|
|
* been done.
|
|
*
|
|
*/
|
|
#define __asm_copy_user_32bit_rapf_loop( \
|
|
to, from, ret, n, id, FIXUP) \
|
|
asm volatile ( \
|
|
".balign 8\n" \
|
|
"MOV RAPF, %1\n" \
|
|
"MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"MOV D0Ar6, #0\n" \
|
|
"LSR D1Ar5, %3, #6\n" \
|
|
"SUB TXRPT, D1Ar5, #2\n" \
|
|
"MOV RAPF, %1\n" \
|
|
"$Lloop"id":\n" \
|
|
"ADD RAPF, %1, #64\n" \
|
|
"21:\n" \
|
|
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"22:\n" \
|
|
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #16\n" \
|
|
"23:\n" \
|
|
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"24:\n" \
|
|
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #16\n" \
|
|
"25:\n" \
|
|
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"26:\n" \
|
|
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #16\n" \
|
|
"27:\n" \
|
|
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"28:\n" \
|
|
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #16\n" \
|
|
"DCACHE [%1+#-64], D0Ar6\n" \
|
|
"BR $Lloop"id"\n" \
|
|
\
|
|
"MOV RAPF, %1\n" \
|
|
"29:\n" \
|
|
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"30:\n" \
|
|
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #16\n" \
|
|
"31:\n" \
|
|
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"32:\n" \
|
|
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #16\n" \
|
|
"33:\n" \
|
|
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"34:\n" \
|
|
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %3, %3, #16\n" \
|
|
"35:\n" \
|
|
"MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
|
|
"36:\n" \
|
|
"MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
|
|
"SUB %0, %0, #4\n" \
|
|
"37:\n" \
|
|
"SETD [%0++], D0.7\n" \
|
|
"SUB %3, %3, #16\n" \
|
|
"1:" \
|
|
"DCACHE [%1+#-64], D0Ar6\n" \
|
|
"GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \
|
|
"GETL D0FrT, D1RtP, [A0StP+#-32]\n" \
|
|
"GETL D0.5, D1.5, [A0StP+#-24]\n" \
|
|
"GETL D0.6, D1.6, [A0StP+#-16]\n" \
|
|
"GETL D0.7, D1.7, [A0StP+#-8]\n" \
|
|
"SUB A0StP, A0StP, #40\n" \
|
|
" .section .fixup,\"ax\"\n" \
|
|
"4:\n" \
|
|
" ADD %0, %0, #4\n" \
|
|
"3:\n" \
|
|
" MOV D0Ar2, TXSTATUS\n" \
|
|
" MOV D1Ar1, TXSTATUS\n" \
|
|
" AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \
|
|
" MOV TXSTATUS, D1Ar1\n" \
|
|
FIXUP \
|
|
" MOVT D0Ar2,#HI(1b)\n" \
|
|
" JUMP D0Ar2,#LO(1b)\n" \
|
|
" .previous\n" \
|
|
" .section __ex_table,\"a\"\n" \
|
|
" .long 21b,3b\n" \
|
|
" .long 22b,3b\n" \
|
|
" .long 23b,3b\n" \
|
|
" .long 24b,3b\n" \
|
|
" .long 25b,3b\n" \
|
|
" .long 26b,3b\n" \
|
|
" .long 27b,3b\n" \
|
|
" .long 28b,3b\n" \
|
|
" .long 29b,3b\n" \
|
|
" .long 30b,3b\n" \
|
|
" .long 31b,3b\n" \
|
|
" .long 32b,3b\n" \
|
|
" .long 33b,3b\n" \
|
|
" .long 34b,3b\n" \
|
|
" .long 35b,3b\n" \
|
|
" .long 36b,3b\n" \
|
|
" .long 37b,4b\n" \
|
|
" .previous\n" \
|
|
: "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \
|
|
: "0" (to), "1" (from), "2" (ret), "3" (n) \
|
|
: "D1Ar1", "D0Ar2", "memory")
|
|
|
|
/* rewind 'to' and 'from' pointers when a fault occurs
|
|
*
|
|
* Rationale:
|
|
* A fault always occurs on writing to user buffer. A fault
|
|
* is at a single address, so we need to rewind by only 4
|
|
* bytes.
|
|
* Since we do a complete read from kernel buffer before
|
|
* writing, we need to rewind it also. The amount to be
|
|
* rewind equals the number of faulty writes in MSETD
|
|
* which is: [4 - (LSM_STEP-1)]*4
|
|
* LSM_STEP is bits 10:8 in TXSTATUS which is already read
|
|
* and stored in D0Ar2
|
|
*
|
|
* NOTE: If a fault occurs at the last operation in M{G,S}ETL
|
|
* LSM_STEP will be 0. ie: we do 4 writes in our case, if
|
|
* a fault happens at the 4th write, LSM_STEP will be 0
|
|
* instead of 4. The code copes with that.
|
|
*
|
|
* n is updated by the number of successful writes, which is:
|
|
* n = n - (LSM_STEP-1)*4
|
|
*/
|
|
#define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\
|
|
__asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \
|
|
"LSR D0Ar2, D0Ar2, #8\n" \
|
|
"AND D0Ar2, D0Ar2, #0x7\n" \
|
|
"ADDZ D0Ar2, D0Ar2, #4\n" \
|
|
"SUB D0Ar2, D0Ar2, #1\n" \
|
|
"MOV D1Ar1, #4\n" \
|
|
"SUB D0Ar2, D1Ar1, D0Ar2\n" \
|
|
"LSL D0Ar2, D0Ar2, #2\n" \
|
|
"LSL D1Ar1, D1Ar1, #2\n" \
|
|
"SUB D1Ar1, D1Ar1, D0Ar2\n" \
|
|
"SUB %0, %0, #4\n" \
|
|
"SUB %1, %1, D0Ar2\n" \
|
|
"SUB %3, %3, D1Ar1\n")
|
|
|
|
unsigned long __copy_user(void __user *pdst, const void *psrc,
|
|
unsigned long n)
|
|
{
|
|
register char __user *dst asm ("A0.2") = pdst;
|
|
register const char *src asm ("A1.2") = psrc;
|
|
unsigned long retn = 0;
|
|
|
|
if (n == 0)
|
|
return 0;
|
|
|
|
if ((unsigned long) src & 1) {
|
|
__asm_copy_to_user_1(dst, src, retn);
|
|
n--;
|
|
}
|
|
if ((unsigned long) dst & 1) {
|
|
/* Worst case - byte copy */
|
|
while (n > 0) {
|
|
__asm_copy_to_user_1(dst, src, retn);
|
|
n--;
|
|
}
|
|
}
|
|
if (((unsigned long) src & 2) && n >= 2) {
|
|
__asm_copy_to_user_2(dst, src, retn);
|
|
n -= 2;
|
|
}
|
|
if ((unsigned long) dst & 2) {
|
|
/* Second worst case - word copy */
|
|
while (n >= 2) {
|
|
__asm_copy_to_user_2(dst, src, retn);
|
|
n -= 2;
|
|
}
|
|
}
|
|
|
|
#ifdef USE_RAPF
|
|
/* 64 bit copy loop */
|
|
if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
|
|
if (n >= RAPF_MIN_BUF_SIZE) {
|
|
/* copy user using 64 bit rapf copy */
|
|
__asm_copy_to_user_64bit_rapf_loop(dst, src, retn,
|
|
n, "64cu");
|
|
}
|
|
while (n >= 8) {
|
|
__asm_copy_to_user_8x64(dst, src, retn);
|
|
n -= 8;
|
|
}
|
|
}
|
|
if (n >= RAPF_MIN_BUF_SIZE) {
|
|
/* copy user using 32 bit rapf copy */
|
|
__asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu");
|
|
}
|
|
#else
|
|
/* 64 bit copy loop */
|
|
if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
|
|
while (n >= 8) {
|
|
__asm_copy_to_user_8x64(dst, src, retn);
|
|
n -= 8;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
while (n >= 16) {
|
|
__asm_copy_to_user_16(dst, src, retn);
|
|
n -= 16;
|
|
}
|
|
|
|
while (n >= 4) {
|
|
__asm_copy_to_user_4(dst, src, retn);
|
|
n -= 4;
|
|
}
|
|
|
|
switch (n) {
|
|
case 0:
|
|
break;
|
|
case 1:
|
|
__asm_copy_to_user_1(dst, src, retn);
|
|
break;
|
|
case 2:
|
|
__asm_copy_to_user_2(dst, src, retn);
|
|
break;
|
|
case 3:
|
|
__asm_copy_to_user_3(dst, src, retn);
|
|
break;
|
|
}
|
|
|
|
return retn;
|
|
}
|
|
EXPORT_SYMBOL(__copy_user);
|
|
|
|
#define __asm_copy_from_user_1(to, from, ret) \
|
|
__asm_copy_user_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
"2: SETB [%0++],D1Ar1\n", \
|
|
"3: ADD %2,%2,#1\n" \
|
|
" SETB [%0++],D1Ar1\n", \
|
|
" .long 2b,3b\n")
|
|
|
|
#define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_user_cont(to, from, ret, \
|
|
" GETW D1Ar1,[%1++]\n" \
|
|
"2: SETW [%0++],D1Ar1\n" COPY, \
|
|
"3: ADD %2,%2,#2\n" \
|
|
" SETW [%0++],D1Ar1\n" FIXUP, \
|
|
" .long 2b,3b\n" TENTRY)
|
|
|
|
#define __asm_copy_from_user_2(to, from, ret) \
|
|
__asm_copy_from_user_2x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_from_user_3(to, from, ret) \
|
|
__asm_copy_from_user_2x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
"4: SETB [%0++],D1Ar1\n", \
|
|
"5: ADD %2,%2,#1\n" \
|
|
" SETB [%0++],D1Ar1\n", \
|
|
" .long 4b,5b\n")
|
|
|
|
#define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_user_cont(to, from, ret, \
|
|
" GETD D1Ar1,[%1++]\n" \
|
|
"2: SETD [%0++],D1Ar1\n" COPY, \
|
|
"3: ADD %2,%2,#4\n" \
|
|
" SETD [%0++],D1Ar1\n" FIXUP, \
|
|
" .long 2b,3b\n" TENTRY)
|
|
|
|
#define __asm_copy_from_user_4(to, from, ret) \
|
|
__asm_copy_from_user_4x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_from_user_5(to, from, ret) \
|
|
__asm_copy_from_user_4x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
"4: SETB [%0++],D1Ar1\n", \
|
|
"5: ADD %2,%2,#1\n" \
|
|
" SETB [%0++],D1Ar1\n", \
|
|
" .long 4b,5b\n")
|
|
|
|
#define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_from_user_4x_cont(to, from, ret, \
|
|
" GETW D1Ar1,[%1++]\n" \
|
|
"4: SETW [%0++],D1Ar1\n" COPY, \
|
|
"5: ADD %2,%2,#2\n" \
|
|
" SETW [%0++],D1Ar1\n" FIXUP, \
|
|
" .long 4b,5b\n" TENTRY)
|
|
|
|
#define __asm_copy_from_user_6(to, from, ret) \
|
|
__asm_copy_from_user_6x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_from_user_7(to, from, ret) \
|
|
__asm_copy_from_user_6x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
"6: SETB [%0++],D1Ar1\n", \
|
|
"7: ADD %2,%2,#1\n" \
|
|
" SETB [%0++],D1Ar1\n", \
|
|
" .long 6b,7b\n")
|
|
|
|
#define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_from_user_4x_cont(to, from, ret, \
|
|
" GETD D1Ar1,[%1++]\n" \
|
|
"4: SETD [%0++],D1Ar1\n" COPY, \
|
|
"5: ADD %2,%2,#4\n" \
|
|
" SETD [%0++],D1Ar1\n" FIXUP, \
|
|
" .long 4b,5b\n" TENTRY)
|
|
|
|
#define __asm_copy_from_user_8(to, from, ret) \
|
|
__asm_copy_from_user_8x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_from_user_9(to, from, ret) \
|
|
__asm_copy_from_user_8x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
"6: SETB [%0++],D1Ar1\n", \
|
|
"7: ADD %2,%2,#1\n" \
|
|
" SETB [%0++],D1Ar1\n", \
|
|
" .long 6b,7b\n")
|
|
|
|
#define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_from_user_8x_cont(to, from, ret, \
|
|
" GETW D1Ar1,[%1++]\n" \
|
|
"6: SETW [%0++],D1Ar1\n" COPY, \
|
|
"7: ADD %2,%2,#2\n" \
|
|
" SETW [%0++],D1Ar1\n" FIXUP, \
|
|
" .long 6b,7b\n" TENTRY)
|
|
|
|
#define __asm_copy_from_user_10(to, from, ret) \
|
|
__asm_copy_from_user_10x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_from_user_11(to, from, ret) \
|
|
__asm_copy_from_user_10x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
"8: SETB [%0++],D1Ar1\n", \
|
|
"9: ADD %2,%2,#1\n" \
|
|
" SETB [%0++],D1Ar1\n", \
|
|
" .long 8b,9b\n")
|
|
|
|
#define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_from_user_8x_cont(to, from, ret, \
|
|
" GETD D1Ar1,[%1++]\n" \
|
|
"6: SETD [%0++],D1Ar1\n" COPY, \
|
|
"7: ADD %2,%2,#4\n" \
|
|
" SETD [%0++],D1Ar1\n" FIXUP, \
|
|
" .long 6b,7b\n" TENTRY)
|
|
|
|
#define __asm_copy_from_user_12(to, from, ret) \
|
|
__asm_copy_from_user_12x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_from_user_13(to, from, ret) \
|
|
__asm_copy_from_user_12x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
"8: SETB [%0++],D1Ar1\n", \
|
|
"9: ADD %2,%2,#1\n" \
|
|
" SETB [%0++],D1Ar1\n", \
|
|
" .long 8b,9b\n")
|
|
|
|
#define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_from_user_12x_cont(to, from, ret, \
|
|
" GETW D1Ar1,[%1++]\n" \
|
|
"8: SETW [%0++],D1Ar1\n" COPY, \
|
|
"9: ADD %2,%2,#2\n" \
|
|
" SETW [%0++],D1Ar1\n" FIXUP, \
|
|
" .long 8b,9b\n" TENTRY)
|
|
|
|
#define __asm_copy_from_user_14(to, from, ret) \
|
|
__asm_copy_from_user_14x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_from_user_15(to, from, ret) \
|
|
__asm_copy_from_user_14x_cont(to, from, ret, \
|
|
" GETB D1Ar1,[%1++]\n" \
|
|
"10: SETB [%0++],D1Ar1\n", \
|
|
"11: ADD %2,%2,#1\n" \
|
|
" SETB [%0++],D1Ar1\n", \
|
|
" .long 10b,11b\n")
|
|
|
|
#define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
|
|
__asm_copy_from_user_12x_cont(to, from, ret, \
|
|
" GETD D1Ar1,[%1++]\n" \
|
|
"8: SETD [%0++],D1Ar1\n" COPY, \
|
|
"9: ADD %2,%2,#4\n" \
|
|
" SETD [%0++],D1Ar1\n" FIXUP, \
|
|
" .long 8b,9b\n" TENTRY)
|
|
|
|
#define __asm_copy_from_user_16(to, from, ret) \
|
|
__asm_copy_from_user_16x_cont(to, from, ret, "", "", "")
|
|
|
|
#define __asm_copy_from_user_8x64(to, from, ret) \
|
|
asm volatile ( \
|
|
" GETL D0Ar2,D1Ar1,[%1++]\n" \
|
|
"2: SETL [%0++],D0Ar2,D1Ar1\n" \
|
|
"1:\n" \
|
|
" .section .fixup,\"ax\"\n" \
|
|
" MOV D1Ar1,#0\n" \
|
|
" MOV D0Ar2,#0\n" \
|
|
"3: ADD %2,%2,#8\n" \
|
|
" SETL [%0++],D0Ar2,D1Ar1\n" \
|
|
" MOVT D0Ar2,#HI(1b)\n" \
|
|
" JUMP D0Ar2,#LO(1b)\n" \
|
|
" .previous\n" \
|
|
" .section __ex_table,\"a\"\n" \
|
|
" .long 2b,3b\n" \
|
|
" .previous\n" \
|
|
: "=a" (to), "=r" (from), "=r" (ret) \
|
|
: "0" (to), "1" (from), "2" (ret) \
|
|
: "D1Ar1", "D0Ar2", "memory")
|
|
|
|
/* rewind 'from' pointer when a fault occurs
|
|
*
|
|
* Rationale:
|
|
* A fault occurs while reading from user buffer, which is the
|
|
* source. Since the fault is at a single address, we only
|
|
* need to rewind by 8 bytes.
|
|
* Since we don't write to kernel buffer until we read first,
|
|
* the kernel buffer is at the right state and needn't be
|
|
* corrected.
|
|
*/
|
|
#define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id) \
|
|
__asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \
|
|
"SUB %1, %1, #8\n")
|
|
|
|
/* rewind 'from' pointer when a fault occurs
|
|
*
|
|
* Rationale:
|
|
* A fault occurs while reading from user buffer, which is the
|
|
* source. Since the fault is at a single address, we only
|
|
* need to rewind by 4 bytes.
|
|
* Since we don't write to kernel buffer until we read first,
|
|
* the kernel buffer is at the right state and needn't be
|
|
* corrected.
|
|
*/
|
|
#define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id) \
|
|
__asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \
|
|
"SUB %1, %1, #4\n")
|
|
|
|
|
|
/* Copy from user to kernel, zeroing the bytes that were inaccessible in
|
|
userland. The return-value is the number of bytes that were
|
|
inaccessible. */
|
|
unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
|
|
unsigned long n)
|
|
{
|
|
register char *dst asm ("A0.2") = pdst;
|
|
register const char __user *src asm ("A1.2") = psrc;
|
|
unsigned long retn = 0;
|
|
|
|
if (n == 0)
|
|
return 0;
|
|
|
|
if ((unsigned long) src & 1) {
|
|
__asm_copy_from_user_1(dst, src, retn);
|
|
n--;
|
|
}
|
|
if ((unsigned long) dst & 1) {
|
|
/* Worst case - byte copy */
|
|
while (n > 0) {
|
|
__asm_copy_from_user_1(dst, src, retn);
|
|
n--;
|
|
if (retn)
|
|
goto copy_exception_bytes;
|
|
}
|
|
}
|
|
if (((unsigned long) src & 2) && n >= 2) {
|
|
__asm_copy_from_user_2(dst, src, retn);
|
|
n -= 2;
|
|
}
|
|
if ((unsigned long) dst & 2) {
|
|
/* Second worst case - word copy */
|
|
while (n >= 2) {
|
|
__asm_copy_from_user_2(dst, src, retn);
|
|
n -= 2;
|
|
if (retn)
|
|
goto copy_exception_bytes;
|
|
}
|
|
}
|
|
|
|
/* We only need one check after the unalignment-adjustments,
|
|
because if both adjustments were done, either both or
|
|
neither reference had an exception. */
|
|
if (retn != 0)
|
|
goto copy_exception_bytes;
|
|
|
|
#ifdef USE_RAPF
|
|
/* 64 bit copy loop */
|
|
if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
|
|
if (n >= RAPF_MIN_BUF_SIZE) {
|
|
/* Copy using fast 64bit rapf */
|
|
__asm_copy_from_user_64bit_rapf_loop(dst, src, retn,
|
|
n, "64cuz");
|
|
}
|
|
while (n >= 8) {
|
|
__asm_copy_from_user_8x64(dst, src, retn);
|
|
n -= 8;
|
|
if (retn)
|
|
goto copy_exception_bytes;
|
|
}
|
|
}
|
|
|
|
if (n >= RAPF_MIN_BUF_SIZE) {
|
|
/* Copy using fast 32bit rapf */
|
|
__asm_copy_from_user_32bit_rapf_loop(dst, src, retn,
|
|
n, "32cuz");
|
|
}
|
|
#else
|
|
/* 64 bit copy loop */
|
|
if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
|
|
while (n >= 8) {
|
|
__asm_copy_from_user_8x64(dst, src, retn);
|
|
n -= 8;
|
|
if (retn)
|
|
goto copy_exception_bytes;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
while (n >= 4) {
|
|
__asm_copy_from_user_4(dst, src, retn);
|
|
n -= 4;
|
|
|
|
if (retn)
|
|
goto copy_exception_bytes;
|
|
}
|
|
|
|
/* If we get here, there were no memory read faults. */
|
|
switch (n) {
|
|
/* These copies are at least "naturally aligned" (so we don't
|
|
have to check each byte), due to the src alignment code.
|
|
The *_3 case *will* get the correct count for retn. */
|
|
case 0:
|
|
/* This case deliberately left in (if you have doubts check the
|
|
generated assembly code). */
|
|
break;
|
|
case 1:
|
|
__asm_copy_from_user_1(dst, src, retn);
|
|
break;
|
|
case 2:
|
|
__asm_copy_from_user_2(dst, src, retn);
|
|
break;
|
|
case 3:
|
|
__asm_copy_from_user_3(dst, src, retn);
|
|
break;
|
|
}
|
|
|
|
/* If we get here, retn correctly reflects the number of failing
|
|
bytes. */
|
|
return retn;
|
|
|
|
copy_exception_bytes:
|
|
/* We already have "retn" bytes cleared, and need to clear the
|
|
remaining "n" bytes. A non-optimized simple byte-for-byte in-line
|
|
memset is preferred here, since this isn't speed-critical code and
|
|
we'd rather have this a leaf-function than calling memset. */
|
|
{
|
|
char *endp;
|
|
for (endp = dst + n; dst < endp; dst++)
|
|
*dst = 0;
|
|
}
|
|
|
|
return retn + n;
|
|
}
|
|
EXPORT_SYMBOL(__copy_user_zeroing);
|
|
|
|
#define __asm_clear_8x64(to, ret) \
|
|
asm volatile ( \
|
|
" MOV D0Ar2,#0\n" \
|
|
" MOV D1Ar1,#0\n" \
|
|
" SETL [%0],D0Ar2,D1Ar1\n" \
|
|
"2: SETL [%0++],D0Ar2,D1Ar1\n" \
|
|
"1:\n" \
|
|
" .section .fixup,\"ax\"\n" \
|
|
"3: ADD %1,%1,#8\n" \
|
|
" MOVT D0Ar2,#HI(1b)\n" \
|
|
" JUMP D0Ar2,#LO(1b)\n" \
|
|
" .previous\n" \
|
|
" .section __ex_table,\"a\"\n" \
|
|
" .long 2b,3b\n" \
|
|
" .previous\n" \
|
|
: "=r" (to), "=r" (ret) \
|
|
: "0" (to), "1" (ret) \
|
|
: "D1Ar1", "D0Ar2", "memory")
|
|
|
|
/* Zero userspace. */
|
|
|
|
#define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \
|
|
asm volatile ( \
|
|
" MOV D1Ar1,#0\n" \
|
|
CLEAR \
|
|
"1:\n" \
|
|
" .section .fixup,\"ax\"\n" \
|
|
FIXUP \
|
|
" MOVT D1Ar1,#HI(1b)\n" \
|
|
" JUMP D1Ar1,#LO(1b)\n" \
|
|
" .previous\n" \
|
|
" .section __ex_table,\"a\"\n" \
|
|
TENTRY \
|
|
" .previous" \
|
|
: "=r" (to), "=r" (ret) \
|
|
: "0" (to), "1" (ret) \
|
|
: "D1Ar1", "memory")
|
|
|
|
#define __asm_clear_1(to, ret) \
|
|
__asm_clear(to, ret, \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"2: SETB [%0++],D1Ar1\n", \
|
|
"3: ADD %1,%1,#1\n", \
|
|
" .long 2b,3b\n")
|
|
|
|
#define __asm_clear_2(to, ret) \
|
|
__asm_clear(to, ret, \
|
|
" SETW [%0],D1Ar1\n" \
|
|
"2: SETW [%0++],D1Ar1\n", \
|
|
"3: ADD %1,%1,#2\n", \
|
|
" .long 2b,3b\n")
|
|
|
|
#define __asm_clear_3(to, ret) \
|
|
__asm_clear(to, ret, \
|
|
"2: SETW [%0++],D1Ar1\n" \
|
|
" SETB [%0],D1Ar1\n" \
|
|
"3: SETB [%0++],D1Ar1\n", \
|
|
"4: ADD %1,%1,#2\n" \
|
|
"5: ADD %1,%1,#1\n", \
|
|
" .long 2b,4b\n" \
|
|
" .long 3b,5b\n")
|
|
|
|
#define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
|
|
__asm_clear(to, ret, \
|
|
" SETD [%0],D1Ar1\n" \
|
|
"2: SETD [%0++],D1Ar1\n" CLEAR, \
|
|
"3: ADD %1,%1,#4\n" FIXUP, \
|
|
" .long 2b,3b\n" TENTRY)
|
|
|
|
#define __asm_clear_4(to, ret) \
|
|
__asm_clear_4x_cont(to, ret, "", "", "")
|
|
|
|
#define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
|
|
__asm_clear_4x_cont(to, ret, \
|
|
" SETD [%0],D1Ar1\n" \
|
|
"4: SETD [%0++],D1Ar1\n" CLEAR, \
|
|
"5: ADD %1,%1,#4\n" FIXUP, \
|
|
" .long 4b,5b\n" TENTRY)
|
|
|
|
#define __asm_clear_8(to, ret) \
|
|
__asm_clear_8x_cont(to, ret, "", "", "")
|
|
|
|
#define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
|
|
__asm_clear_8x_cont(to, ret, \
|
|
" SETD [%0],D1Ar1\n" \
|
|
"6: SETD [%0++],D1Ar1\n" CLEAR, \
|
|
"7: ADD %1,%1,#4\n" FIXUP, \
|
|
" .long 6b,7b\n" TENTRY)
|
|
|
|
#define __asm_clear_12(to, ret) \
|
|
__asm_clear_12x_cont(to, ret, "", "", "")
|
|
|
|
#define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
|
|
__asm_clear_12x_cont(to, ret, \
|
|
" SETD [%0],D1Ar1\n" \
|
|
"8: SETD [%0++],D1Ar1\n" CLEAR, \
|
|
"9: ADD %1,%1,#4\n" FIXUP, \
|
|
" .long 8b,9b\n" TENTRY)
|
|
|
|
#define __asm_clear_16(to, ret) \
|
|
__asm_clear_16x_cont(to, ret, "", "", "")
|
|
|
|
unsigned long __do_clear_user(void __user *pto, unsigned long pn)
|
|
{
|
|
register char __user *dst asm ("D0Re0") = pto;
|
|
register unsigned long n asm ("D1Re0") = pn;
|
|
register unsigned long retn asm ("D0Ar6") = 0;
|
|
|
|
if ((unsigned long) dst & 1) {
|
|
__asm_clear_1(dst, retn);
|
|
n--;
|
|
}
|
|
|
|
if ((unsigned long) dst & 2) {
|
|
__asm_clear_2(dst, retn);
|
|
n -= 2;
|
|
}
|
|
|
|
/* 64 bit copy loop */
|
|
if (!((__force unsigned long) dst & 7)) {
|
|
while (n >= 8) {
|
|
__asm_clear_8x64(dst, retn);
|
|
n -= 8;
|
|
}
|
|
}
|
|
|
|
while (n >= 16) {
|
|
__asm_clear_16(dst, retn);
|
|
n -= 16;
|
|
}
|
|
|
|
while (n >= 4) {
|
|
__asm_clear_4(dst, retn);
|
|
n -= 4;
|
|
}
|
|
|
|
switch (n) {
|
|
case 0:
|
|
break;
|
|
case 1:
|
|
__asm_clear_1(dst, retn);
|
|
break;
|
|
case 2:
|
|
__asm_clear_2(dst, retn);
|
|
break;
|
|
case 3:
|
|
__asm_clear_3(dst, retn);
|
|
break;
|
|
}
|
|
|
|
return retn;
|
|
}
|
|
EXPORT_SYMBOL(__do_clear_user);
|
|
|
|
unsigned char __get_user_asm_b(const void __user *addr, long *err)
|
|
{
|
|
register unsigned char x asm ("D0Re0") = 0;
|
|
asm volatile (
|
|
" GETB %0,[%2]\n"
|
|
"1:\n"
|
|
" GETB %0,[%2]\n"
|
|
"2:\n"
|
|
" .section .fixup,\"ax\"\n"
|
|
"3: MOV D0FrT,%3\n"
|
|
" SETD [%1],D0FrT\n"
|
|
" MOVT D0FrT,#HI(2b)\n"
|
|
" JUMP D0FrT,#LO(2b)\n"
|
|
" .previous\n"
|
|
" .section __ex_table,\"a\"\n"
|
|
" .long 1b,3b\n"
|
|
" .previous\n"
|
|
: "=r" (x)
|
|
: "r" (err), "r" (addr), "P" (-EFAULT)
|
|
: "D0FrT");
|
|
return x;
|
|
}
|
|
EXPORT_SYMBOL(__get_user_asm_b);
|
|
|
|
unsigned short __get_user_asm_w(const void __user *addr, long *err)
|
|
{
|
|
register unsigned short x asm ("D0Re0") = 0;
|
|
asm volatile (
|
|
" GETW %0,[%2]\n"
|
|
"1:\n"
|
|
" GETW %0,[%2]\n"
|
|
"2:\n"
|
|
" .section .fixup,\"ax\"\n"
|
|
"3: MOV D0FrT,%3\n"
|
|
" SETD [%1],D0FrT\n"
|
|
" MOVT D0FrT,#HI(2b)\n"
|
|
" JUMP D0FrT,#LO(2b)\n"
|
|
" .previous\n"
|
|
" .section __ex_table,\"a\"\n"
|
|
" .long 1b,3b\n"
|
|
" .previous\n"
|
|
: "=r" (x)
|
|
: "r" (err), "r" (addr), "P" (-EFAULT)
|
|
: "D0FrT");
|
|
return x;
|
|
}
|
|
EXPORT_SYMBOL(__get_user_asm_w);
|
|
|
|
unsigned int __get_user_asm_d(const void __user *addr, long *err)
|
|
{
|
|
register unsigned int x asm ("D0Re0") = 0;
|
|
asm volatile (
|
|
" GETD %0,[%2]\n"
|
|
"1:\n"
|
|
" GETD %0,[%2]\n"
|
|
"2:\n"
|
|
" .section .fixup,\"ax\"\n"
|
|
"3: MOV D0FrT,%3\n"
|
|
" SETD [%1],D0FrT\n"
|
|
" MOVT D0FrT,#HI(2b)\n"
|
|
" JUMP D0FrT,#LO(2b)\n"
|
|
" .previous\n"
|
|
" .section __ex_table,\"a\"\n"
|
|
" .long 1b,3b\n"
|
|
" .previous\n"
|
|
: "=r" (x)
|
|
: "r" (err), "r" (addr), "P" (-EFAULT)
|
|
: "D0FrT");
|
|
return x;
|
|
}
|
|
EXPORT_SYMBOL(__get_user_asm_d);
|
|
|
|
long __put_user_asm_b(unsigned int x, void __user *addr)
|
|
{
|
|
register unsigned int err asm ("D0Re0") = 0;
|
|
asm volatile (
|
|
" MOV %0,#0\n"
|
|
" SETB [%2],%1\n"
|
|
"1:\n"
|
|
" SETB [%2],%1\n"
|
|
"2:\n"
|
|
".section .fixup,\"ax\"\n"
|
|
"3: MOV %0,%3\n"
|
|
" MOVT D0FrT,#HI(2b)\n"
|
|
" JUMP D0FrT,#LO(2b)\n"
|
|
".previous\n"
|
|
".section __ex_table,\"a\"\n"
|
|
" .long 1b,3b\n"
|
|
".previous"
|
|
: "=r"(err)
|
|
: "d" (x), "a" (addr), "P"(-EFAULT)
|
|
: "D0FrT");
|
|
return err;
|
|
}
|
|
EXPORT_SYMBOL(__put_user_asm_b);
|
|
|
|
long __put_user_asm_w(unsigned int x, void __user *addr)
|
|
{
|
|
register unsigned int err asm ("D0Re0") = 0;
|
|
asm volatile (
|
|
" MOV %0,#0\n"
|
|
" SETW [%2],%1\n"
|
|
"1:\n"
|
|
" SETW [%2],%1\n"
|
|
"2:\n"
|
|
".section .fixup,\"ax\"\n"
|
|
"3: MOV %0,%3\n"
|
|
" MOVT D0FrT,#HI(2b)\n"
|
|
" JUMP D0FrT,#LO(2b)\n"
|
|
".previous\n"
|
|
".section __ex_table,\"a\"\n"
|
|
" .long 1b,3b\n"
|
|
".previous"
|
|
: "=r"(err)
|
|
: "d" (x), "a" (addr), "P"(-EFAULT)
|
|
: "D0FrT");
|
|
return err;
|
|
}
|
|
EXPORT_SYMBOL(__put_user_asm_w);
|
|
|
|
long __put_user_asm_d(unsigned int x, void __user *addr)
|
|
{
|
|
register unsigned int err asm ("D0Re0") = 0;
|
|
asm volatile (
|
|
" MOV %0,#0\n"
|
|
" SETD [%2],%1\n"
|
|
"1:\n"
|
|
" SETD [%2],%1\n"
|
|
"2:\n"
|
|
".section .fixup,\"ax\"\n"
|
|
"3: MOV %0,%3\n"
|
|
" MOVT D0FrT,#HI(2b)\n"
|
|
" JUMP D0FrT,#LO(2b)\n"
|
|
".previous\n"
|
|
".section __ex_table,\"a\"\n"
|
|
" .long 1b,3b\n"
|
|
".previous"
|
|
: "=r"(err)
|
|
: "d" (x), "a" (addr), "P"(-EFAULT)
|
|
: "D0FrT");
|
|
return err;
|
|
}
|
|
EXPORT_SYMBOL(__put_user_asm_d);
|
|
|
|
long __put_user_asm_l(unsigned long long x, void __user *addr)
|
|
{
|
|
register unsigned int err asm ("D0Re0") = 0;
|
|
asm volatile (
|
|
" MOV %0,#0\n"
|
|
" SETL [%2],%1,%t1\n"
|
|
"1:\n"
|
|
" SETL [%2],%1,%t1\n"
|
|
"2:\n"
|
|
".section .fixup,\"ax\"\n"
|
|
"3: MOV %0,%3\n"
|
|
" MOVT D0FrT,#HI(2b)\n"
|
|
" JUMP D0FrT,#LO(2b)\n"
|
|
".previous\n"
|
|
".section __ex_table,\"a\"\n"
|
|
" .long 1b,3b\n"
|
|
".previous"
|
|
: "=r"(err)
|
|
: "d" (x), "a" (addr), "P"(-EFAULT)
|
|
: "D0FrT");
|
|
return err;
|
|
}
|
|
EXPORT_SYMBOL(__put_user_asm_l);
|
|
|
|
long strnlen_user(const char __user *src, long count)
|
|
{
|
|
long res;
|
|
|
|
if (!access_ok(VERIFY_READ, src, 0))
|
|
return 0;
|
|
|
|
asm volatile (" MOV D0Ar4, %1\n"
|
|
" MOV D0Ar6, %2\n"
|
|
"0:\n"
|
|
" SUBS D0FrT, D0Ar6, #0\n"
|
|
" SUB D0Ar6, D0Ar6, #1\n"
|
|
" BLE 2f\n"
|
|
" GETB D0FrT, [D0Ar4+#1++]\n"
|
|
"1:\n"
|
|
" TST D0FrT, #255\n"
|
|
" BNE 0b\n"
|
|
"2:\n"
|
|
" SUB %0, %2, D0Ar6\n"
|
|
"3:\n"
|
|
" .section .fixup,\"ax\"\n"
|
|
"4:\n"
|
|
" MOV %0, #0\n"
|
|
" MOVT D0FrT,#HI(3b)\n"
|
|
" JUMP D0FrT,#LO(3b)\n"
|
|
" .previous\n"
|
|
" .section __ex_table,\"a\"\n"
|
|
" .long 1b,4b\n"
|
|
" .previous\n"
|
|
: "=r" (res)
|
|
: "r" (src), "r" (count)
|
|
: "D0FrT", "D0Ar4", "D0Ar6", "cc");
|
|
|
|
return res;
|
|
}
|
|
EXPORT_SYMBOL(strnlen_user);
|
|
|
|
long __strncpy_from_user(char *dst, const char __user *src, long count)
|
|
{
|
|
long res;
|
|
|
|
if (count == 0)
|
|
return 0;
|
|
|
|
/*
|
|
* Currently, in 2.4.0-test9, most ports use a simple byte-copy loop.
|
|
* So do we.
|
|
*
|
|
* This code is deduced from:
|
|
*
|
|
* char tmp2;
|
|
* long tmp1, tmp3;
|
|
* tmp1 = count;
|
|
* while ((*dst++ = (tmp2 = *src++)) != 0
|
|
* && --tmp1)
|
|
* ;
|
|
*
|
|
* res = count - tmp1;
|
|
*
|
|
* with tweaks.
|
|
*/
|
|
|
|
asm volatile (" MOV %0,%3\n"
|
|
"1:\n"
|
|
" GETB D0FrT,[%2++]\n"
|
|
"2:\n"
|
|
" CMP D0FrT,#0\n"
|
|
" SETB [%1++],D0FrT\n"
|
|
" BEQ 3f\n"
|
|
" SUBS %0,%0,#1\n"
|
|
" BNZ 1b\n"
|
|
"3:\n"
|
|
" SUB %0,%3,%0\n"
|
|
"4:\n"
|
|
" .section .fixup,\"ax\"\n"
|
|
"5:\n"
|
|
" MOV %0,%7\n"
|
|
" MOVT D0FrT,#HI(4b)\n"
|
|
" JUMP D0FrT,#LO(4b)\n"
|
|
" .previous\n"
|
|
" .section __ex_table,\"a\"\n"
|
|
" .long 2b,5b\n"
|
|
" .previous"
|
|
: "=r" (res), "=r" (dst), "=r" (src), "=r" (count)
|
|
: "3" (count), "1" (dst), "2" (src), "P" (-EFAULT)
|
|
: "D0FrT", "memory", "cc");
|
|
|
|
return res;
|
|
}
|
|
EXPORT_SYMBOL(__strncpy_from_user);
|