From 7e168319a52d9c0a4871a1c5a2f4be975550852a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 15 Sep 2013 17:10:43 +0200 Subject: ARM: move AES typedefs and function prototypes to separate header Put the struct definitions for AES keys and the asm function prototypes in a separate header and export the asm functions from the module. This allows other drivers to use them directly. Signed-off-by: Ard Biesheuvel --- arch/arm/crypto/aes_glue.c | 22 ++++++---------------- arch/arm/crypto/aes_glue.h | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 arch/arm/crypto/aes_glue.h diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c index 59f7877ead6a..3003fa1f6fb4 100644 --- a/arch/arm/crypto/aes_glue.c +++ b/arch/arm/crypto/aes_glue.c @@ -6,22 +6,12 @@ #include #include -#define AES_MAXNR 14 +#include "aes_glue.h" -typedef struct { - unsigned int rd_key[4 *(AES_MAXNR + 1)]; - int rounds; -} AES_KEY; - -struct AES_CTX { - AES_KEY enc_key; - AES_KEY dec_key; -}; - -asmlinkage void AES_encrypt(const u8 *in, u8 *out, AES_KEY *ctx); -asmlinkage void AES_decrypt(const u8 *in, u8 *out, AES_KEY *ctx); -asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key); -asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key); +EXPORT_SYMBOL(AES_encrypt); +EXPORT_SYMBOL(AES_decrypt); +EXPORT_SYMBOL(private_AES_set_encrypt_key); +EXPORT_SYMBOL(private_AES_set_decrypt_key); static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { @@ -81,7 +71,7 @@ static struct crypto_alg aes_alg = { .cipher = { .cia_min_keysize = AES_MIN_KEY_SIZE, .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = aes_set_key, + .cia_setkey = aes_set_key, .cia_encrypt = aes_encrypt, .cia_decrypt = aes_decrypt } diff --git a/arch/arm/crypto/aes_glue.h b/arch/arm/crypto/aes_glue.h new file mode 100644 index 000000000000..cca3e51eb606 --- /dev/null +++ b/arch/arm/crypto/aes_glue.h @@ -0,0 +1,19 @@ + +#define AES_MAXNR 14 + +struct AES_KEY { + unsigned int rd_key[4 * (AES_MAXNR + 1)]; + int rounds; +}; + +struct AES_CTX { + struct AES_KEY enc_key; + struct AES_KEY dec_key; +}; + +asmlinkage void AES_encrypt(const u8 *in, u8 *out, struct AES_KEY *ctx); +asmlinkage void AES_decrypt(const u8 *in, u8 *out, struct AES_KEY *ctx); +asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, + const int bits, struct AES_KEY *key); +asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, + const int bits, struct AES_KEY *key); -- cgit v1.2.3 From e1b76588c21b54e97e54f5e6063758c1b7e09be0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 16 Sep 2013 18:31:38 +0200 Subject: ARM: add support for bit sliced AES using NEON instructions Bit sliced AES gives around 45% speedup on Cortex-A15 for encryption and around 25% for decryption. This implementation of the AES algorithm does not rely on any lookup tables so it is believed to be invulnerable to cache timing attacks. This algorithm processes up to 8 blocks in parallel in constant time. This means that it is not usable by chaining modes that are strictly sequential in nature, such as CBC encryption. CBC decryption, however, can benefit from this implementation and runs about 25% faster. The other chaining modes implemented in this module, XTS and CTR, can execute fully in parallel in both directions. The core code has been adopted from the OpenSSL project (in collaboration with the original author, on cc). For ease of maintenance, this version is identical to the upstream OpenSSL code, i.e., all modifications that were required to make it suitable for inclusion into the kernel have been made upstream. The original can be found here: http://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=6f6a6130 Note to integrators: While this implementation is significantly faster than the existing table based ones (generic or ARM asm), especially in CTR mode, the effects on power efficiency are unclear as of yet. This code does fundamentally more work, by calculating values that the table based code obtains by a simple lookup; only by doing all of that work in a SIMD fashion, it manages to perform better. Cc: Andy Polyakov Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel --- arch/arm/crypto/Makefile | 14 +- arch/arm/crypto/aesbs-core.S_shipped | 2544 ++++++++++++++++++++++++++++++++++ arch/arm/crypto/aesbs-glue.c | 434 ++++++ arch/arm/crypto/bsaes-armv7.pl | 2467 +++++++++++++++++++++++++++++++++ crypto/Kconfig | 16 + 5 files changed, 5473 insertions(+), 2 deletions(-) create mode 100644 arch/arm/crypto/aesbs-core.S_shipped create mode 100644 arch/arm/crypto/aesbs-glue.c create mode 100644 arch/arm/crypto/bsaes-armv7.pl diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index a2c83851bc90..81cda39860c5 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -3,7 +3,17 @@ # obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o +obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o -aes-arm-y := aes-armv4.o aes_glue.o -sha1-arm-y := sha1-armv4-large.o sha1_glue.o +aes-arm-y := aes-armv4.o aes_glue.o +aes-arm-bs-y := aesbs-core.o aesbs-glue.o +sha1-arm-y := sha1-armv4-large.o sha1_glue.o + +quiet_cmd_perl = PERL $@ + cmd_perl = $(PERL) $(<) > $(@) + +$(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl + $(call cmd,perl) + +.PRECIOUS: $(obj)/aesbs-core.S diff --git a/arch/arm/crypto/aesbs-core.S_shipped b/arch/arm/crypto/aesbs-core.S_shipped new file mode 100644 index 000000000000..64205d453260 --- /dev/null +++ b/arch/arm/crypto/aesbs-core.S_shipped @@ -0,0 +1,2544 @@ + +@ ==================================================================== +@ Written by Andy Polyakov for the OpenSSL +@ project. The module is, however, dual licensed under OpenSSL and +@ CRYPTOGAMS licenses depending on where you obtain it. For further +@ details see http://www.openssl.org/~appro/cryptogams/. +@ +@ Specific modes and adaptation for Linux kernel by Ard Biesheuvel +@ . Permission to use under GPL terms is +@ granted. +@ ==================================================================== + +@ Bit-sliced AES for ARM NEON +@ +@ February 2012. +@ +@ This implementation is direct adaptation of bsaes-x86_64 module for +@ ARM NEON. Except that this module is endian-neutral [in sense that +@ it can be compiled for either endianness] by courtesy of vld1.8's +@ neutrality. Initial version doesn't implement interface to OpenSSL, +@ only low-level primitives and unsupported entry points, just enough +@ to collect performance results, which for Cortex-A8 core are: +@ +@ encrypt 19.5 cycles per byte processed with 128-bit key +@ decrypt 22.1 cycles per byte processed with 128-bit key +@ key conv. 440 cycles per 128-bit key/0.18 of 8x block +@ +@ Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7, +@ which is [much] worse than anticipated (for further details see +@ http://www.openssl.org/~appro/Snapdragon-S4.html). +@ +@ Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code +@ manages in 20.0 cycles]. +@ +@ When comparing to x86_64 results keep in mind that NEON unit is +@ [mostly] single-issue and thus can't [fully] benefit from +@ instruction-level parallelism. And when comparing to aes-armv4 +@ results keep in mind key schedule conversion overhead (see +@ bsaes-x86_64.pl for further details)... +@ +@ + +@ April-August 2013 +@ +@ Add CBC, CTR and XTS subroutines, adapt for kernel use. +@ +@ + +#ifndef __KERNEL__ +# include "arm_arch.h" + +# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} +# define VFP_ABI_POP vldmia sp!,{d8-d15} +# define VFP_ABI_FRAME 0x40 +#else +# define VFP_ABI_PUSH +# define VFP_ABI_POP +# define VFP_ABI_FRAME 0 +# define BSAES_ASM_EXTENDED_KEY +# define XTS_CHAIN_TWEAK +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +#endif + +#ifdef __thumb__ +# define adrl adr +#endif + +#if __ARM_ARCH__>=7 +.text +.syntax unified @ ARMv7-capable assembler is expected to handle this +#ifdef __thumb2__ +.thumb +#else +.code 32 +#endif + +.fpu neon + +.type _bsaes_decrypt8,%function +.align 4 +_bsaes_decrypt8: + adr r6,_bsaes_decrypt8 + vldmia r4!, {q9} @ round 0 key + add r6,r6,#.LM0ISR-_bsaes_decrypt8 + + vldmia r6!, {q8} @ .LM0ISR + veor q10, q0, q9 @ xor with round0 key + veor q11, q1, q9 + vtbl.8 d0, {q10}, d16 + vtbl.8 d1, {q10}, d17 + veor q12, q2, q9 + vtbl.8 d2, {q11}, d16 + vtbl.8 d3, {q11}, d17 + veor q13, q3, q9 + vtbl.8 d4, {q12}, d16 + vtbl.8 d5, {q12}, d17 + veor q14, q4, q9 + vtbl.8 d6, {q13}, d16 + vtbl.8 d7, {q13}, d17 + veor q15, q5, q9 + vtbl.8 d8, {q14}, d16 + vtbl.8 d9, {q14}, d17 + veor q10, q6, q9 + vtbl.8 d10, {q15}, d16 + vtbl.8 d11, {q15}, d17 + veor q11, q7, q9 + vtbl.8 d12, {q10}, d16 + vtbl.8 d13, {q10}, d17 + vtbl.8 d14, {q11}, d16 + vtbl.8 d15, {q11}, d17 + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q4, #1 + veor q10, q10, q7 + veor q11, q11, q5 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #1 + veor q5, q5, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q3 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q3, q3, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q5, #2 + vshr.u64 q11, q4, #2 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q7, q7, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q5, q5, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q3 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q3, q3, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q3, #4 + vshr.u64 q11, q2, #4 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q6, q6, q11 + vshl.u64 q11, q11, #4 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q5 + veor q11, q11, q4 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q4, q4, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + sub r5,r5,#1 + b .Ldec_sbox +.align 4 +.Ldec_loop: + vldmia r4!, {q8-q11} + veor q8, q8, q0 + veor q9, q9, q1 + vtbl.8 d0, {q8}, d24 + vtbl.8 d1, {q8}, d25 + vldmia r4!, {q8} + veor q10, q10, q2 + vtbl.8 d2, {q9}, d24 + vtbl.8 d3, {q9}, d25 + vldmia r4!, {q9} + veor q11, q11, q3 + vtbl.8 d4, {q10}, d24 + vtbl.8 d5, {q10}, d25 + vldmia r4!, {q10} + vtbl.8 d6, {q11}, d24 + vtbl.8 d7, {q11}, d25 + vldmia r4!, {q11} + veor q8, q8, q4 + veor q9, q9, q5 + vtbl.8 d8, {q8}, d24 + vtbl.8 d9, {q8}, d25 + veor q10, q10, q6 + vtbl.8 d10, {q9}, d24 + vtbl.8 d11, {q9}, d25 + veor q11, q11, q7 + vtbl.8 d12, {q10}, d24 + vtbl.8 d13, {q10}, d25 + vtbl.8 d14, {q11}, d24 + vtbl.8 d15, {q11}, d25 +.Ldec_sbox: + veor q1, q1, q4 + veor q3, q3, q4 + + veor q4, q4, q7 + veor q1, q1, q6 + veor q2, q2, q7 + veor q6, q6, q4 + + veor q0, q0, q1 + veor q2, q2, q5 + veor q7, q7, q6 + veor q3, q3, q0 + veor q5, q5, q0 + veor q1, q1, q3 + veor q11, q3, q0 + veor q10, q7, q4 + veor q9, q1, q6 + veor q13, q4, q0 + vmov q8, q10 + veor q12, q5, q2 + + vorr q10, q10, q9 + veor q15, q11, q8 + vand q14, q11, q12 + vorr q11, q11, q12 + veor q12, q12, q9 + vand q8, q8, q9 + veor q9, q6, q2 + vand q15, q15, q12 + vand q13, q13, q9 + veor q9, q3, q7 + veor q12, q1, q5 + veor q11, q11, q13 + veor q10, q10, q13 + vand q13, q9, q12 + vorr q9, q9, q12 + veor q11, q11, q15 + veor q8, q8, q13 + veor q10, q10, q14 + veor q9, q9, q15 + veor q8, q8, q14 + vand q12, q4, q6 + veor q9, q9, q14 + vand q13, q0, q2 + vand q14, q7, q1 + vorr q15, q3, q5 + veor q11, q11, q12 + veor q9, q9, q14 + veor q8, q8, q15 + veor q10, q10, q13 + + @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3 + + @ new smaller inversion + + vand q14, q11, q9 + vmov q12, q8 + + veor q13, q10, q14 + veor q15, q8, q14 + veor q14, q8, q14 @ q14=q15 + + vbsl q13, q9, q8 + vbsl q15, q11, q10 + veor q11, q11, q10 + + vbsl q12, q13, q14 + vbsl q8, q14, q13 + + vand q14, q12, q15 + veor q9, q9, q8 + + veor q14, q14, q11 + veor q12, q5, q2 + veor q8, q1, q6 + veor q10, q15, q14 + vand q10, q10, q5 + veor q5, q5, q1 + vand q11, q1, q15 + vand q5, q5, q14 + veor q1, q11, q10 + veor q5, q5, q11 + veor q15, q15, q13 + veor q14, q14, q9 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q2 + veor q12, q12, q8 + veor q2, q2, q6 + vand q8, q8, q15 + vand q6, q6, q13 + vand q12, q12, q14 + vand q2, q2, q9 + veor q8, q8, q12 + veor q2, q2, q6 + veor q12, q12, q11 + veor q6, q6, q10 + veor q5, q5, q12 + veor q2, q2, q12 + veor q1, q1, q8 + veor q6, q6, q8 + + veor q12, q3, q0 + veor q8, q7, q4 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q0 + veor q12, q12, q8 + veor q0, q0, q4 + vand q8, q8, q15 + vand q4, q4, q13 + vand q12, q12, q14 + vand q0, q0, q9 + veor q8, q8, q12 + veor q0, q0, q4 + veor q12, q12, q11 + veor q4, q4, q10 + veor q15, q15, q13 + veor q14, q14, q9 + veor q10, q15, q14 + vand q10, q10, q3 + veor q3, q3, q7 + vand q11, q7, q15 + vand q3, q3, q14 + veor q7, q11, q10 + veor q3, q3, q11 + veor q3, q3, q12 + veor q0, q0, q12 + veor q7, q7, q8 + veor q4, q4, q8 + veor q1, q1, q7 + veor q6, q6, q5 + + veor q4, q4, q1 + veor q2, q2, q7 + veor q5, q5, q7 + veor q4, q4, q2 + veor q7, q7, q0 + veor q4, q4, q5 + veor q3, q3, q6 + veor q6, q6, q1 + veor q3, q3, q4 + + veor q4, q4, q0 + veor q7, q7, q3 + subs r5,r5,#1 + bcc .Ldec_done + @ multiplication by 0x05-0x00-0x04-0x00 + vext.8 q8, q0, q0, #8 + vext.8 q14, q3, q3, #8 + vext.8 q15, q5, q5, #8 + veor q8, q8, q0 + vext.8 q9, q1, q1, #8 + veor q14, q14, q3 + vext.8 q10, q6, q6, #8 + veor q15, q15, q5 + vext.8 q11, q4, q4, #8 + veor q9, q9, q1 + vext.8 q12, q2, q2, #8 + veor q10, q10, q6 + vext.8 q13, q7, q7, #8 + veor q11, q11, q4 + veor q12, q12, q2 + veor q13, q13, q7 + + veor q0, q0, q14 + veor q1, q1, q14 + veor q6, q6, q8 + veor q2, q2, q10 + veor q4, q4, q9 + veor q1, q1, q15 + veor q6, q6, q15 + veor q2, q2, q14 + veor q7, q7, q11 + veor q4, q4, q14 + veor q3, q3, q12 + veor q2, q2, q15 + veor q7, q7, q15 + veor q5, q5, q13 + vext.8 q8, q0, q0, #12 @ x0 <<< 32 + vext.8 q9, q1, q1, #12 + veor q0, q0, q8 @ x0 ^ (x0 <<< 32) + vext.8 q10, q6, q6, #12 + veor q1, q1, q9 + vext.8 q11, q4, q4, #12 + veor q6, q6, q10 + vext.8 q12, q2, q2, #12 + veor q4, q4, q11 + vext.8 q13, q7, q7, #12 + veor q2, q2, q12 + vext.8 q14, q3, q3, #12 + veor q7, q7, q13 + vext.8 q15, q5, q5, #12 + veor q3, q3, q14 + + veor q9, q9, q0 + veor q5, q5, q15 + vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor q10, q10, q1 + veor q8, q8, q5 + veor q9, q9, q5 + vext.8 q1, q1, q1, #8 + veor q13, q13, q2 + veor q0, q0, q8 + veor q14, q14, q7 + veor q1, q1, q9 + vext.8 q8, q2, q2, #8 + veor q12, q12, q4 + vext.8 q9, q7, q7, #8 + veor q15, q15, q3 + vext.8 q2, q4, q4, #8 + veor q11, q11, q6 + vext.8 q7, q5, q5, #8 + veor q12, q12, q5 + vext.8 q4, q3, q3, #8 + veor q11, q11, q5 + vext.8 q3, q6, q6, #8 + veor q5, q9, q13 + veor q11, q11, q2 + veor q7, q7, q15 + veor q6, q4, q14 + veor q4, q8, q12 + veor q2, q3, q10 + vmov q3, q11 + @ vmov q5, q9 + vldmia r6, {q12} @ .LISR + ite eq @ Thumb2 thing, sanity check in ARM + addeq r6,r6,#0x10 + bne .Ldec_loop + vldmia r6, {q12} @ .LISRM0 + b .Ldec_loop +.align 4 +.Ldec_done: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q3, #1 + vshr.u64 q11, q2, #1 + veor q10, q10, q5 + veor q11, q11, q7 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #1 + veor q7, q7, q11 + vshl.u64 q11, q11, #1 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q4 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q4, q4, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q7, #2 + vshr.u64 q11, q2, #2 + veor q10, q10, q5 + veor q11, q11, q3 + vand q10, q10, q9 + vand q11, q11, q9 + veor q5, q5, q10 + vshl.u64 q10, q10, #2 + veor q3, q3, q11 + vshl.u64 q11, q11, #2 + veor q7, q7, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q4 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q4, q4, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q4, #4 + vshr.u64 q11, q6, #4 + veor q10, q10, q5 + veor q11, q11, q3 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q3, q3, q11 + vshl.u64 q11, q11, #4 + veor q4, q4, q10 + veor q6, q6, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q7 + veor q11, q11, q2 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q2, q2, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + vldmia r4, {q8} @ last round key + veor q6, q6, q8 + veor q4, q4, q8 + veor q2, q2, q8 + veor q7, q7, q8 + veor q3, q3, q8 + veor q5, q5, q8 + veor q0, q0, q8 + veor q1, q1, q8 + bx lr +.size _bsaes_decrypt8,.-_bsaes_decrypt8 + +.type _bsaes_const,%object +.align 6 +_bsaes_const: +.LM0ISR: @ InvShiftRows constants + .quad 0x0a0e0206070b0f03, 0x0004080c0d010509 +.LISR: + .quad 0x0504070602010003, 0x0f0e0d0c080b0a09 +.LISRM0: + .quad 0x01040b0e0205080f, 0x0306090c00070a0d +.LM0SR: @ ShiftRows constants + .quad 0x0a0e02060f03070b, 0x0004080c05090d01 +.LSR: + .quad 0x0504070600030201, 0x0f0e0d0c0a09080b +.LSRM0: + .quad 0x0304090e00050a0f, 0x01060b0c0207080d +.LM0: + .quad 0x02060a0e03070b0f, 0x0004080c0105090d +.LREVM0SR: + .quad 0x090d01050c000408, 0x03070b0f060a0e02 +.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by " +.align 6 +.size _bsaes_const,.-_bsaes_const + +.type _bsaes_encrypt8,%function +.align 4 +_bsaes_encrypt8: + adr r6,_bsaes_encrypt8 + vldmia r4!, {q9} @ round 0 key + sub r6,r6,#_bsaes_encrypt8-.LM0SR + + vldmia r6!, {q8} @ .LM0SR +_bsaes_encrypt8_alt: + veor q10, q0, q9 @ xor with round0 key + veor q11, q1, q9 + vtbl.8 d0, {q10}, d16 + vtbl.8 d1, {q10}, d17 + veor q12, q2, q9 + vtbl.8 d2, {q11}, d16 + vtbl.8 d3, {q11}, d17 + veor q13, q3, q9 + vtbl.8 d4, {q12}, d16 + vtbl.8 d5, {q12}, d17 + veor q14, q4, q9 + vtbl.8 d6, {q13}, d16 + vtbl.8 d7, {q13}, d17 + veor q15, q5, q9 + vtbl.8 d8, {q14}, d16 + vtbl.8 d9, {q14}, d17 + veor q10, q6, q9 + vtbl.8 d10, {q15}, d16 + vtbl.8 d11, {q15}, d17 + veor q11, q7, q9 + vtbl.8 d12, {q10}, d16 + vtbl.8 d13, {q10}, d17 + vtbl.8 d14, {q11}, d16 + vtbl.8 d15, {q11}, d17 +_bsaes_encrypt8_bitslice: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q4, #1 + veor q10, q10, q7 + veor q11, q11, q5 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #1 + veor q5, q5, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q3 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q3, q3, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q5, #2 + vshr.u64 q11, q4, #2 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q7, q7, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q5, q5, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q3 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q3, q3, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q3, #4 + vshr.u64 q11, q2, #4 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q6, q6, q11 + vshl.u64 q11, q11, #4 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q5 + veor q11, q11, q4 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q4, q4, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + sub r5,r5,#1 + b .Lenc_sbox +.align 4 +.Lenc_loop: + vldmia r4!, {q8-q11} + veor q8, q8, q0 + veor q9, q9, q1 + vtbl.8 d0, {q8}, d24 + vtbl.8 d1, {q8}, d25 + vldmia r4!, {q8} + veor q10, q10, q2 + vtbl.8 d2, {q9}, d24 + vtbl.8 d3, {q9}, d25 + vldmia r4!, {q9} + veor q11, q11, q3 + vtbl.8 d4, {q10}, d24 + vtbl.8 d5, {q10}, d25 + vldmia r4!, {q10} + vtbl.8 d6, {q11}, d24 + vtbl.8 d7, {q11}, d25 + vldmia r4!, {q11} + veor q8, q8, q4 + veor q9, q9, q5 + vtbl.8 d8, {q8}, d24 + vtbl.8 d9, {q8}, d25 + veor q10, q10, q6 + vtbl.8 d10, {q9}, d24 + vtbl.8 d11, {q9}, d25 + veor q11, q11, q7 + vtbl.8 d12, {q10}, d24 + vtbl.8 d13, {q10}, d25 + vtbl.8 d14, {q11}, d24 + vtbl.8 d15, {q11}, d25 +.Lenc_sbox: + veor q2, q2, q1 + veor q5, q5, q6 + veor q3, q3, q0 + veor q6, q6, q2 + veor q5, q5, q0 + + veor q6, q6, q3 + veor q3, q3, q7 + veor q7, q7, q5 + veor q3, q3, q4 + veor q4, q4, q5 + + veor q2, q2, q7 + veor q3, q3, q1 + veor q1, q1, q5 + veor q11, q7, q4 + veor q10, q1, q2 + veor q9, q5, q3 + veor q13, q2, q4 + vmov q8, q10 + veor q12, q6, q0 + + vorr q10, q10, q9 + veor q15, q11, q8 + vand q14, q11, q12 + vorr q11, q11, q12 + veor q12, q12, q9 + vand q8, q8, q9 + veor q9, q3, q0 + vand q15, q15, q12 + vand q13, q13, q9 + veor q9, q7, q1 + veor q12, q5, q6 + veor q11, q11, q13 + veor q10, q10, q13 + vand q13, q9, q12 + vorr q9, q9, q12 + veor q11, q11, q15 + veor q8, q8, q13 + veor q10, q10, q14 + veor q9, q9, q15 + veor q8, q8, q14 + vand q12, q2, q3 + veor q9, q9, q14 + vand q13, q4, q0 + vand q14, q1, q5 + vorr q15, q7, q6 + veor q11, q11, q12 + veor q9, q9, q14 + veor q8, q8, q15 + veor q10, q10, q13 + + @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3 + + @ new smaller inversion + + vand q14, q11, q9 + vmov q12, q8 + + veor q13, q10, q14 + veor q15, q8, q14 + veor q14, q8, q14 @ q14=q15 + + vbsl q13, q9, q8 + vbsl q15, q11, q10 + veor q11, q11, q10 + + vbsl q12, q13, q14 + vbsl q8, q14, q13 + + vand q14, q12, q15 + veor q9, q9, q8 + + veor q14, q14, q11 + veor q12, q6, q0 + veor q8, q5, q3 + veor q10, q15, q14 + vand q10, q10, q6 + veor q6, q6, q5 + vand q11, q5, q15 + vand q6, q6, q14 + veor q5, q11, q10 + veor q6, q6, q11 + veor q15, q15, q13 + veor q14, q14, q9 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q0 + veor q12, q12, q8 + veor q0, q0, q3 + vand q8, q8, q15 + vand q3, q3, q13 + vand q12, q12, q14 + vand q0, q0, q9 + veor q8, q8, q12 + veor q0, q0, q3 + veor q12, q12, q11 + veor q3, q3, q10 + veor q6, q6, q12 + veor q0, q0, q12 + veor q5, q5, q8 + veor q3, q3, q8 + + veor q12, q7, q4 + veor q8, q1, q2 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q4 + veor q12, q12, q8 + veor q4, q4, q2 + vand q8, q8, q15 + vand q2, q2, q13 + vand q12, q12, q14 + vand q4, q4, q9 + veor q8, q8, q12 + veor q4, q4, q2 + veor q12, q12, q11 + veor q2, q2, q10 + veor q15, q15, q13 + veor q14, q14, q9 + veor q10, q15, q14 + vand q10, q10, q7 + veor q7, q7, q1 + vand q11, q1, q15 + vand q7, q7, q14 + veor q1, q11, q10 + veor q7, q7, q11 + veor q7, q7, q12 + veor q4, q4, q12 + veor q1, q1, q8 + veor q2, q2, q8 + veor q7, q7, q0 + veor q1, q1, q6 + veor q6, q6, q0 + veor q4, q4, q7 + veor q0, q0, q1 + + veor q1, q1, q5 + veor q5, q5, q2 + veor q2, q2, q3 + veor q3, q3, q5 + veor q4, q4, q5 + + veor q6, q6, q3 + subs r5,r5,#1 + bcc .Lenc_done + vext.8 q8, q0, q0, #12 @ x0 <<< 32 + vext.8 q9, q1, q1, #12 + veor q0, q0, q8 @ x0 ^ (x0 <<< 32) + vext.8 q10, q4, q4, #12 + veor q1, q1, q9 + vext.8 q11, q6, q6, #12 + veor q4, q4, q10 + vext.8 q12, q3, q3, #12 + veor q6, q6, q11 + vext.8 q13, q7, q7, #12 + veor q3, q3, q12 + vext.8 q14, q2, q2, #12 + veor q7, q7, q13 + vext.8 q15, q5, q5, #12 + veor q2, q2, q14 + + veor q9, q9, q0 + veor q5, q5, q15 + vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor q10, q10, q1 + veor q8, q8, q5 + veor q9, q9, q5 + vext.8 q1, q1, q1, #8 + veor q13, q13, q3 + veor q0, q0, q8 + veor q14, q14, q7 + veor q1, q1, q9 + vext.8 q8, q3, q3, #8 + veor q12, q12, q6 + vext.8 q9, q7, q7, #8 + veor q15, q15, q2 + vext.8 q3, q6, q6, #8 + veor q11, q11, q4 + vext.8 q7, q5, q5, #8 + veor q12, q12, q5 + vext.8 q6, q2, q2, #8 + veor q11, q11, q5 + vext.8 q2, q4, q4, #8 + veor q5, q9, q13 + veor q4, q8, q12 + veor q3, q3, q11 + veor q7, q7, q15 + veor q6, q6, q14 + @ vmov q4, q8 + veor q2, q2, q10 + @ vmov q5, q9 + vldmia r6, {q12} @ .LSR + ite eq @ Thumb2 thing, samity check in ARM + addeq r6,r6,#0x10 + bne .Lenc_loop + vldmia r6, {q12} @ .LSRM0 + b .Lenc_loop +.align 4 +.Lenc_done: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q3, #1 + veor q10, q10, q5 + veor q11, q11, q7 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #1 + veor q7, q7, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q3, q3, q11 + vshr.u64 q10, q4, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q6 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q6, q6, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q4, q4, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q7, #2 + vshr.u64 q11, q3, #2 + veor q10, q10, q5 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q5, q5, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q7, q7, q10 + veor q3, q3, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q6 + veor q11, q11, q4 + vand q10, q10, q9 + vand q11, q11, q9 + veor q6, q6, q10 + vshl.u64 q10, q10, #2 + veor q4, q4, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q6, #4 + vshr.u64 q11, q4, #4 + veor q10, q10, q5 + veor q11, q11, q2 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q2, q2, q11 + vshl.u64 q11, q11, #4 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q7 + veor q11, q11, q3 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q3, q3, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + vldmia r4, {q8} @ last round key + veor q4, q4, q8 + veor q6, q6, q8 + veor q3, q3, q8 + veor q7, q7, q8 + veor q2, q2, q8 + veor q5, q5, q8 + veor q0, q0, q8 + veor q1, q1, q8 + bx lr +.size _bsaes_encrypt8,.-_bsaes_encrypt8 +.type _bsaes_key_convert,%function +.align 4 +_bsaes_key_convert: + adr r6,_bsaes_key_convert + vld1.8 {q7}, [r4]! @ load round 0 key + sub r6,r6,#_bsaes_key_convert-.LM0 + vld1.8 {q15}, [r4]! @ load round 1 key + + vmov.i8 q8, #0x01 @ bit masks + vmov.i8 q9, #0x02 + vmov.i8 q10, #0x04 + vmov.i8 q11, #0x08 + vmov.i8 q12, #0x10 + vmov.i8 q13, #0x20 + vldmia r6, {q14} @ .LM0 + +#ifdef __ARMEL__ + vrev32.8 q7, q7 + vrev32.8 q15, q15 +#endif + sub r5,r5,#1 + vstmia r12!, {q7} @ save round 0 key + b .Lkey_loop + +.align 4 +.Lkey_loop: + vtbl.8 d14,{q15},d28 + vtbl.8 d15,{q15},d29 + vmov.i8 q6, #0x40 + vmov.i8 q15, #0x80 + + vtst.8 q0, q7, q8 + vtst.8 q1, q7, q9 + vtst.8 q2, q7, q10 + vtst.8 q3, q7, q11 + vtst.8 q4, q7, q12 + vtst.8 q5, q7, q13 + vtst.8 q6, q7, q6 + vtst.8 q7, q7, q15 + vld1.8 {q15}, [r4]! @ load next round key + vmvn q0, q0 @ "pnot" + vmvn q1, q1 + vmvn q5, q5 + vmvn q6, q6 +#ifdef __ARMEL__ + vrev32.8 q15, q15 +#endif + subs r5,r5,#1 + vstmia r12!,{q0-q7} @ write bit-sliced round key + bne .Lkey_loop + + vmov.i8 q7,#0x63 @ compose .L63 + @ don't save last round key + bx lr +.size _bsaes_key_convert,.-_bsaes_key_convert +.extern AES_cbc_encrypt +.extern AES_decrypt + +.global bsaes_cbc_encrypt +.type bsaes_cbc_encrypt,%function +.align 5 +bsaes_cbc_encrypt: +#ifndef __KERNEL__ + cmp r2, #128 +#ifndef __thumb__ + blo AES_cbc_encrypt +#else + bhs 1f + b AES_cbc_encrypt +1: +#endif +#endif + + @ it is up to the caller to make sure we are called with enc == 0 + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr r8, [ip] @ IV is 1st arg on the stack + mov r2, r2, lsr#4 @ len in 16 byte blocks + sub sp, #0x10 @ scratch space to carry over the IV + mov r9, sp @ save sp + + ldr r10, [r3, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key + add r12, #96 @ sifze of bit-slices key schedule + + @ populate the key schedule + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + mov sp, r12 @ sp is sp + bl _bsaes_key_convert + vldmia sp, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia sp, {q7} +#else + ldr r12, [r3, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [r3, #244] + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + add r12, r3, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, r3, #248 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} + +.align 2 +0: +#endif + + vld1.8 {q15}, [r8] @ load IV + b .Lcbc_dec_loop + +.align 4 +.Lcbc_dec_loop: + subs r2, r2, #0x8 + bmi .Lcbc_dec_loop_finish + + vld1.8 {q0-q1}, [r0]! @ load input + vld1.8 {q2-q3}, [r0]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, sp @ pass the key +#else + add r4, r3, #248 +#endif + vld1.8 {q4-q5}, [r0]! + mov r5, r10 + vld1.8 {q6-q7}, [r0] + sub r0, r0, #0x60 + vstmia r9, {q15} @ put aside IV + + bl _bsaes_decrypt8 + + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q14-q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q3, q3, q13 + vst1.8 {q6}, [r1]! + veor q5, q5, q14 + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + vst1.8 {q3}, [r1]! + vst1.8 {q5}, [r1]! + + b .Lcbc_dec_loop + +.Lcbc_dec_loop_finish: + adds r2, r2, #8 + beq .Lcbc_dec_done + + vld1.8 {q0}, [r0]! @ load input + cmp r2, #2 + blo .Lcbc_dec_one + vld1.8 {q1}, [r0]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, sp @ pass the key +#else + add r4, r3, #248 +#endif + mov r5, r10 + vstmia r9, {q15} @ put aside IV + beq .Lcbc_dec_two + vld1.8 {q2}, [r0]! + cmp r2, #4 + blo .Lcbc_dec_three + vld1.8 {q3}, [r0]! + beq .Lcbc_dec_four + vld1.8 {q4}, [r0]! + cmp r2, #6 + blo .Lcbc_dec_five + vld1.8 {q5}, [r0]! + beq .Lcbc_dec_six + vld1.8 {q6}, [r0]! + sub r0, r0, #0x70 + + bl _bsaes_decrypt8 + + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q3, q3, q13 + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + vst1.8 {q3}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_six: + sub r0, r0, #0x60 + bl _bsaes_decrypt8 + vldmia r9,{q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_five: + sub r0, r0, #0x50 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q15}, [r0]! + veor q4, q4, q10 + vst1.8 {q0-q1}, [r1]! @ write output + veor q2, q2, q11 + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_four: + sub r0, r0, #0x40 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q15}, [r0]! + veor q4, q4, q10 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_three: + sub r0, r0, #0x30 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q15}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_two: + sub r0, r0, #0x20 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q15}, [r0]! @ reload input + veor q1, q1, q8 + vst1.8 {q0-q1}, [r1]! @ write output + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_one: + sub r0, r0, #0x10 + mov r10, r1 @ save original out pointer + mov r1, r9 @ use the iv scratch space as out buffer + mov r2, r3 + vmov q4,q15 @ just in case ensure that IV + vmov q5,q0 @ and input are preserved + bl AES_decrypt + vld1.8 {q0}, [r9,:64] @ load result + veor q0, q0, q4 @ ^= IV + vmov q15, q5 @ q5 holds input + vst1.8 {q0}, [r10] @ write output + +.Lcbc_dec_done: +#ifndef BSAES_ASM_EXTENDED_KEY + vmov.i32 q0, #0 + vmov.i32 q1, #0 +.Lcbc_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r9 + bne .Lcbc_dec_bzero +#endif + + mov sp, r9 + add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb + vst1.8 {q15}, [r8] @ return IV + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} +.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt +.extern AES_encrypt +.global bsaes_ctr32_encrypt_blocks +.type bsaes_ctr32_encrypt_blocks,%function +.align 5 +bsaes_ctr32_encrypt_blocks: + cmp r2, #8 @ use plain AES for + blo .Lctr_enc_short @ small sizes + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr r8, [ip] @ ctr is 1st arg on the stack + sub sp, sp, #0x10 @ scratch space to carry over the ctr + mov r9, sp @ save sp + + ldr r10, [r3, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key + add r12, #96 @ size of bit-sliced key schedule + + @ populate the key schedule + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + mov sp, r12 @ sp is sp + bl _bsaes_key_convert + veor q7,q7,q15 @ fix up last round key + vstmia r12, {q7} @ save last round key + + vld1.8 {q0}, [r8] @ load counter + add r8, r6, #.LREVM0SR-.LM0 @ borrow r8 + vldmia sp, {q4} @ load round0 key +#else + ldr r12, [r3, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [r3, #244] + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + add r12, r3, #248 @ pass key schedule + bl _bsaes_key_convert + veor q7,q7,q15 @ fix up last round key + vstmia r12, {q7} @ save last round key + +.align 2 +0: add r12, r3, #248 + vld1.8 {q0}, [r8] @ load counter + adrl r8, .LREVM0SR @ borrow r8 + vldmia r12, {q4} @ load round0 key + sub sp, #0x10 @ place for adjusted round0 key +#endif + + vmov.i32 q8,#1 @ compose 1<<96 + veor q9,q9,q9 + vrev32.8 q0,q0 + vext.8 q8,q9,q8,#4 + vrev32.8 q4,q4 + vadd.u32 q9,q8,q8 @ compose 2<<96 + vstmia sp, {q4} @ save adjusted round0 key + b .Lctr_enc_loop + +.align 4 +.Lctr_enc_loop: + vadd.u32 q10, q8, q9 @ compose 3<<96 + vadd.u32 q1, q0, q8 @ +1 + vadd.u32 q2, q0, q9 @ +2 + vadd.u32 q3, q0, q10 @ +3 + vadd.u32 q4, q1, q10 + vadd.u32 q5, q2, q10 + vadd.u32 q6, q3, q10 + vadd.u32 q7, q4, q10 + vadd.u32 q10, q5, q10 @ next counter + + @ Borrow prologue from _bsaes_encrypt8 to use the opportunity + @ to flip byte order in 32-bit counter + + vldmia sp, {q9} @ load round0 key +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x10 @ pass next round key +#else + add r4, r3, #264 +#endif + vldmia r8, {q8} @ .LREVM0SR + mov r5, r10 @ pass rounds + vstmia r9, {q10} @ save next counter + sub r6, r8, #.LREVM0SR-.LSR @ pass constants + + bl _bsaes_encrypt8_alt + + subs r2, r2, #8 + blo .Lctr_enc_loop_done + + vld1.8 {q8-q9}, [r0]! @ load input + vld1.8 {q10-q11}, [r0]! + veor q0, q8 + veor q1, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q10 + veor q6, q11 + vld1.8 {q14-q15}, [r0]! + veor q3, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q7, q13 + veor q2, q14 + vst1.8 {q4}, [r1]! + veor q5, q15 + vst1.8 {q6}, [r1]! + vmov.i32 q8, #1 @ compose 1<<96 + vst1.8 {q3}, [r1]! + veor q9, q9, q9 + vst1.8 {q7}, [r1]! + vext.8 q8, q9, q8, #4 + vst1.8 {q2}, [r1]! + vadd.u32 q9,q8,q8 @ compose 2<<96 + vst1.8 {q5}, [r1]! + vldmia r9, {q0} @ load counter + + bne .Lctr_enc_loop + b .Lctr_enc_done + +.align 4 +.Lctr_enc_loop_done: + add r2, r2, #8 + vld1.8 {q8}, [r0]! @ load input + veor q0, q8 + vst1.8 {q0}, [r1]! @ write output + cmp r2, #2 + blo .Lctr_enc_done + vld1.8 {q9}, [r0]! + veor q1, q9 + vst1.8 {q1}, [r1]! + beq .Lctr_enc_done + vld1.8 {q10}, [r0]! + veor q4, q10 + vst1.8 {q4}, [r1]! + cmp r2, #4 + blo .Lctr_enc_done + vld1.8 {q11}, [r0]! + veor q6, q11 + vst1.8 {q6}, [r1]! + beq .Lctr_enc_done + vld1.8 {q12}, [r0]! + veor q3, q12 + vst1.8 {q3}, [r1]! + cmp r2, #6 + blo .Lctr_enc_done + vld1.8 {q13}, [r0]! + veor q7, q13 + vst1.8 {q7}, [r1]! + beq .Lctr_enc_done + vld1.8 {q14}, [r0] + veor q2, q14 + vst1.8 {q2}, [r1]! + +.Lctr_enc_done: + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifndef BSAES_ASM_EXTENDED_KEY +.Lctr_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r9 + bne .Lctr_enc_bzero +#else + vstmia sp, {q0-q1} +#endif + + mov sp, r9 + add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.align 4 +.Lctr_enc_short: + ldr ip, [sp] @ ctr pointer is passed on stack + stmdb sp!, {r4-r8, lr} + + mov r4, r0 @ copy arguments + mov r5, r1 + mov r6, r2 + mov r7, r3 + ldr r8, [ip, #12] @ load counter LSW + vld1.8 {q1}, [ip] @ load whole counter value +#ifdef __ARMEL__ + rev r8, r8 +#endif + sub sp, sp, #0x10 + vst1.8 {q1}, [sp,:64] @ copy counter value + sub sp, sp, #0x10 + +.Lctr_enc_short_loop: + add r0, sp, #0x10 @ input counter value + mov r1, sp @ output on the stack + mov r2, r7 @ key + + bl AES_encrypt + + vld1.8 {q0}, [r4]! @ load input + vld1.8 {q1}, [sp,:64] @ load encrypted counter + add r8, r8, #1 +#ifdef __ARMEL__ + rev r0, r8 + str r0, [sp, #0x1c] @ next counter value +#else + str r8, [sp, #0x1c] @ next counter value +#endif + veor q0,q0,q1 + vst1.8 {q0}, [r5]! @ store output + subs r6, r6, #1 + bne .Lctr_enc_short_loop + + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vstmia sp!, {q0-q1} + + ldmia sp!, {r4-r8, pc} +.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks +.globl bsaes_xts_encrypt +.type bsaes_xts_encrypt,%function +.align 4 +bsaes_xts_encrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future r3 + + mov r7, r0 + mov r8, r1 + mov r9, r2 + mov r10, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0,sp @ pointer to initial tweak +#endif + + ldr r1, [r10, #240] @ get # of rounds + mov r3, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key + @ add r12, #96 @ size of bit-sliced key schedule + sub r12, #48 @ place for tweak[9] + + @ populate the key schedule + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + veor q7, q7, q15 @ fix up last round key + vstmia r12, {q7} @ save last round key +#else + ldr r12, [r10, #244] + eors r12, #1 + beq 0f + + str r12, [r10, #244] + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + add r12, r10, #248 @ pass key schedule + bl _bsaes_key_convert + veor q7, q7, q15 @ fix up last round key + vstmia r12, {q7} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + + vld1.8 {q8}, [r0] @ initial tweak + adr r2, .Lxts_magic + + subs r9, #0x80 + blo .Lxts_enc_short + b .Lxts_enc_loop + +.align 4 +.Lxts_enc_loop: + vldmia r2, {q5} @ load XTS magic + vshr.s64 q6, q8, #63 + mov r0, sp + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q9, #63 + veor q9, q9, q6 + vand q7, q7, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q10, #63 + veor q10, q10, q7 + vand q6, q6, q5 + vld1.8 {q0}, [r7]! + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q11, #63 + veor q11, q11, q6 + vand q7, q7, q5 + vld1.8 {q1}, [r7]! + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q12, #63 + veor q12, q12, q7 + vand q6, q6, q5 + vld1.8 {q2}, [r7]! + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q13, #63 + veor q13, q13, q6 + vand q7, q7, q5 + vld1.8 {q3}, [r7]! + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q14, #63 + veor q14, q14, q7 + vand q6, q6, q5 + vld1.8 {q4}, [r7]! + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q15, #63 + veor q15, q15, q6 + vand q7, q7, q5 + vld1.8 {q5}, [r7]! + veor q4, q4, q12 + vadd.u64 q8, q15, q15 + vst1.64 {q15}, [r0,:128]! + vswp d15,d14 + veor q8, q8, q7 + vst1.64 {q8}, [r0,:128] @ next round tweak + + vld1.8 {q6-q7}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + veor q7, q7, q15 + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vld1.64 {q14-q15}, [r0,:128]! + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q2, q14 + vst1.8 {q10-q11}, [r8]! + veor q13, q5, q15 + vst1.8 {q12-q13}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + + subs r9, #0x80 + bpl .Lxts_enc_loop + +.Lxts_enc_short: + adds r9, #0x70 + bmi .Lxts_enc_done + + vldmia r2, {q5} @ load XTS magic + vshr.s64 q7, q8, #63 + mov r0, sp + vand q7, q7, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q9, #63 + veor q9, q9, q7 + vand q6, q6, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q10, #63 + veor q10, q10, q6 + vand q7, q7, q5 + vld1.8 {q0}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_1 + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q11, #63 + veor q11, q11, q7 + vand q6, q6, q5 + vld1.8 {q1}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_2 + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q12, #63 + veor q12, q12, q6 + vand q7, q7, q5 + vld1.8 {q2}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_3 + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q13, #63 + veor q13, q13, q7 + vand q6, q6, q5 + vld1.8 {q3}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_4 + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q14, #63 + veor q14, q14, q6 + vand q7, q7, q5 + vld1.8 {q4}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_5 + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q15, #63 + veor q15, q15, q7 + vand q6, q6, q5 + vld1.8 {q5}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_6 + veor q4, q4, q12 + sub r9, #0x10 + vst1.64 {q15}, [r0,:128] @ next round tweak + + vld1.8 {q6}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vld1.64 {q14}, [r0,:128]! + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q2, q14 + vst1.8 {q10-q11}, [r8]! + vst1.8 {q12}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_6: + vst1.64 {q14}, [r0,:128] @ next round tweak + + veor q4, q4, q12 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q5, q5, q13 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + vst1.8 {q10-q11}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done + +@ put this in range for both ARM and Thumb mode adr instructions +.align 5 +.Lxts_magic: + .quad 1, 0x87 + +.align 5 +.Lxts_enc_5: + vst1.64 {q13}, [r0,:128] @ next round tweak + + veor q3, q3, q11 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q4, q4, q12 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + vst1.8 {q10}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_4: + vst1.64 {q12}, [r0,:128] @ next round tweak + + veor q2, q2, q10 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q3, q3, q11 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vst1.8 {q8-q9}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_3: + vst1.64 {q11}, [r0,:128] @ next round tweak + + veor q1, q1, q9 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q2, q2, q10 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + vst1.8 {q8}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_2: + vst1.64 {q10}, [r0,:128] @ next round tweak + + veor q0, q0, q8 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q1, q1, q9 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + vst1.8 {q0-q1}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_1: + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_encrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r8]! + mov r3, r4 + + vmov q8, q9 @ next round tweak + +.Lxts_enc_done: +#ifndef XTS_CHAIN_TWEAK + adds r9, #0x10 + beq .Lxts_enc_ret + sub r6, r8, #0x10 + +.Lxts_enc_steal: + ldrb r0, [r7], #1 + ldrb r1, [r8, #-0x10] + strb r0, [r8, #-0x10] + strb r1, [r8], #1 + + subs r9, #1 + bhi .Lxts_enc_steal + + vld1.8 {q0}, [r6] + mov r0, sp + veor q0, q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_encrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r6] + mov r3, r4 +#endif + +.Lxts_enc_ret: + bic r0, r3, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_enc_bzero + + mov sp, r3 +#ifdef XTS_CHAIN_TWEAK + vst1.8 {q8}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_encrypt,.-bsaes_xts_encrypt + +.globl bsaes_xts_decrypt +.type bsaes_xts_decrypt,%function +.align 4 +bsaes_xts_decrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future r3 + + mov r7, r0 + mov r8, r1 + mov r9, r2 + mov r10, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0, sp @ pointer to initial tweak +#endif + + ldr r1, [r10, #240] @ get # of rounds + mov r3, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key + @ add r12, #96 @ size of bit-sliced key schedule + sub r12, #48 @ place for tweak[9] + + @ populate the key schedule + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + add r4, sp, #0x90 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} +#else + ldr r12, [r10, #244] + eors r12, #1 + beq 0f + + str r12, [r10, #244] + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + add r12, r10, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, r10, #248 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + vld1.8 {q8}, [r0] @ initial tweak + adr r2, .Lxts_magic + + tst r9, #0xf @ if not multiple of 16 + it ne @ Thumb2 thing, sanity check in ARM + subne r9, #0x10 @ subtract another 16 bytes + subs r9, #0x80 + + blo .Lxts_dec_short + b .Lxts_dec_loop + +.align 4 +.Lxts_dec_loop: + vldmia r2, {q5} @ load XTS magic + vshr.s64 q6, q8, #63 + mov r0, sp + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q9, #63 + veor q9, q9, q6 + vand q7, q7, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q10, #63 + veor q10, q10, q7 + vand q6, q6, q5 + vld1.8 {q0}, [r7]! + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q11, #63 + veor q11, q11, q6 + vand q7, q7, q5 + vld1.8 {q1}, [r7]! + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q12, #63 + veor q12, q12, q7 + vand q6, q6, q5 + vld1.8 {q2}, [r7]! + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q13, #63 + veor q13, q13, q6 + vand q7, q7, q5 + vld1.8 {q3}, [r7]! + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q14, #63 + veor q14, q14, q7 + vand q6, q6, q5 + vld1.8 {q4}, [r7]! + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q15, #63 + veor q15, q15, q6 + vand q7, q7, q5 + vld1.8 {q5}, [r7]! + veor q4, q4, q12 + vadd.u64 q8, q15, q15 + vst1.64 {q15}, [r0,:128]! + vswp d15,d14 + veor q8, q8, q7 + vst1.64 {q8}, [r0,:128] @ next round tweak + + vld1.8 {q6-q7}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + veor q7, q7, q15 + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vld1.64 {q14-q15}, [r0,:128]! + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q3, q14 + vst1.8 {q10-q11}, [r8]! + veor q13, q5, q15 + vst1.8 {q12-q13}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + + subs r9, #0x80 + bpl .Lxts_dec_loop + +.Lxts_dec_short: + adds r9, #0x70 + bmi .Lxts_dec_done + + vldmia r2, {q5} @ load XTS magic + vshr.s64 q7, q8, #63 + mov r0, sp + vand q7, q7, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q9, #63 + veor q9, q9, q7 + vand q6, q6, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q10, #63 + veor q10, q10, q6 + vand q7, q7, q5 + vld1.8 {q0}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_1 + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q11, #63 + veor q11, q11, q7 + vand q6, q6, q5 + vld1.8 {q1}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_2 + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q12, #63 + veor q12, q12, q6 + vand q7, q7, q5 + vld1.8 {q2}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_3 + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q13, #63 + veor q13, q13, q7 + vand q6, q6, q5 + vld1.8 {q3}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_4 + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q14, #63 + veor q14, q14, q6 + vand q7, q7, q5 + vld1.8 {q4}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_5 + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q15, #63 + veor q15, q15, q7 + vand q6, q6, q5 + vld1.8 {q5}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_6 + veor q4, q4, q12 + sub r9, #0x10 + vst1.64 {q15}, [r0,:128] @ next round tweak + + vld1.8 {q6}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vld1.64 {q14}, [r0,:128]! + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q3, q14 + vst1.8 {q10-q11}, [r8]! + vst1.8 {q12}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_6: + vst1.64 {q14}, [r0,:128] @ next round tweak + + veor q4, q4, q12 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q5, q5, q13 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + vst1.8 {q10-q11}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_5: + vst1.64 {q13}, [r0,:128] @ next round tweak + + veor q3, q3, q11 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q4, q4, q12 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + vst1.8 {q10}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_4: + vst1.64 {q12}, [r0,:128] @ next round tweak + + veor q2, q2, q10 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q3, q3, q11 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vst1.8 {q8-q9}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_3: + vst1.64 {q11}, [r0,:128] @ next round tweak + + veor q1, q1, q9 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q2, q2, q10 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + vst1.8 {q8}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_2: + vst1.64 {q10}, [r0,:128] @ next round tweak + + veor q0, q0, q8 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q1, q1, q9 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + vst1.8 {q0-q1}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_1: + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + mov r5, r2 @ preserve magic + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r8]! + mov r3, r4 + mov r2, r5 + + vmov q8, q9 @ next round tweak + +.Lxts_dec_done: +#ifndef XTS_CHAIN_TWEAK + adds r9, #0x10 + beq .Lxts_dec_ret + + @ calculate one round of extra tweak for the stolen ciphertext + vldmia r2, {q5} + vshr.s64 q6, q8, #63 + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vswp d13,d12 + veor q9, q9, q6 + + @ perform the final decryption with the last tweak value + vld1.8 {q0}, [r7]! + mov r0, sp + veor q0, q0, q9 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q9 + vst1.8 {q0}, [r8] + + mov r6, r8 +.Lxts_dec_steal: + ldrb r1, [r8] + ldrb r0, [r7], #1 + strb r1, [r8, #0x10] + strb r0, [r8], #1 + + subs r9, #1 + bhi .Lxts_dec_steal + + vld1.8 {q0}, [r6] + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r6] + mov r3, r4 +#endif + +.Lxts_dec_ret: + bic r0, r3, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_dec_bzero + + mov sp, r3 +#ifdef XTS_CHAIN_TWEAK + vst1.8 {q8}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_decrypt,.-bsaes_xts_decrypt +#endif diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c new file mode 100644 index 000000000000..4522366da759 --- /dev/null +++ b/arch/arm/crypto/aesbs-glue.c @@ -0,0 +1,434 @@ +/* + * linux/arch/arm/crypto/aesbs-glue.c - glue code for NEON bit sliced AES + * + * Copyright (C) 2013 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include + +#include "aes_glue.h" + +#define BIT_SLICED_KEY_MAXSIZE (128 * (AES_MAXNR - 1) + 2 * AES_BLOCK_SIZE) + +struct BS_KEY { + struct AES_KEY rk; + int converted; + u8 __aligned(8) bs[BIT_SLICED_KEY_MAXSIZE]; +} __aligned(8); + +asmlinkage void bsaes_enc_key_convert(u8 out[], struct AES_KEY const *in); +asmlinkage void bsaes_dec_key_convert(u8 out[], struct AES_KEY const *in); + +asmlinkage void bsaes_cbc_encrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 iv[]); + +asmlinkage void bsaes_ctr32_encrypt_blocks(u8 const in[], u8 out[], u32 blocks, + struct BS_KEY *key, u8 const iv[]); + +asmlinkage void bsaes_xts_encrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 tweak[]); + +asmlinkage void bsaes_xts_decrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 tweak[]); + +struct aesbs_cbc_ctx { + struct AES_KEY enc; + struct BS_KEY dec; +}; + +struct aesbs_ctr_ctx { + struct BS_KEY enc; +}; + +struct aesbs_xts_ctx { + struct BS_KEY enc; + struct BS_KEY dec; + struct AES_KEY twkey; +}; + +static int aesbs_cbc_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 8; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->dec.rk = ctx->enc; + private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk); + ctx->dec.converted = 0; + return 0; +} + +static int aesbs_ctr_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 8; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->enc.converted = 0; + return 0; +} + +static int aesbs_xts_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 4; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->dec.rk = ctx->enc.rk; + private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk); + private_AES_set_encrypt_key(in_key + key_len / 2, bits, &ctx->twkey); + ctx->enc.converted = ctx->dec.converted = 0; + return 0; +} + +static int aesbs_cbc_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while (walk.nbytes) { + u32 blocks = walk.nbytes / AES_BLOCK_SIZE; + u8 *src = walk.src.virt.addr; + + if (walk.dst.virt.addr == walk.src.virt.addr) { + u8 *iv = walk.iv; + + do { + crypto_xor(src, iv, AES_BLOCK_SIZE); + AES_encrypt(src, src, &ctx->enc); + iv = src; + src += AES_BLOCK_SIZE; + } while (--blocks); + memcpy(walk.iv, iv, AES_BLOCK_SIZE); + } else { + u8 *dst = walk.dst.virt.addr; + + do { + crypto_xor(walk.iv, src, AES_BLOCK_SIZE); + AES_encrypt(walk.iv, dst, &ctx->enc); + memcpy(walk.iv, dst, AES_BLOCK_SIZE); + src += AES_BLOCK_SIZE; + dst += AES_BLOCK_SIZE; + } while (--blocks); + } + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static int aesbs_cbc_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + while ((walk.nbytes / AES_BLOCK_SIZE) >= 8) { + kernel_neon_begin(); + bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->dec, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, 0); + } + while (walk.nbytes) { + u32 blocks = walk.nbytes / AES_BLOCK_SIZE; + u8 *dst = walk.dst.virt.addr; + u8 *src = walk.src.virt.addr; + u8 bk[2][AES_BLOCK_SIZE]; + u8 *iv = walk.iv; + + do { + if (walk.dst.virt.addr == walk.src.virt.addr) + memcpy(bk[blocks & 1], src, AES_BLOCK_SIZE); + + AES_decrypt(src, dst, &ctx->dec.rk); + crypto_xor(dst, iv, AES_BLOCK_SIZE); + + if (walk.dst.virt.addr == walk.src.virt.addr) + iv = bk[blocks & 1]; + else + iv = src; + + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } while (--blocks); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static void inc_be128_ctr(__be32 ctr[], u32 addend) +{ + int i; + + for (i = 3; i >= 0; i--, addend = 1) { + u32 n = be32_to_cpu(ctr[i]) + addend; + + ctr[i] = cpu_to_be32(n); + if (n >= addend) + break; + } +} + +static int aesbs_ctr_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct aesbs_ctr_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + u32 blocks; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) { + u32 tail = walk.nbytes % AES_BLOCK_SIZE; + __be32 *ctr = (__be32 *)walk.iv; + u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]); + + /* avoid 32 bit counter overflow in the NEON code */ + if (unlikely(headroom < blocks)) { + blocks = headroom + 1; + tail = walk.nbytes - blocks * AES_BLOCK_SIZE; + } + kernel_neon_begin(); + bsaes_ctr32_encrypt_blocks(walk.src.virt.addr, + walk.dst.virt.addr, blocks, + &ctx->enc, walk.iv); + kernel_neon_end(); + inc_be128_ctr(ctr, blocks); + + nbytes -= blocks * AES_BLOCK_SIZE; + if (nbytes && nbytes == tail && nbytes <= AES_BLOCK_SIZE) + break; + + err = blkcipher_walk_done(desc, &walk, tail); + } + if (walk.nbytes) { + u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; + u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; + u8 ks[AES_BLOCK_SIZE]; + + AES_encrypt(walk.iv, ks, &ctx->enc.rk); + if (tdst != tsrc) + memcpy(tdst, tsrc, nbytes); + crypto_xor(tdst, ks, nbytes); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static int aesbs_xts_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + /* generate the initial tweak */ + AES_encrypt(walk.iv, walk.iv, &ctx->twkey); + + while (walk.nbytes) { + kernel_neon_begin(); + bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->enc, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static int aesbs_xts_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + /* generate the initial tweak */ + AES_encrypt(walk.iv, walk.iv, &ctx->twkey); + + while (walk.nbytes) { + kernel_neon_begin(); + bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->dec, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static struct crypto_alg aesbs_algs[] = { { + .cra_name = "__cbc-aes-neonbs", + .cra_driver_name = "__driver-cbc-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aesbs_cbc_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_cbc_set_key, + .encrypt = aesbs_cbc_encrypt, + .decrypt = aesbs_cbc_decrypt, + }, +}, { + .cra_name = "__ctr-aes-neonbs", + .cra_driver_name = "__driver-ctr-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aesbs_ctr_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_ctr_set_key, + .encrypt = aesbs_ctr_encrypt, + .decrypt = aesbs_ctr_encrypt, + }, +}, { + .cra_name = "__xts-aes-neonbs", + .cra_driver_name = "__driver-xts-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aesbs_xts_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_xts_set_key, + .encrypt = aesbs_xts_encrypt, + .decrypt = aesbs_xts_decrypt, + }, +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +} }; + +static int __init aesbs_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs)); +} + +static void __exit aesbs_mod_exit(void) +{ + crypto_unregister_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs)); +} + +module_init(aesbs_mod_init); +module_exit(aesbs_mod_exit); + +MODULE_DESCRIPTION("Bit sliced AES in CBC/CTR/XTS modes using NEON"); +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_LICENSE("GPL"); diff --git a/arch/arm/crypto/bsaes-armv7.pl b/arch/arm/crypto/bsaes-armv7.pl new file mode 100644 index 000000000000..f3d96d932573 --- /dev/null +++ b/arch/arm/crypto/bsaes-armv7.pl @@ -0,0 +1,2467 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# +# Specific modes and adaptation for Linux kernel by Ard Biesheuvel +# . Permission to use under GPL terms is +# granted. +# ==================================================================== + +# Bit-sliced AES for ARM NEON +# +# February 2012. +# +# This implementation is direct adaptation of bsaes-x86_64 module for +# ARM NEON. Except that this module is endian-neutral [in sense that +# it can be compiled for either endianness] by courtesy of vld1.8's +# neutrality. Initial version doesn't implement interface to OpenSSL, +# only low-level primitives and unsupported entry points, just enough +# to collect performance results, which for Cortex-A8 core are: +# +# encrypt 19.5 cycles per byte processed with 128-bit key +# decrypt 22.1 cycles per byte processed with 128-bit key +# key conv. 440 cycles per 128-bit key/0.18 of 8x block +# +# Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7, +# which is [much] worse than anticipated (for further details see +# http://www.openssl.org/~appro/Snapdragon-S4.html). +# +# Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code +# manages in 20.0 cycles]. +# +# When comparing to x86_64 results keep in mind that NEON unit is +# [mostly] single-issue and thus can't [fully] benefit from +# instruction-level parallelism. And when comparing to aes-armv4 +# results keep in mind key schedule conversion overhead (see +# bsaes-x86_64.pl for further details)... +# +# + +# April-August 2013 +# +# Add CBC, CTR and XTS subroutines, adapt for kernel use. +# +# + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +my ($inp,$out,$len,$key)=("r0","r1","r2","r3"); +my @XMM=map("q$_",(0..15)); + +{ +my ($key,$rounds,$const)=("r4","r5","r6"); + +sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } +sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } + +sub Sbox { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb +my @b=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; + &InBasisChange (@b); + &Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s); + &OutBasisChange (@b[7,1,4,2,6,5,0,3]); +} + +sub InBasisChange { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb +my @b=@_[0..7]; +$code.=<<___; + veor @b[2], @b[2], @b[1] + veor @b[5], @b[5], @b[6] + veor @b[3], @b[3], @b[0] + veor @b[6], @b[6], @b[2] + veor @b[5], @b[5], @b[0] + + veor @b[6], @b[6], @b[3] + veor @b[3], @b[3], @b[7] + veor @b[7], @b[7], @b[5] + veor @b[3], @b[3], @b[4] + veor @b[4], @b[4], @b[5] + + veor @b[2], @b[2], @b[7] + veor @b[3], @b[3], @b[1] + veor @b[1], @b[1], @b[5] +___ +} + +sub OutBasisChange { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb +my @b=@_[0..7]; +$code.=<<___; + veor @b[0], @b[0], @b[6] + veor @b[1], @b[1], @b[4] + veor @b[4], @b[4], @b[6] + veor @b[2], @b[2], @b[0] + veor @b[6], @b[6], @b[1] + + veor @b[1], @b[1], @b[5] + veor @b[5], @b[5], @b[3] + veor @b[3], @b[3], @b[7] + veor @b[7], @b[7], @b[5] + veor @b[2], @b[2], @b[5] + + veor @b[4], @b[4], @b[7] +___ +} + +sub InvSbox { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb +my @b=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; + &InvInBasisChange (@b); + &Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s); + &InvOutBasisChange (@b[3,7,0,4,5,1,2,6]); +} + +sub InvInBasisChange { # OutBasisChange in reverse (with twist) +my @b=@_[5,1,2,6,3,7,0,4]; +$code.=<<___ + veor @b[1], @b[1], @b[7] + veor @b[4], @b[4], @b[7] + + veor @b[7], @b[7], @b[5] + veor @b[1], @b[1], @b[3] + veor @b[2], @b[2], @b[5] + veor @b[3], @b[3], @b[7] + + veor @b[6], @b[6], @b[1] + veor @b[2], @b[2], @b[0] + veor @b[5], @b[5], @b[3] + veor @b[4], @b[4], @b[6] + veor @b[0], @b[0], @b[6] + veor @b[1], @b[1], @b[4] +___ +} + +sub InvOutBasisChange { # InBasisChange in reverse +my @b=@_[2,5,7,3,6,1,0,4]; +$code.=<<___; + veor @b[1], @b[1], @b[5] + veor @b[2], @b[2], @b[7] + + veor @b[3], @b[3], @b[1] + veor @b[4], @b[4], @b[5] + veor @b[7], @b[7], @b[5] + veor @b[3], @b[3], @b[4] + veor @b[5], @b[5], @b[0] + veor @b[3], @b[3], @b[7] + veor @b[6], @b[6], @b[2] + veor @b[2], @b[2], @b[1] + veor @b[6], @b[6], @b[3] + + veor @b[3], @b[3], @b[0] + veor @b[5], @b[5], @b[6] +___ +} + +sub Mul_GF4 { +#;************************************************************* +#;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) * +#;************************************************************* +my ($x0,$x1,$y0,$y1,$t0,$t1)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + vand $t0, $t0, $x0 + veor $x0, $x0, $x1 + vand $t1, $x1, $y0 + vand $x0, $x0, $y1 + veor $x1, $t1, $t0 + veor $x0, $x0, $t1 +___ +} + +sub Mul_GF4_N { # not used, see next subroutine +# multiply and scale by N +my ($x0,$x1,$y0,$y1,$t0)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + vand $t0, $t0, $x0 + veor $x0, $x0, $x1 + vand $x1, $x1, $y0 + vand $x0, $x0, $y1 + veor $x1, $x1, $x0 + veor $x0, $x0, $t0 +___ +} + +sub Mul_GF4_N_GF4 { +# interleaved Mul_GF4_N and Mul_GF4 +my ($x0,$x1,$y0,$y1,$t0, + $x2,$x3,$y2,$y3,$t1)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + veor $t1, $y2, $y3 + vand $t0, $t0, $x0 + vand $t1, $t1, $x2 + veor $x0, $x0, $x1 + veor $x2, $x2, $x3 + vand $x1, $x1, $y0 + vand $x3, $x3, $y2 + vand $x0, $x0, $y1 + vand $x2, $x2, $y3 + veor $x1, $x1, $x0 + veor $x2, $x2, $x3 + veor $x0, $x0, $t0 + veor $x3, $x3, $t1 +___ +} +sub Mul_GF16_2 { +my @x=@_[0..7]; +my @y=@_[8..11]; +my @t=@_[12..15]; +$code.=<<___; + veor @t[0], @x[0], @x[2] + veor @t[1], @x[1], @x[3] +___ + &Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2..3]); +$code.=<<___; + veor @y[0], @y[0], @y[2] + veor @y[1], @y[1], @y[3] +___ + Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], + @x[2], @x[3], @y[2], @y[3], @t[2]); +$code.=<<___; + veor @x[0], @x[0], @t[0] + veor @x[2], @x[2], @t[0] + veor @x[1], @x[1], @t[1] + veor @x[3], @x[3], @t[1] + + veor @t[0], @x[4], @x[6] + veor @t[1], @x[5], @x[7] +___ + &Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], + @x[6], @x[7], @y[2], @y[3], @t[2]); +$code.=<<___; + veor @y[0], @y[0], @y[2] + veor @y[1], @y[1], @y[3] +___ + &Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[2..3]); +$code.=<<___; + veor @x[4], @x[4], @t[0] + veor @x[6], @x[6], @t[0] + veor @x[5], @x[5], @t[1] + veor @x[7], @x[7], @t[1] +___ +} +sub Inv_GF256 { +#;******************************************************************** +#;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) * +#;******************************************************************** +my @x=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; +# direct optimizations from hardware +$code.=<<___; + veor @t[3], @x[4], @x[6] + veor @t[2], @x[5], @x[7] + veor @t[1], @x[1], @x[3] + veor @s[1], @x[7], @x[6] + vmov @t[0], @t[2] + veor @s[0], @x[0], @x[2] + + vorr @t[2], @t[2], @t[1] + veor @s[3], @t[3], @t[0] + vand @s[2], @t[3], @s[0] + vorr @t[3], @t[3], @s[0] + veor @s[0], @s[0], @t[1] + vand @t[0], @t[0], @t[1] + veor @t[1], @x[3], @x[2] + vand @s[3], @s[3], @s[0] + vand @s[1], @s[1], @t[1] + veor @t[1], @x[4], @x[5] + veor @s[0], @x[1], @x[0] + veor @t[3], @t[3], @s[1] + veor @t[2], @t[2], @s[1] + vand @s[1], @t[1], @s[0] + vorr @t[1], @t[1], @s[0] + veor @t[3], @t[3], @s[3] + veor @t[0], @t[0], @s[1] + veor @t[2], @t[2], @s[2] + veor @t[1], @t[1], @s[3] + veor @t[0], @t[0], @s[2] + vand @s[0], @x[7], @x[3] + veor @t[1], @t[1], @s[2] + vand @s[1], @x[6], @x[2] + vand @s[2], @x[5], @x[1] + vorr @s[3], @x[4], @x[0] + veor @t[3], @t[3], @s[0] + veor @t[1], @t[1], @s[2] + veor @t[0], @t[0], @s[3] + veor @t[2], @t[2], @s[1] + + @ Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3 + + @ new smaller inversion + + vand @s[2], @t[3], @t[1] + vmov @s[0], @t[0] + + veor @s[1], @t[2], @s[2] + veor @s[3], @t[0], @s[2] + veor @s[2], @t[0], @s[2] @ @s[2]=@s[3] + + vbsl @s[1], @t[1], @t[0] + vbsl @s[3], @t[3], @t[2] + veor @t[3], @t[3], @t[2] + + vbsl @s[0], @s[1], @s[2] + vbsl @t[0], @s[2], @s[1] + + vand @s[2], @s[0], @s[3] + veor @t[1], @t[1], @t[0] + + veor @s[2], @s[2], @t[3] +___ +# output in s3, s2, s1, t1 + +# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3 + +# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3 + &Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]); + +### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb +} + +# AES linear components + +sub ShiftRows { +my @x=@_[0..7]; +my @t=@_[8..11]; +my $mask=pop; +$code.=<<___; + vldmia $key!, {@t[0]-@t[3]} + veor @t[0], @t[0], @x[0] + veor @t[1], @t[1], @x[1] + vtbl.8 `&Dlo(@x[0])`, {@t[0]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[0])`, {@t[0]}, `&Dhi($mask)` + vldmia $key!, {@t[0]} + veor @t[2], @t[2], @x[2] + vtbl.8 `&Dlo(@x[1])`, {@t[1]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[1])`, {@t[1]}, `&Dhi($mask)` + vldmia $key!, {@t[1]} + veor @t[3], @t[3], @x[3] + vtbl.8 `&Dlo(@x[2])`, {@t[2]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[2])`, {@t[2]}, `&Dhi($mask)` + vldmia $key!, {@t[2]} + vtbl.8 `&Dlo(@x[3])`, {@t[3]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[3])`, {@t[3]}, `&Dhi($mask)` + vldmia $key!, {@t[3]} + veor @t[0], @t[0], @x[4] + veor @t[1], @t[1], @x[5] + vtbl.8 `&Dlo(@x[4])`, {@t[0]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[4])`, {@t[0]}, `&Dhi($mask)` + veor @t[2], @t[2], @x[6] + vtbl.8 `&Dlo(@x[5])`, {@t[1]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[5])`, {@t[1]}, `&Dhi($mask)` + veor @t[3], @t[3], @x[7] + vtbl.8 `&Dlo(@x[6])`, {@t[2]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[6])`, {@t[2]}, `&Dhi($mask)` + vtbl.8 `&Dlo(@x[7])`, {@t[3]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[7])`, {@t[3]}, `&Dhi($mask)` +___ +} + +sub MixColumns { +# modified to emit output in order suitable for feeding back to aesenc[last] +my @x=@_[0..7]; +my @t=@_[8..15]; +my $inv=@_[16]; # optional +$code.=<<___; + vext.8 @t[0], @x[0], @x[0], #12 @ x0 <<< 32 + vext.8 @t[1], @x[1], @x[1], #12 + veor @x[0], @x[0], @t[0] @ x0 ^ (x0 <<< 32) + vext.8 @t[2], @x[2], @x[2], #12 + veor @x[1], @x[1], @t[1] + vext.8 @t[3], @x[3], @x[3], #12 + veor @x[2], @x[2], @t[2] + vext.8 @t[4], @x[4], @x[4], #12 + veor @x[3], @x[3], @t[3] + vext.8 @t[5], @x[5], @x[5], #12 + veor @x[4], @x[4], @t[4] + vext.8 @t[6], @x[6], @x[6], #12 + veor @x[5], @x[5], @t[5] + vext.8 @t[7], @x[7], @x[7], #12 + veor @x[6], @x[6], @t[6] + + veor @t[1], @t[1], @x[0] + veor @x[7], @x[7], @t[7] + vext.8 @x[0], @x[0], @x[0], #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor @t[2], @t[2], @x[1] + veor @t[0], @t[0], @x[7] + veor @t[1], @t[1], @x[7] + vext.8 @x[1], @x[1], @x[1], #8 + veor @t[5], @t[5], @x[4] + veor @x[0], @x[0], @t[0] + veor @t[6], @t[6], @x[5] + veor @x[1], @x[1], @t[1] + vext.8 @t[0], @x[4], @x[4], #8 + veor @t[4], @t[4], @x[3] + vext.8 @t[1], @x[5], @x[5], #8 + veor @t[7], @t[7], @x[6] + vext.8 @x[4], @x[3], @x[3], #8 + veor @t[3], @t[3], @x[2] + vext.8 @x[5], @x[7], @x[7], #8 + veor @t[4], @t[4], @x[7] + vext.8 @x[3], @x[6], @x[6], #8 + veor @t[3], @t[3], @x[7] + vext.8 @x[6], @x[2], @x[2], #8 + veor @x[7], @t[1], @t[5] +___ +$code.=<<___ if (!$inv); + veor @x[2], @t[0], @t[4] + veor @x[4], @x[4], @t[3] + veor @x[5], @x[5], @t[7] + veor @x[3], @x[3], @t[6] + @ vmov @x[2], @t[0] + veor @x[6], @x[6], @t[2] + @ vmov @x[7], @t[1] +___ +$code.=<<___ if ($inv); + veor @t[3], @t[3], @x[4] + veor @x[5], @x[5], @t[7] + veor @x[2], @x[3], @t[6] + veor @x[3], @t[0], @t[4] + veor @x[4], @x[6], @t[2] + vmov @x[6], @t[3] + @ vmov @x[7], @t[1] +___ +} + +sub InvMixColumns_orig { +my @x=@_[0..7]; +my @t=@_[8..15]; + +$code.=<<___; + @ multiplication by 0x0e + vext.8 @t[7], @x[7], @x[7], #12 + vmov @t[2], @x[2] + veor @x[2], @x[2], @x[5] @ 2 5 + veor @x[7], @x[7], @x[5] @ 7 5 + vext.8 @t[0], @x[0], @x[0], #12 + vmov @t[5], @x[5] + veor @x[5], @x[5], @x[0] @ 5 0 [1] + veor @x[0], @x[0], @x[1] @ 0 1 + vext.8 @t[1], @x[1], @x[1], #12 + veor @x[1], @x[1], @x[2] @ 1 25 + veor @x[0], @x[0], @x[6] @ 01 6 [2] + vext.8 @t[3], @x[3], @x[3], #12 + veor @x[1], @x[1], @x[3] @ 125 3 [4] + veor @x[2], @x[2], @x[0] @ 25 016 [3] + veor @x[3], @x[3], @x[7] @ 3 75 + veor @x[7], @x[7], @x[6] @ 75 6 [0] + vext.8 @t[6], @x[6], @x[6], #12 + vmov @t[4], @x[4] + veor @x[6], @x[6], @x[4] @ 6 4 + veor @x[4], @x[4], @x[3] @ 4 375 [6] + veor @x[3], @x[3], @x[7] @ 375 756=36 + veor @x[6], @x[6], @t[5] @ 64 5 [7] + veor @x[3], @x[3], @t[2] @ 36 2 + vext.8 @t[5], @t[5], @t[5], #12 + veor @x[3], @x[3], @t[4] @ 362 4 [5] +___ + my @y = @x[7,5,0,2,1,3,4,6]; +$code.=<<___; + @ multiplication by 0x0b + veor @y[1], @y[1], @y[0] + veor @y[0], @y[0], @t[0] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[1], @y[1], @t[1] + veor @y[0], @y[0], @t[5] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[1], @y[1], @t[6] + veor @y[0], @y[0], @t[7] + veor @t[7], @t[7], @t[6] @ clobber t[7] + + veor @y[3], @y[3], @t[0] + veor @y[1], @y[1], @y[0] + vext.8 @t[0], @t[0], @t[0], #12 + veor @y[2], @y[2], @t[1] + veor @y[4], @y[4], @t[1] + vext.8 @t[1], @t[1], @t[1], #12 + veor @y[2], @y[2], @t[2] + veor @y[3], @y[3], @t[2] + veor @y[5], @y[5], @t[2] + veor @y[2], @y[2], @t[7] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[3], @y[3], @t[3] + veor @y[6], @y[6], @t[3] + veor @y[4], @y[4], @t[3] + veor @y[7], @y[7], @t[4] + vext.8 @t[3], @t[3], @t[3], #12 + veor @y[5], @y[5], @t[4] + veor @y[7], @y[7], @t[7] + veor @t[7], @t[7], @t[5] @ clobber t[7] even more + veor @y[3], @y[3], @t[5] + veor @y[4], @y[4], @t[4] + + veor @y[5], @y[5], @t[7] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[6], @y[6], @t[7] + veor @y[4], @y[4], @t[7] + + veor @t[7], @t[7], @t[5] + vext.8 @t[5], @t[5], @t[5], #12 + + @ multiplication by 0x0d + veor @y[4], @y[4], @y[7] + veor @t[7], @t[7], @t[6] @ restore t[7] + veor @y[7], @y[7], @t[4] + vext.8 @t[6], @t[6], @t[6], #12 + veor @y[2], @y[2], @t[0] + veor @y[7], @y[7], @t[5] + vext.8 @t[7], @t[7], @t[7], #12 + veor @y[2], @y[2], @t[2] + + veor @y[3], @y[3], @y[1] + veor @y[1], @y[1], @t[1] + veor @y[0], @y[0], @t[0] + veor @y[3], @y[3], @t[0] + veor @y[1], @y[1], @t[5] + veor @y[0], @y[0], @t[5] + vext.8 @t[0], @t[0], @t[0], #12 + veor @y[1], @y[1], @t[7] + veor @y[0], @y[0], @t[6] + veor @y[3], @y[3], @y[1] + veor @y[4], @y[4], @t[1] + vext.8 @t[1], @t[1], @t[1], #12 + + veor @y[7], @y[7], @t[7] + veor @y[4], @y[4], @t[2] + veor @y[5], @y[5], @t[2] + veor @y[2], @y[2], @t[6] + veor @t[6], @t[6], @t[3] @ clobber t[6] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[4], @y[4], @y[7] + veor @y[3], @y[3], @t[6] + + veor @y[6], @y[6], @t[6] + veor @y[5], @y[5], @t[5] + vext.8 @t[5], @t[5], @t[5], #12 + veor @y[6], @y[6], @t[4] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[5], @y[5], @t[6] + veor @y[6], @y[6], @t[7] + vext.8 @t[7], @t[7], @t[7], #12 + veor @t[6], @t[6], @t[3] @ restore t[6] + vext.8 @t[3], @t[3], @t[3], #12 + + @ multiplication by 0x09 + veor @y[4], @y[4], @y[1] + veor @t[1], @t[1], @y[1] @ t[1]=y[1] + veor @t[0], @t[0], @t[5] @ clobber t[0] + vext.8 @t[6], @t[6], @t[6], #12 + veor @t[1], @t[1], @t[5] + veor @y[3], @y[3], @t[0] + veor @t[0], @t[0], @y[0] @ t[0]=y[0] + veor @t[1], @t[1], @t[6] + veor @t[6], @t[6], @t[7] @ clobber t[6] + veor @y[4], @y[4], @t[1] + veor @y[7], @y[7], @t[4] + veor @y[6], @y[6], @t[3] + veor @y[5], @y[5], @t[2] + veor @t[4], @t[4], @y[4] @ t[4]=y[4] + veor @t[3], @t[3], @y[3] @ t[3]=y[3] + veor @t[5], @t[5], @y[5] @ t[5]=y[5] + veor @t[2], @t[2], @y[2] @ t[2]=y[2] + veor @t[3], @t[3], @t[7] + veor @XMM[5], @t[5], @t[6] + veor @XMM[6], @t[6], @y[6] @ t[6]=y[6] + veor @XMM[2], @t[2], @t[6] + veor @XMM[7], @t[7], @y[7] @ t[7]=y[7] + + vmov @XMM[0], @t[0] + vmov @XMM[1], @t[1] + @ vmov @XMM[2], @t[2] + vmov @XMM[3], @t[3] + vmov @XMM[4], @t[4] + @ vmov @XMM[5], @t[5] + @ vmov @XMM[6], @t[6] + @ vmov @XMM[7], @t[7] +___ +} + +sub InvMixColumns { +my @x=@_[0..7]; +my @t=@_[8..15]; + +# Thanks to Jussi Kivilinna for providing pointer to +# +# | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 | +# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 | +# | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 | +# | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 | + +$code.=<<___; + @ multiplication by 0x05-0x00-0x04-0x00 + vext.8 @t[0], @x[0], @x[0], #8 + vext.8 @t[6], @x[6], @x[6], #8 + vext.8 @t[7], @x[7], @x[7], #8 + veor @t[0], @t[0], @x[0] + vext.8 @t[1], @x[1], @x[1], #8 + veor @t[6], @t[6], @x[6] + vext.8 @t[2], @x[2], @x[2], #8 + veor @t[7], @t[7], @x[7] + vext.8 @t[3], @x[3], @x[3], #8 + veor @t[1], @t[1], @x[1] + vext.8 @t[4], @x[4], @x[4], #8 + veor @t[2], @t[2], @x[2] + vext.8 @t[5], @x[5], @x[5], #8 + veor @t[3], @t[3], @x[3] + veor @t[4], @t[4], @x[4] + veor @t[5], @t[5], @x[5] + + veor @x[0], @x[0], @t[6] + veor @x[1], @x[1], @t[6] + veor @x[2], @x[2], @t[0] + veor @x[4], @x[4], @t[2] + veor @x[3], @x[3], @t[1] + veor @x[1], @x[1], @t[7] + veor @x[2], @x[2], @t[7] + veor @x[4], @x[4], @t[6] + veor @x[5], @x[5], @t[3] + veor @x[3], @x[3], @t[6] + veor @x[6], @x[6], @t[4] + veor @x[4], @x[4], @t[7] + veor @x[5], @x[5], @t[7] + veor @x[7], @x[7], @t[5] +___ + &MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6 +} + +sub swapmove { +my ($a,$b,$n,$mask,$t)=@_; +$code.=<<___; + vshr.u64 $t, $b, #$n + veor $t, $t, $a + vand $t, $t, $mask + veor $a, $a, $t + vshl.u64 $t, $t, #$n + veor $b, $b, $t +___ +} +sub swapmove2x { +my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_; +$code.=<<___; + vshr.u64 $t0, $b0, #$n + vshr.u64 $t1, $b1, #$n + veor $t0, $t0, $a0 + veor $t1, $t1, $a1 + vand $t0, $t0, $mask + vand $t1, $t1, $mask + veor $a0, $a0, $t0 + vshl.u64 $t0, $t0, #$n + veor $a1, $a1, $t1 + vshl.u64 $t1, $t1, #$n + veor $b0, $b0, $t0 + veor $b1, $b1, $t1 +___ +} + +sub bitslice { +my @x=reverse(@_[0..7]); +my ($t0,$t1,$t2,$t3)=@_[8..11]; +$code.=<<___; + vmov.i8 $t0,#0x55 @ compose .LBS0 + vmov.i8 $t1,#0x33 @ compose .LBS1 +___ + &swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3); + &swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); +$code.=<<___; + vmov.i8 $t0,#0x0f @ compose .LBS2 +___ + &swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3); + &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); + + &swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3); + &swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3); +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" + +# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} +# define VFP_ABI_POP vldmia sp!,{d8-d15} +# define VFP_ABI_FRAME 0x40 +#else +# define VFP_ABI_PUSH +# define VFP_ABI_POP +# define VFP_ABI_FRAME 0 +# define BSAES_ASM_EXTENDED_KEY +# define XTS_CHAIN_TWEAK +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +#endif + +#ifdef __thumb__ +# define adrl adr +#endif + +#if __ARM_ARCH__>=7 +.text +.syntax unified @ ARMv7-capable assembler is expected to handle this +#ifdef __thumb2__ +.thumb +#else +.code 32 +#endif + +.fpu neon + +.type _bsaes_decrypt8,%function +.align 4 +_bsaes_decrypt8: + adr $const,_bsaes_decrypt8 + vldmia $key!, {@XMM[9]} @ round 0 key + add $const,$const,#.LM0ISR-_bsaes_decrypt8 + + vldmia $const!, {@XMM[8]} @ .LM0ISR + veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key + veor @XMM[11], @XMM[1], @XMM[9] + vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` + veor @XMM[12], @XMM[2], @XMM[9] + vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` + veor @XMM[13], @XMM[3], @XMM[9] + vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` + veor @XMM[14], @XMM[4], @XMM[9] + vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` + veor @XMM[15], @XMM[5], @XMM[9] + vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` + veor @XMM[10], @XMM[6], @XMM[9] + vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` + veor @XMM[11], @XMM[7], @XMM[9] + vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` + vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` +___ + &bitslice (@XMM[0..7, 8..11]); +$code.=<<___; + sub $rounds,$rounds,#1 + b .Ldec_sbox +.align 4 +.Ldec_loop: +___ + &ShiftRows (@XMM[0..7, 8..12]); +$code.=".Ldec_sbox:\n"; + &InvSbox (@XMM[0..7, 8..15]); +$code.=<<___; + subs $rounds,$rounds,#1 + bcc .Ldec_done +___ + &InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]); +$code.=<<___; + vldmia $const, {@XMM[12]} @ .LISR + ite eq @ Thumb2 thing, sanity check in ARM + addeq $const,$const,#0x10 + bne .Ldec_loop + vldmia $const, {@XMM[12]} @ .LISRM0 + b .Ldec_loop +.align 4 +.Ldec_done: +___ + &bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]); +$code.=<<___; + vldmia $key, {@XMM[8]} @ last round key + veor @XMM[6], @XMM[6], @XMM[8] + veor @XMM[4], @XMM[4], @XMM[8] + veor @XMM[2], @XMM[2], @XMM[8] + veor @XMM[7], @XMM[7], @XMM[8] + veor @XMM[3], @XMM[3], @XMM[8] + veor @XMM[5], @XMM[5], @XMM[8] + veor @XMM[0], @XMM[0], @XMM[8] + veor @XMM[1], @XMM[1], @XMM[8] + bx lr +.size _bsaes_decrypt8,.-_bsaes_decrypt8 + +.type _bsaes_const,%object +.align 6 +_bsaes_const: +.LM0ISR: @ InvShiftRows constants + .quad 0x0a0e0206070b0f03, 0x0004080c0d010509 +.LISR: + .quad 0x0504070602010003, 0x0f0e0d0c080b0a09 +.LISRM0: + .quad 0x01040b0e0205080f, 0x0306090c00070a0d +.LM0SR: @ ShiftRows constants + .quad 0x0a0e02060f03070b, 0x0004080c05090d01 +.LSR: + .quad 0x0504070600030201, 0x0f0e0d0c0a09080b +.LSRM0: + .quad 0x0304090e00050a0f, 0x01060b0c0207080d +.LM0: + .quad 0x02060a0e03070b0f, 0x0004080c0105090d +.LREVM0SR: + .quad 0x090d01050c000408, 0x03070b0f060a0e02 +.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by " +.align 6 +.size _bsaes_const,.-_bsaes_const + +.type _bsaes_encrypt8,%function +.align 4 +_bsaes_encrypt8: + adr $const,_bsaes_encrypt8 + vldmia $key!, {@XMM[9]} @ round 0 key + sub $const,$const,#_bsaes_encrypt8-.LM0SR + + vldmia $const!, {@XMM[8]} @ .LM0SR +_bsaes_encrypt8_alt: + veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key + veor @XMM[11], @XMM[1], @XMM[9] + vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` + veor @XMM[12], @XMM[2], @XMM[9] + vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` + veor @XMM[13], @XMM[3], @XMM[9] + vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` + veor @XMM[14], @XMM[4], @XMM[9] + vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` + veor @XMM[15], @XMM[5], @XMM[9] + vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` + veor @XMM[10], @XMM[6], @XMM[9] + vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` + veor @XMM[11], @XMM[7], @XMM[9] + vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` + vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` +_bsaes_encrypt8_bitslice: +___ + &bitslice (@XMM[0..7, 8..11]); +$code.=<<___; + sub $rounds,$rounds,#1 + b .Lenc_sbox +.align 4 +.Lenc_loop: +___ + &ShiftRows (@XMM[0..7, 8..12]); +$code.=".Lenc_sbox:\n"; + &Sbox (@XMM[0..7, 8..15]); +$code.=<<___; + subs $rounds,$rounds,#1 + bcc .Lenc_done +___ + &MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]); +$code.=<<___; + vldmia $const, {@XMM[12]} @ .LSR + ite eq @ Thumb2 thing, samity check in ARM + addeq $const,$const,#0x10 + bne .Lenc_loop + vldmia $const, {@XMM[12]} @ .LSRM0 + b .Lenc_loop +.align 4 +.Lenc_done: +___ + # output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb + &bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]); +$code.=<<___; + vldmia $key, {@XMM[8]} @ last round key + veor @XMM[4], @XMM[4], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[8] + veor @XMM[3], @XMM[3], @XMM[8] + veor @XMM[7], @XMM[7], @XMM[8] + veor @XMM[2], @XMM[2], @XMM[8] + veor @XMM[5], @XMM[5], @XMM[8] + veor @XMM[0], @XMM[0], @XMM[8] + veor @XMM[1], @XMM[1], @XMM[8] + bx lr +.size _bsaes_encrypt8,.-_bsaes_encrypt8 +___ +} +{ +my ($out,$inp,$rounds,$const)=("r12","r4","r5","r6"); + +sub bitslice_key { +my @x=reverse(@_[0..7]); +my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12]; + + &swapmove (@x[0,1],1,$bs0,$t2,$t3); +$code.=<<___; + @ &swapmove(@x[2,3],1,$t0,$t2,$t3); + vmov @x[2], @x[0] + vmov @x[3], @x[1] +___ + #&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); + + &swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3); +$code.=<<___; + @ &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); + vmov @x[4], @x[0] + vmov @x[6], @x[2] + vmov @x[5], @x[1] + vmov @x[7], @x[3] +___ + &swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3); + &swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3); +} + +$code.=<<___; +.type _bsaes_key_convert,%function +.align 4 +_bsaes_key_convert: + adr $const,_bsaes_key_convert + vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key + sub $const,$const,#_bsaes_key_convert-.LM0 + vld1.8 {@XMM[15]}, [$inp]! @ load round 1 key + + vmov.i8 @XMM[8], #0x01 @ bit masks + vmov.i8 @XMM[9], #0x02 + vmov.i8 @XMM[10], #0x04 + vmov.i8 @XMM[11], #0x08 + vmov.i8 @XMM[12], #0x10 + vmov.i8 @XMM[13], #0x20 + vldmia $const, {@XMM[14]} @ .LM0 + +#ifdef __ARMEL__ + vrev32.8 @XMM[7], @XMM[7] + vrev32.8 @XMM[15], @XMM[15] +#endif + sub $rounds,$rounds,#1 + vstmia $out!, {@XMM[7]} @ save round 0 key + b .Lkey_loop + +.align 4 +.Lkey_loop: + vtbl.8 `&Dlo(@XMM[7])`,{@XMM[15]},`&Dlo(@XMM[14])` + vtbl.8 `&Dhi(@XMM[7])`,{@XMM[15]},`&Dhi(@XMM[14])` + vmov.i8 @XMM[6], #0x40 + vmov.i8 @XMM[15], #0x80 + + vtst.8 @XMM[0], @XMM[7], @XMM[8] + vtst.8 @XMM[1], @XMM[7], @XMM[9] + vtst.8 @XMM[2], @XMM[7], @XMM[10] + vtst.8 @XMM[3], @XMM[7], @XMM[11] + vtst.8 @XMM[4], @XMM[7], @XMM[12] + vtst.8 @XMM[5], @XMM[7], @XMM[13] + vtst.8 @XMM[6], @XMM[7], @XMM[6] + vtst.8 @XMM[7], @XMM[7], @XMM[15] + vld1.8 {@XMM[15]}, [$inp]! @ load next round key + vmvn @XMM[0], @XMM[0] @ "pnot" + vmvn @XMM[1], @XMM[1] + vmvn @XMM[5], @XMM[5] + vmvn @XMM[6], @XMM[6] +#ifdef __ARMEL__ + vrev32.8 @XMM[15], @XMM[15] +#endif + subs $rounds,$rounds,#1 + vstmia $out!,{@XMM[0]-@XMM[7]} @ write bit-sliced round key + bne .Lkey_loop + + vmov.i8 @XMM[7],#0x63 @ compose .L63 + @ don't save last round key + bx lr +.size _bsaes_key_convert,.-_bsaes_key_convert +___ +} + +if (0) { # following four functions are unsupported interface + # used for benchmarking... +$code.=<<___; +.globl bsaes_enc_key_convert +.type bsaes_enc_key_convert,%function +.align 4 +bsaes_enc_key_convert: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + + ldr r5,[$inp,#240] @ pass rounds + mov r4,$inp @ pass key + mov r12,$out @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_enc_key_convert,.-bsaes_enc_key_convert + +.globl bsaes_encrypt_128 +.type bsaes_encrypt_128,%function +.align 4 +bsaes_encrypt_128: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so +.Lenc128_loop: + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! + mov r4,$key @ pass the key + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5,#10 @ pass rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + + bl _bsaes_encrypt8 + + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + subs $len,$len,#0x80 + vst1.8 {@XMM[5]}, [$out]! + bhi .Lenc128_loop + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_encrypt_128,.-bsaes_encrypt_128 + +.globl bsaes_dec_key_convert +.type bsaes_dec_key_convert,%function +.align 4 +bsaes_dec_key_convert: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + + ldr r5,[$inp,#240] @ pass rounds + mov r4,$inp @ pass key + mov r12,$out @ pass key schedule + bl _bsaes_key_convert + vldmia $out, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia $out, {@XMM[7]} + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_dec_key_convert,.-bsaes_dec_key_convert + +.globl bsaes_decrypt_128 +.type bsaes_decrypt_128,%function +.align 4 +bsaes_decrypt_128: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so +.Ldec128_loop: + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! + mov r4,$key @ pass the key + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5,#10 @ pass rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + + bl _bsaes_decrypt8 + + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + subs $len,$len,#0x80 + vst1.8 {@XMM[5]}, [$out]! + bhi .Ldec128_loop + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_decrypt_128,.-bsaes_decrypt_128 +___ +} +{ +my ($inp,$out,$len,$key, $ivp,$fp,$rounds)=map("r$_",(0..3,8..10)); +my ($keysched)=("sp"); + +$code.=<<___; +.extern AES_cbc_encrypt +.extern AES_decrypt + +.global bsaes_cbc_encrypt +.type bsaes_cbc_encrypt,%function +.align 5 +bsaes_cbc_encrypt: +#ifndef __KERNEL__ + cmp $len, #128 +#ifndef __thumb__ + blo AES_cbc_encrypt +#else + bhs 1f + b AES_cbc_encrypt +1: +#endif +#endif + + @ it is up to the caller to make sure we are called with enc == 0 + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr $ivp, [ip] @ IV is 1st arg on the stack + mov $len, $len, lsr#4 @ len in 16 byte blocks + sub sp, #0x10 @ scratch space to carry over the IV + mov $fp, sp @ save sp + + ldr $rounds, [$key, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + add r12, #`128-32` @ sifze of bit-slices key schedule + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 @ sp is $keysched + bl _bsaes_key_convert + vldmia $keysched, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia $keysched, {@XMM[7]} +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, $key, #248 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} + +.align 2 +0: +#endif + + vld1.8 {@XMM[15]}, [$ivp] @ load IV + b .Lcbc_dec_loop + +.align 4 +.Lcbc_dec_loop: + subs $len, $len, #0x8 + bmi .Lcbc_dec_loop_finish + + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, $keysched @ pass the key +#else + add r4, $key, #248 +#endif + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5, $rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp] + sub $inp, $inp, #0x60 + vstmia $fp, {@XMM[15]} @ put aside IV + + bl _bsaes_decrypt8 + + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[3], @XMM[3], @XMM[13] + vst1.8 {@XMM[6]}, [$out]! + veor @XMM[5], @XMM[5], @XMM[14] + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + vst1.8 {@XMM[5]}, [$out]! + + b .Lcbc_dec_loop + +.Lcbc_dec_loop_finish: + adds $len, $len, #8 + beq .Lcbc_dec_done + + vld1.8 {@XMM[0]}, [$inp]! @ load input + cmp $len, #2 + blo .Lcbc_dec_one + vld1.8 {@XMM[1]}, [$inp]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, $keysched @ pass the key +#else + add r4, $key, #248 +#endif + mov r5, $rounds + vstmia $fp, {@XMM[15]} @ put aside IV + beq .Lcbc_dec_two + vld1.8 {@XMM[2]}, [$inp]! + cmp $len, #4 + blo .Lcbc_dec_three + vld1.8 {@XMM[3]}, [$inp]! + beq .Lcbc_dec_four + vld1.8 {@XMM[4]}, [$inp]! + cmp $len, #6 + blo .Lcbc_dec_five + vld1.8 {@XMM[5]}, [$inp]! + beq .Lcbc_dec_six + vld1.8 {@XMM[6]}, [$inp]! + sub $inp, $inp, #0x70 + + bl _bsaes_decrypt8 + + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[3], @XMM[3], @XMM[13] + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_six: + sub $inp, $inp, #0x60 + bl _bsaes_decrypt8 + vldmia $fp,{@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_five: + sub $inp, $inp, #0x50 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[2], @XMM[2], @XMM[11] + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_four: + sub $inp, $inp, #0x40 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_three: + sub $inp, $inp, #0x30 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_two: + sub $inp, $inp, #0x20 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[15]}, [$inp]! @ reload input + veor @XMM[1], @XMM[1], @XMM[8] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_one: + sub $inp, $inp, #0x10 + mov $rounds, $out @ save original out pointer + mov $out, $fp @ use the iv scratch space as out buffer + mov r2, $key + vmov @XMM[4],@XMM[15] @ just in case ensure that IV + vmov @XMM[5],@XMM[0] @ and input are preserved + bl AES_decrypt + vld1.8 {@XMM[0]}, [$fp,:64] @ load result + veor @XMM[0], @XMM[0], @XMM[4] @ ^= IV + vmov @XMM[15], @XMM[5] @ @XMM[5] holds input + vst1.8 {@XMM[0]}, [$rounds] @ write output + +.Lcbc_dec_done: +#ifndef BSAES_ASM_EXTENDED_KEY + vmov.i32 q0, #0 + vmov.i32 q1, #0 +.Lcbc_dec_bzero: @ wipe key schedule [if any] + vstmia $keysched!, {q0-q1} + cmp $keysched, $fp + bne .Lcbc_dec_bzero +#endif + + mov sp, $fp + add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb + vst1.8 {@XMM[15]}, [$ivp] @ return IV + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} +.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt +___ +} +{ +my ($inp,$out,$len,$key, $ctr,$fp,$rounds)=(map("r$_",(0..3,8..10))); +my $const = "r6"; # shared with _bsaes_encrypt8_alt +my $keysched = "sp"; + +$code.=<<___; +.extern AES_encrypt +.global bsaes_ctr32_encrypt_blocks +.type bsaes_ctr32_encrypt_blocks,%function +.align 5 +bsaes_ctr32_encrypt_blocks: + cmp $len, #8 @ use plain AES for + blo .Lctr_enc_short @ small sizes + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr $ctr, [ip] @ ctr is 1st arg on the stack + sub sp, sp, #0x10 @ scratch space to carry over the ctr + mov $fp, sp @ save sp + + ldr $rounds, [$key, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + add r12, #`128-32` @ size of bit-sliced key schedule + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 @ sp is $keysched + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + + vld1.8 {@XMM[0]}, [$ctr] @ load counter + add $ctr, $const, #.LREVM0SR-.LM0 @ borrow $ctr + vldmia $keysched, {@XMM[4]} @ load round0 key +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + +.align 2 +0: add r12, $key, #248 + vld1.8 {@XMM[0]}, [$ctr] @ load counter + adrl $ctr, .LREVM0SR @ borrow $ctr + vldmia r12, {@XMM[4]} @ load round0 key + sub sp, #0x10 @ place for adjusted round0 key +#endif + + vmov.i32 @XMM[8],#1 @ compose 1<<96 + veor @XMM[9],@XMM[9],@XMM[9] + vrev32.8 @XMM[0],@XMM[0] + vext.8 @XMM[8],@XMM[9],@XMM[8],#4 + vrev32.8 @XMM[4],@XMM[4] + vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 + vstmia $keysched, {@XMM[4]} @ save adjusted round0 key + b .Lctr_enc_loop + +.align 4 +.Lctr_enc_loop: + vadd.u32 @XMM[10], @XMM[8], @XMM[9] @ compose 3<<96 + vadd.u32 @XMM[1], @XMM[0], @XMM[8] @ +1 + vadd.u32 @XMM[2], @XMM[0], @XMM[9] @ +2 + vadd.u32 @XMM[3], @XMM[0], @XMM[10] @ +3 + vadd.u32 @XMM[4], @XMM[1], @XMM[10] + vadd.u32 @XMM[5], @XMM[2], @XMM[10] + vadd.u32 @XMM[6], @XMM[3], @XMM[10] + vadd.u32 @XMM[7], @XMM[4], @XMM[10] + vadd.u32 @XMM[10], @XMM[5], @XMM[10] @ next counter + + @ Borrow prologue from _bsaes_encrypt8 to use the opportunity + @ to flip byte order in 32-bit counter + + vldmia $keysched, {@XMM[9]} @ load round0 key +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, $keysched, #0x10 @ pass next round key +#else + add r4, $key, #`248+16` +#endif + vldmia $ctr, {@XMM[8]} @ .LREVM0SR + mov r5, $rounds @ pass rounds + vstmia $fp, {@XMM[10]} @ save next counter + sub $const, $ctr, #.LREVM0SR-.LSR @ pass constants + + bl _bsaes_encrypt8_alt + + subs $len, $len, #8 + blo .Lctr_enc_loop_done + + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ load input + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[0], @XMM[8] + veor @XMM[1], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[10] + veor @XMM[6], @XMM[11] + vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! + veor @XMM[3], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[7], @XMM[13] + veor @XMM[2], @XMM[14] + vst1.8 {@XMM[4]}, [$out]! + veor @XMM[5], @XMM[15] + vst1.8 {@XMM[6]}, [$out]! + vmov.i32 @XMM[8], #1 @ compose 1<<96 + vst1.8 {@XMM[3]}, [$out]! + veor @XMM[9], @XMM[9], @XMM[9] + vst1.8 {@XMM[7]}, [$out]! + vext.8 @XMM[8], @XMM[9], @XMM[8], #4 + vst1.8 {@XMM[2]}, [$out]! + vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 + vst1.8 {@XMM[5]}, [$out]! + vldmia $fp, {@XMM[0]} @ load counter + + bne .Lctr_enc_loop + b .Lctr_enc_done + +.align 4 +.Lctr_enc_loop_done: + add $len, $len, #8 + vld1.8 {@XMM[8]}, [$inp]! @ load input + veor @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! @ write output + cmp $len, #2 + blo .Lctr_enc_done + vld1.8 {@XMM[9]}, [$inp]! + veor @XMM[1], @XMM[9] + vst1.8 {@XMM[1]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[10]}, [$inp]! + veor @XMM[4], @XMM[10] + vst1.8 {@XMM[4]}, [$out]! + cmp $len, #4 + blo .Lctr_enc_done + vld1.8 {@XMM[11]}, [$inp]! + veor @XMM[6], @XMM[11] + vst1.8 {@XMM[6]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[12]}, [$inp]! + veor @XMM[3], @XMM[12] + vst1.8 {@XMM[3]}, [$out]! + cmp $len, #6 + blo .Lctr_enc_done + vld1.8 {@XMM[13]}, [$inp]! + veor @XMM[7], @XMM[13] + vst1.8 {@XMM[7]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[14]}, [$inp] + veor @XMM[2], @XMM[14] + vst1.8 {@XMM[2]}, [$out]! + +.Lctr_enc_done: + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifndef BSAES_ASM_EXTENDED_KEY +.Lctr_enc_bzero: @ wipe key schedule [if any] + vstmia $keysched!, {q0-q1} + cmp $keysched, $fp + bne .Lctr_enc_bzero +#else + vstmia $keysched, {q0-q1} +#endif + + mov sp, $fp + add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.align 4 +.Lctr_enc_short: + ldr ip, [sp] @ ctr pointer is passed on stack + stmdb sp!, {r4-r8, lr} + + mov r4, $inp @ copy arguments + mov r5, $out + mov r6, $len + mov r7, $key + ldr r8, [ip, #12] @ load counter LSW + vld1.8 {@XMM[1]}, [ip] @ load whole counter value +#ifdef __ARMEL__ + rev r8, r8 +#endif + sub sp, sp, #0x10 + vst1.8 {@XMM[1]}, [sp,:64] @ copy counter value + sub sp, sp, #0x10 + +.Lctr_enc_short_loop: + add r0, sp, #0x10 @ input counter value + mov r1, sp @ output on the stack + mov r2, r7 @ key + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [r4]! @ load input + vld1.8 {@XMM[1]}, [sp,:64] @ load encrypted counter + add r8, r8, #1 +#ifdef __ARMEL__ + rev r0, r8 + str r0, [sp, #0x1c] @ next counter value +#else + str r8, [sp, #0x1c] @ next counter value +#endif + veor @XMM[0],@XMM[0],@XMM[1] + vst1.8 {@XMM[0]}, [r5]! @ store output + subs r6, r6, #1 + bne .Lctr_enc_short_loop + + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vstmia sp!, {q0-q1} + + ldmia sp!, {r4-r8, pc} +.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks +___ +} +{ +###################################################################### +# void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len, +# const AES_KEY *key1, const AES_KEY *key2, +# const unsigned char iv[16]); +# +my ($inp,$out,$len,$key,$rounds,$magic,$fp)=(map("r$_",(7..10,1..3))); +my $const="r6"; # returned by _bsaes_key_convert +my $twmask=@XMM[5]; +my @T=@XMM[6..7]; + +$code.=<<___; +.globl bsaes_xts_encrypt +.type bsaes_xts_encrypt,%function +.align 4 +bsaes_xts_encrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future $fp + + mov $inp, r0 + mov $out, r1 + mov $len, r2 + mov $key, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0,sp @ pointer to initial tweak +#endif + + ldr $rounds, [$key, #240] @ get # of rounds + mov $fp, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + @ add r12, #`128-32` @ size of bit-sliced key schedule + sub r12, #`32+16` @ place for tweak[9] + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + + vld1.8 {@XMM[8]}, [r0] @ initial tweak + adr $magic, .Lxts_magic + + subs $len, #0x80 + blo .Lxts_enc_short + b .Lxts_enc_loop + +.align 4 +.Lxts_enc_loop: + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + vadd.u64 @XMM[8], @XMM[15], @XMM[15] + vst1.64 {@XMM[15]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + veor @XMM[8], @XMM[8], @T[0] + vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + veor @XMM[7], @XMM[7], @XMM[15] + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[2], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + veor @XMM[13], @XMM[5], @XMM[15] + vst1.8 {@XMM[12]-@XMM[13]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + subs $len, #0x80 + bpl .Lxts_enc_loop + +.Lxts_enc_short: + adds $len, #0x70 + bmi .Lxts_enc_done + + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! + subs $len, #0x10 + bmi .Lxts_enc_`$i-9` +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + sub $len, #0x10 + vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vld1.64 {@XMM[14]}, [r0,:128]! + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[2], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + vst1.8 {@XMM[12]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_6: + vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak + + veor @XMM[4], @XMM[4], @XMM[12] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[5], @XMM[5], @XMM[13] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done + +@ put this in range for both ARM and Thumb mode adr instructions +.align 5 +.Lxts_magic: + .quad 1, 0x87 + +.align 5 +.Lxts_enc_5: + vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak + + veor @XMM[3], @XMM[3], @XMM[11] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[4], @XMM[4], @XMM[12] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + vst1.8 {@XMM[10]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_4: + vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak + + veor @XMM[2], @XMM[2], @XMM[10] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[3], @XMM[3], @XMM[11] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_3: + vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak + + veor @XMM[1], @XMM[1], @XMM[9] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[2], @XMM[2], @XMM[10] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + vld1.64 {@XMM[10]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + vst1.8 {@XMM[8]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_2: + vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak + + veor @XMM[0], @XMM[0], @XMM[8] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[1], @XMM[1], @XMM[9] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_1: + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! + mov $fp, r4 + + vmov @XMM[8], @XMM[9] @ next round tweak + +.Lxts_enc_done: +#ifndef XTS_CHAIN_TWEAK + adds $len, #0x10 + beq .Lxts_enc_ret + sub r6, $out, #0x10 + +.Lxts_enc_steal: + ldrb r0, [$inp], #1 + ldrb r1, [$out, #-0x10] + strb r0, [$out, #-0x10] + strb r1, [$out], #1 + + subs $len, #1 + bhi .Lxts_enc_steal + + vld1.8 {@XMM[0]}, [r6] + mov r0, sp + veor @XMM[0], @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [r6] + mov $fp, r4 +#endif + +.Lxts_enc_ret: + bic r0, $fp, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_enc_bzero + + mov sp, $fp +#ifdef XTS_CHAIN_TWEAK + vst1.8 {@XMM[8]}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_encrypt,.-bsaes_xts_encrypt + +.globl bsaes_xts_decrypt +.type bsaes_xts_decrypt,%function +.align 4 +bsaes_xts_decrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future $fp + + mov $inp, r0 + mov $out, r1 + mov $len, r2 + mov $key, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0, sp @ pointer to initial tweak +#endif + + ldr $rounds, [$key, #240] @ get # of rounds + mov $fp, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + @ add r12, #`128-32` @ size of bit-sliced key schedule + sub r12, #`32+16` @ place for tweak[9] + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + add r4, sp, #0x90 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, $key, #248 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + vld1.8 {@XMM[8]}, [r0] @ initial tweak + adr $magic, .Lxts_magic + + tst $len, #0xf @ if not multiple of 16 + it ne @ Thumb2 thing, sanity check in ARM + subne $len, #0x10 @ subtract another 16 bytes + subs $len, #0x80 + + blo .Lxts_dec_short + b .Lxts_dec_loop + +.align 4 +.Lxts_dec_loop: + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + vadd.u64 @XMM[8], @XMM[15], @XMM[15] + vst1.64 {@XMM[15]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + veor @XMM[8], @XMM[8], @T[0] + vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + veor @XMM[7], @XMM[7], @XMM[15] + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[3], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + veor @XMM[13], @XMM[5], @XMM[15] + vst1.8 {@XMM[12]-@XMM[13]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + subs $len, #0x80 + bpl .Lxts_dec_loop + +.Lxts_dec_short: + adds $len, #0x70 + bmi .Lxts_dec_done + + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! + subs $len, #0x10 + bmi .Lxts_dec_`$i-9` +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + sub $len, #0x10 + vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vld1.64 {@XMM[14]}, [r0,:128]! + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[3], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + vst1.8 {@XMM[12]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_6: + vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak + + veor @XMM[4], @XMM[4], @XMM[12] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[5], @XMM[5], @XMM[13] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_5: + vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak + + veor @XMM[3], @XMM[3], @XMM[11] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[4], @XMM[4], @XMM[12] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + vst1.8 {@XMM[10]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_4: + vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak + + veor @XMM[2], @XMM[2], @XMM[10] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[3], @XMM[3], @XMM[11] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_3: + vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak + + veor @XMM[1], @XMM[1], @XMM[9] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[2], @XMM[2], @XMM[10] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + vld1.64 {@XMM[10]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + vst1.8 {@XMM[8]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_2: + vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak + + veor @XMM[0], @XMM[0], @XMM[8] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[1], @XMM[1], @XMM[9] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_1: + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + mov r5, $magic @ preserve magic + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! + mov $fp, r4 + mov $magic, r5 + + vmov @XMM[8], @XMM[9] @ next round tweak + +.Lxts_dec_done: +#ifndef XTS_CHAIN_TWEAK + adds $len, #0x10 + beq .Lxts_dec_ret + + @ calculate one round of extra tweak for the stolen ciphertext + vldmia $magic, {$twmask} + vshr.s64 @XMM[6], @XMM[8], #63 + vand @XMM[6], @XMM[6], $twmask + vadd.u64 @XMM[9], @XMM[8], @XMM[8] + vswp `&Dhi("@XMM[6]")`,`&Dlo("@XMM[6]")` + veor @XMM[9], @XMM[9], @XMM[6] + + @ perform the final decryption with the last tweak value + vld1.8 {@XMM[0]}, [$inp]! + mov r0, sp + veor @XMM[0], @XMM[0], @XMM[9] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[9] + vst1.8 {@XMM[0]}, [$out] + + mov r6, $out +.Lxts_dec_steal: + ldrb r1, [$out] + ldrb r0, [$inp], #1 + strb r1, [$out, #0x10] + strb r0, [$out], #1 + + subs $len, #1 + bhi .Lxts_dec_steal + + vld1.8 {@XMM[0]}, [r6] + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [r6] + mov $fp, r4 +#endif + +.Lxts_dec_ret: + bic r0, $fp, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_dec_bzero + + mov sp, $fp +#ifdef XTS_CHAIN_TWEAK + vst1.8 {@XMM[8]}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_decrypt,.-bsaes_xts_decrypt +___ +} +$code.=<<___; +#endif +___ + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +print $code; + +close STDOUT; diff --git a/crypto/Kconfig b/crypto/Kconfig index a1eba1845367..66808e1355e3 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -761,6 +761,22 @@ config CRYPTO_AES_ARM See for more information. +config CRYPTO_AES_ARM_BS + tristate "Bit sliced AES using NEON instructions" + depends on ARM && KERNEL_MODE_NEON + select CRYPTO_ALGAPI + select CRYPTO_AES_ARM + select CRYPTO_ABLK_HELPER + help + Use a faster and more secure NEON based implementation of AES in CBC, + CTR and XTS modes + + Bit sliced AES gives around 45% speedup on Cortex-A15 for CTR mode + and for XTS mode encryption, CBC and XTS mode decryption speedup is + around 25%. (CBC encryption speed is not affected by this driver.) + This implementation does not rely on any lookup tables so it is + believed to be invulnerable to cache timing attacks. + config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" select CRYPTO_ALGAPI -- cgit v1.2.3 From 9d691d9e8e573ec4205cca8d20777c8f27c7dc69 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 7 Oct 2013 15:43:53 +0100 Subject: ARM: add .gitignore entry for aesbs-core.S This avoids this file being incorrectly added to git. Signed-off-by: Russell King --- arch/arm/crypto/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 arch/arm/crypto/.gitignore diff --git a/arch/arm/crypto/.gitignore b/arch/arm/crypto/.gitignore new file mode 100644 index 000000000000..6231d36b3635 --- /dev/null +++ b/arch/arm/crypto/.gitignore @@ -0,0 +1 @@ +aesbs-core.S -- cgit v1.2.3 From 06abd742a8657de597f6be838e7c1d46311b8ebb Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 2 Jan 2014 17:14:45 +0000 Subject: CRYPTO: Fix more AES build errors Building a multi-arch kernel results in: arch/arm/crypto/built-in.o: In function `aesbs_xts_decrypt': sha1_glue.c:(.text+0x15c8): undefined reference to `bsaes_xts_decrypt' arch/arm/crypto/built-in.o: In function `aesbs_xts_encrypt': sha1_glue.c:(.text+0x1664): undefined reference to `bsaes_xts_encrypt' arch/arm/crypto/built-in.o: In function `aesbs_ctr_encrypt': sha1_glue.c:(.text+0x184c): undefined reference to `bsaes_ctr32_encrypt_blocks' arch/arm/crypto/built-in.o: In function `aesbs_cbc_decrypt': sha1_glue.c:(.text+0x19b4): undefined reference to `bsaes_cbc_encrypt' This code is already runtime-conditional on NEON being supported, so there's no point compiling it out depending on the minimum build architecture. Acked-by: Ard Biesheuvel Signed-off-by: Russell King --- arch/arm/crypto/aesbs-core.S_shipped | 2 +- arch/arm/crypto/bsaes-armv7.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/crypto/aesbs-core.S_shipped b/arch/arm/crypto/aesbs-core.S_shipped index 64205d453260..71e5fc7cfb18 100644 --- a/arch/arm/crypto/aesbs-core.S_shipped +++ b/arch/arm/crypto/aesbs-core.S_shipped @@ -58,7 +58,7 @@ # define VFP_ABI_FRAME 0 # define BSAES_ASM_EXTENDED_KEY # define XTS_CHAIN_TWEAK -# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_ARCH__ 7 #endif #ifdef __thumb__ diff --git a/arch/arm/crypto/bsaes-armv7.pl b/arch/arm/crypto/bsaes-armv7.pl index f3d96d932573..be068db960ee 100644 --- a/arch/arm/crypto/bsaes-armv7.pl +++ b/arch/arm/crypto/bsaes-armv7.pl @@ -701,7 +701,7 @@ $code.=<<___; # define VFP_ABI_FRAME 0 # define BSAES_ASM_EXTENDED_KEY # define XTS_CHAIN_TWEAK -# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_ARCH__ 7 #endif #ifdef __thumb__ -- cgit v1.2.3 From 0fd9b0f24a579f70547de2aa12f66a0fa13558fc Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Fri, 25 Jul 2014 19:42:30 -0400 Subject: crypto: arm-aes - fix encryption of unaligned data Fix the same alignment bug as in arm64 - we need to pass residue unprocessed bytes as the last argument to blkcipher_walk_done. Signed-off-by: Mikulas Patocka Cc: stable@vger.kernel.org # 3.13+ Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/aesbs-glue.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c index 4522366da759..15468fbbdea3 100644 --- a/arch/arm/crypto/aesbs-glue.c +++ b/arch/arm/crypto/aesbs-glue.c @@ -137,7 +137,7 @@ static int aesbs_cbc_encrypt(struct blkcipher_desc *desc, dst += AES_BLOCK_SIZE; } while (--blocks); } - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } return err; } @@ -158,7 +158,7 @@ static int aesbs_cbc_decrypt(struct blkcipher_desc *desc, bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes, &ctx->dec, walk.iv); kernel_neon_end(); - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } while (walk.nbytes) { u32 blocks = walk.nbytes / AES_BLOCK_SIZE; @@ -182,7 +182,7 @@ static int aesbs_cbc_decrypt(struct blkcipher_desc *desc, dst += AES_BLOCK_SIZE; src += AES_BLOCK_SIZE; } while (--blocks); - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } return err; } @@ -268,7 +268,7 @@ static int aesbs_xts_encrypt(struct blkcipher_desc *desc, bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes, &ctx->enc, walk.iv); kernel_neon_end(); - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } return err; } @@ -292,7 +292,7 @@ static int aesbs_xts_decrypt(struct blkcipher_desc *desc, bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes, &ctx->dec, walk.iv); kernel_neon_end(); - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } return err; } -- cgit v1.2.3 From 8127ebfa1a6645cf7cb6e33efafbe7835bdb1d95 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Tue, 29 Jul 2014 17:14:09 +0100 Subject: ARM: 8118/1: crypto: sha1/make use of common SHA-1 structures Common SHA-1 structures are defined in for code sharing. This patch changes SHA-1/ARM glue code to use these structures. Acked-by: Ard Biesheuvel Signed-off-by: Jussi Kivilinna Signed-off-by: Russell King --- arch/arm/crypto/sha1_glue.c | 50 ++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 76cd976230bc..c494e579ffc3 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -24,31 +24,25 @@ #include #include -struct SHA1_CTX { - uint32_t h0,h1,h2,h3,h4; - u64 count; - u8 data[SHA1_BLOCK_SIZE]; -}; -asmlinkage void sha1_block_data_order(struct SHA1_CTX *digest, +asmlinkage void sha1_block_data_order(u32 *digest, const unsigned char *data, unsigned int rounds); static int sha1_init(struct shash_desc *desc) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); - memset(sctx, 0, sizeof(*sctx)); - sctx->h0 = SHA1_H0; - sctx->h1 = SHA1_H1; - sctx->h2 = SHA1_H2; - sctx->h3 = SHA1_H3; - sctx->h4 = SHA1_H4; + struct sha1_state *sctx = shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + return 0; } -static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data, - unsigned int len, unsigned int partial) +static int __sha1_update(struct sha1_state *sctx, const u8 *data, + unsigned int len, unsigned int partial) { unsigned int done = 0; @@ -56,17 +50,17 @@ static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data, if (partial) { done = SHA1_BLOCK_SIZE - partial; - memcpy(sctx->data + partial, data, done); - sha1_block_data_order(sctx, sctx->data, 1); + memcpy(sctx->buffer + partial, data, done); + sha1_block_data_order(sctx->state, sctx->buffer, 1); } if (len - done >= SHA1_BLOCK_SIZE) { const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; - sha1_block_data_order(sctx, data + done, rounds); + sha1_block_data_order(sctx->state, data + done, rounds); done += rounds * SHA1_BLOCK_SIZE; } - memcpy(sctx->data, data + done, len - done); + memcpy(sctx->buffer, data + done, len - done); return 0; } @@ -74,14 +68,14 @@ static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data, static int sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; int res; /* Handle the fast case right here */ if (partial + len < SHA1_BLOCK_SIZE) { sctx->count += len; - memcpy(sctx->data + partial, data, len); + memcpy(sctx->buffer + partial, data, len); return 0; } res = __sha1_update(sctx, data, len, partial); @@ -92,7 +86,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, /* Add padding and return the message digest. */ static int sha1_final(struct shash_desc *desc, u8 *out) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); unsigned int i, index, padlen; __be32 *dst = (__be32 *)out; __be64 bits; @@ -106,7 +100,7 @@ static int sha1_final(struct shash_desc *desc, u8 *out) /* We need to fill a whole block for __sha1_update() */ if (padlen <= 56) { sctx->count += padlen; - memcpy(sctx->data + index, padding, padlen); + memcpy(sctx->buffer + index, padding, padlen); } else { __sha1_update(sctx, padding, padlen, index); } @@ -114,7 +108,7 @@ static int sha1_final(struct shash_desc *desc, u8 *out) /* Store state in digest */ for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(((u32 *)sctx)[i]); + dst[i] = cpu_to_be32(sctx->state[i]); /* Wipe context */ memset(sctx, 0, sizeof(*sctx)); @@ -124,7 +118,7 @@ static int sha1_final(struct shash_desc *desc, u8 *out) static int sha1_export(struct shash_desc *desc, void *out) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); memcpy(out, sctx, sizeof(*sctx)); return 0; } @@ -132,7 +126,7 @@ static int sha1_export(struct shash_desc *desc, void *out) static int sha1_import(struct shash_desc *desc, const void *in) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); memcpy(sctx, in, sizeof(*sctx)); return 0; } @@ -145,8 +139,8 @@ static struct shash_alg alg = { .final = sha1_final, .export = sha1_export, .import = sha1_import, - .descsize = sizeof(struct SHA1_CTX), - .statesize = sizeof(struct SHA1_CTX), + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name= "sha1-asm", -- cgit v1.2.3 From 67aa286333a3153849aef30160b174bad8f92987 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Tue, 29 Jul 2014 17:14:14 +0100 Subject: ARM: 8119/1: crypto: sha1: add ARM NEON implementation This patch adds ARM NEON assembly implementation of SHA-1 algorithm. tcrypt benchmark results on Cortex-A8, sha1-arm-asm vs sha1-neon-asm: block-size bytes/update old-vs-new 16 16 1.04x 64 16 1.02x 64 64 1.05x 256 16 1.03x 256 64 1.04x 256 256 1.30x 1024 16 1.03x 1024 256 1.36x 1024 1024 1.52x 2048 16 1.03x 2048 256 1.39x 2048 1024 1.55x 2048 2048 1.59x 4096 16 1.03x 4096 256 1.40x 4096 1024 1.57x 4096 4096 1.62x 8192 16 1.03x 8192 256 1.40x 8192 1024 1.58x 8192 4096 1.63x 8192 8192 1.63x Acked-by: Ard Biesheuvel Tested-by: Ard Biesheuvel Signed-off-by: Jussi Kivilinna Signed-off-by: Russell King --- arch/arm/crypto/Makefile | 2 + arch/arm/crypto/sha1-armv7-neon.S | 634 +++++++++++++++++++++++++++++++++++++ arch/arm/crypto/sha1_glue.c | 8 +- arch/arm/crypto/sha1_neon_glue.c | 197 ++++++++++++ arch/arm/include/asm/crypto/sha1.h | 10 + crypto/Kconfig | 11 + 6 files changed, 859 insertions(+), 3 deletions(-) create mode 100644 arch/arm/crypto/sha1-armv7-neon.S create mode 100644 arch/arm/crypto/sha1_neon_glue.c create mode 100644 arch/arm/include/asm/crypto/sha1.h diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 81cda39860c5..374956d2f896 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -5,10 +5,12 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o +obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o aes-arm-y := aes-armv4.o aes_glue.o aes-arm-bs-y := aesbs-core.o aesbs-glue.o sha1-arm-y := sha1-armv4-large.o sha1_glue.o +sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $(<) > $(@) diff --git a/arch/arm/crypto/sha1-armv7-neon.S b/arch/arm/crypto/sha1-armv7-neon.S new file mode 100644 index 000000000000..50013c0e2864 --- /dev/null +++ b/arch/arm/crypto/sha1-armv7-neon.S @@ -0,0 +1,634 @@ +/* sha1-armv7-neon.S - ARM/NEON accelerated SHA-1 transform function + * + * Copyright © 2013-2014 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include + + +.syntax unified +.code 32 +.fpu neon + +.text + + +/* Context structure */ + +#define state_h0 0 +#define state_h1 4 +#define state_h2 8 +#define state_h3 12 +#define state_h4 16 + + +/* Constants */ + +#define K1 0x5A827999 +#define K2 0x6ED9EBA1 +#define K3 0x8F1BBCDC +#define K4 0xCA62C1D6 +.align 4 +.LK_VEC: +.LK1: .long K1, K1, K1, K1 +.LK2: .long K2, K2, K2, K2 +.LK3: .long K3, K3, K3, K3 +.LK4: .long K4, K4, K4, K4 + + +/* Register macros */ + +#define RSTATE r0 +#define RDATA r1 +#define RNBLKS r2 +#define ROLDSTACK r3 +#define RWK lr + +#define _a r4 +#define _b r5 +#define _c r6 +#define _d r7 +#define _e r8 + +#define RT0 r9 +#define RT1 r10 +#define RT2 r11 +#define RT3 r12 + +#define W0 q0 +#define W1 q1 +#define W2 q2 +#define W3 q3 +#define W4 q4 +#define W5 q5 +#define W6 q6 +#define W7 q7 + +#define tmp0 q8 +#define tmp1 q9 +#define tmp2 q10 +#define tmp3 q11 + +#define qK1 q12 +#define qK2 q13 +#define qK3 q14 +#define qK4 q15 + + +/* Round function macros. */ + +#define WK_offs(i) (((i) & 15) * 4) + +#define _R_F1(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + bic RT0, d, b; \ + add e, e, a, ror #(32 - 5); \ + and RT1, c, b; \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add RT0, RT0, RT3; \ + add e, e, RT1; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT0; + +#define _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + eor RT0, d, b; \ + add e, e, a, ror #(32 - 5); \ + eor RT0, RT0, c; \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT3; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT0; \ + +#define _R_F3(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + eor RT0, b, c; \ + and RT1, b, c; \ + add e, e, a, ror #(32 - 5); \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + and RT0, RT0, d; \ + add RT1, RT1, RT3; \ + add e, e, RT0; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT1; + +#define _R_F4(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define _R(a,b,c,d,e,f,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + _R_##f(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define R(a,b,c,d,e,f,i) \ + _R_##f(a,b,c,d,e,i,dummy,dummy,dummy,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define dummy(...) + + +/* Input expansion macros. */ + +/********* Precalc macros for rounds 0-15 *************************************/ + +#define W_PRECALC_00_15() \ + add RWK, sp, #(WK_offs(0)); \ + \ + vld1.32 {tmp0, tmp1}, [RDATA]!; \ + vrev32.8 W0, tmp0; /* big => little */ \ + vld1.32 {tmp2, tmp3}, [RDATA]!; \ + vadd.u32 tmp0, W0, curK; \ + vrev32.8 W7, tmp1; /* big => little */ \ + vrev32.8 W6, tmp2; /* big => little */ \ + vadd.u32 tmp1, W7, curK; \ + vrev32.8 W5, tmp3; /* big => little */ \ + vadd.u32 tmp2, W6, curK; \ + vst1.32 {tmp0, tmp1}, [RWK]!; \ + vadd.u32 tmp3, W5, curK; \ + vst1.32 {tmp2, tmp3}, [RWK]; \ + +#define WPRECALC_00_15_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vld1.32 {tmp0, tmp1}, [RDATA]!; \ + +#define WPRECALC_00_15_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(0)); \ + +#define WPRECALC_00_15_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W0, tmp0; /* big => little */ \ + +#define WPRECALC_00_15_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vld1.32 {tmp2, tmp3}, [RDATA]!; \ + +#define WPRECALC_00_15_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W0, curK; \ + +#define WPRECALC_00_15_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W7, tmp1; /* big => little */ \ + +#define WPRECALC_00_15_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W6, tmp2; /* big => little */ \ + +#define WPRECALC_00_15_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp1, W7, curK; \ + +#define WPRECALC_00_15_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W5, tmp3; /* big => little */ \ + +#define WPRECALC_00_15_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp2, W6, curK; \ + +#define WPRECALC_00_15_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0, tmp1}, [RWK]!; \ + +#define WPRECALC_00_15_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp3, W5, curK; \ + +#define WPRECALC_00_15_12(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp2, tmp3}, [RWK]; \ + + +/********* Precalc macros for rounds 16-31 ************************************/ + +#define WPRECALC_16_31_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0; \ + vext.8 W, W_m16, W_m12, #8; \ + +#define WPRECALC_16_31_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(i)); \ + vext.8 tmp0, W_m04, tmp0, #4; \ + +#define WPRECALC_16_31_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0, W_m16; \ + veor.32 W, W, W_m08; \ + +#define WPRECALC_16_31_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp1, tmp1; \ + veor W, W, tmp0; \ + +#define WPRECALC_16_31_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp0, W, #1; \ + +#define WPRECALC_16_31_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vext.8 tmp1, tmp1, W, #(16-12); \ + vshr.u32 W, W, #31; \ + +#define WPRECALC_16_31_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vorr tmp0, tmp0, W; \ + vshr.u32 W, tmp1, #30; \ + +#define WPRECALC_16_31_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp1, tmp1, #2; \ + +#define WPRECALC_16_31_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0, W; \ + +#define WPRECALC_16_31_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, tmp0, tmp1; \ + +#define WPRECALC_16_31_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W, curK; \ + +#define WPRECALC_16_31_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0}, [RWK]; + + +/********* Precalc macros for rounds 32-79 ************************************/ + +#define WPRECALC_32_79_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, W_m28; \ + +#define WPRECALC_32_79_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vext.8 tmp0, W_m08, W_m04, #8; \ + +#define WPRECALC_32_79_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, W_m16; \ + +#define WPRECALC_32_79_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, tmp0; \ + +#define WPRECALC_32_79_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(i&~3)); \ + +#define WPRECALC_32_79_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp1, W, #2; \ + +#define WPRECALC_32_79_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshr.u32 tmp0, W, #30; \ + +#define WPRECALC_32_79_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vorr W, tmp0, tmp1; \ + +#define WPRECALC_32_79_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W, curK; \ + +#define WPRECALC_32_79_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0}, [RWK]; + + +/* + * Transform nblks*64 bytes (nblks*16 32-bit words) at DATA. + * + * unsigned int + * sha1_transform_neon (void *ctx, const unsigned char *data, + * unsigned int nblks) + */ +.align 3 +ENTRY(sha1_transform_neon) + /* input: + * r0: ctx, CTX + * r1: data (64*nblks bytes) + * r2: nblks + */ + + cmp RNBLKS, #0; + beq .Ldo_nothing; + + push {r4-r12, lr}; + /*vpush {q4-q7};*/ + + adr RT3, .LK_VEC; + + mov ROLDSTACK, sp; + + /* Align stack. */ + sub RT0, sp, #(16*4); + and RT0, #(~(16-1)); + mov sp, RT0; + + vld1.32 {qK1-qK2}, [RT3]!; /* Load K1,K2 */ + + /* Get the values of the chaining variables. */ + ldm RSTATE, {_a-_e}; + + vld1.32 {qK3-qK4}, [RT3]; /* Load K3,K4 */ + +#undef curK +#define curK qK1 + /* Precalc 0-15. */ + W_PRECALC_00_15(); + +.Loop: + /* Transform 0-15 + Precalc 16-31. */ + _R( _a, _b, _c, _d, _e, F1, 0, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 1, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _d, _e, _a, _b, _c, F1, 2, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _c, _d, _e, _a, _b, F1, 3, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,16, + W4, W5, W6, W7, W0, _, _, _ ); + +#undef curK +#define curK qK2 + _R( _b, _c, _d, _e, _a, F1, 4, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 5, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 6, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _d, _e, _a, _b, _c, F1, 7, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,20, + W3, W4, W5, W6, W7, _, _, _ ); + + _R( _c, _d, _e, _a, _b, F1, 8, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _b, _c, _d, _e, _a, F1, 9, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 10, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 11, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,24, + W2, W3, W4, W5, W6, _, _, _ ); + + _R( _d, _e, _a, _b, _c, F1, 12, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _c, _d, _e, _a, _b, F1, 13, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _b, _c, _d, _e, _a, F1, 14, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 15, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,28, + W1, W2, W3, W4, W5, _, _, _ ); + + /* Transform 16-63 + Precalc 32-79. */ + _R( _e, _a, _b, _c, _d, F1, 16, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _d, _e, _a, _b, _c, F1, 17, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _c, _d, _e, _a, _b, F1, 18, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _b, _c, _d, _e, _a, F1, 19, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + + _R( _a, _b, _c, _d, _e, F2, 20, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _e, _a, _b, _c, _d, F2, 21, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _d, _e, _a, _b, _c, F2, 22, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _c, _d, _e, _a, _b, F2, 23, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + +#undef curK +#define curK qK3 + _R( _b, _c, _d, _e, _a, F2, 24, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _a, _b, _c, _d, _e, F2, 25, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _e, _a, _b, _c, _d, F2, 26, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _d, _e, _a, _b, _c, F2, 27, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + + _R( _c, _d, _e, _a, _b, F2, 28, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _b, _c, _d, _e, _a, F2, 29, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _a, _b, _c, _d, _e, F2, 30, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _e, _a, _b, _c, _d, F2, 31, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + + _R( _d, _e, _a, _b, _c, F2, 32, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _c, _d, _e, _a, _b, F2, 33, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _b, _c, _d, _e, _a, F2, 34, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _a, _b, _c, _d, _e, F2, 35, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + + _R( _e, _a, _b, _c, _d, F2, 36, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _d, _e, _a, _b, _c, F2, 37, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _c, _d, _e, _a, _b, F2, 38, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _b, _c, _d, _e, _a, F2, 39, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + + _R( _a, _b, _c, _d, _e, F3, 40, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _e, _a, _b, _c, _d, F3, 41, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _d, _e, _a, _b, _c, F3, 42, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _c, _d, _e, _a, _b, F3, 43, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + +#undef curK +#define curK qK4 + _R( _b, _c, _d, _e, _a, F3, 44, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _a, _b, _c, _d, _e, F3, 45, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _e, _a, _b, _c, _d, F3, 46, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _d, _e, _a, _b, _c, F3, 47, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + + _R( _c, _d, _e, _a, _b, F3, 48, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _b, _c, _d, _e, _a, F3, 49, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _a, _b, _c, _d, _e, F3, 50, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _e, _a, _b, _c, _d, F3, 51, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + + _R( _d, _e, _a, _b, _c, F3, 52, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _c, _d, _e, _a, _b, F3, 53, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _b, _c, _d, _e, _a, F3, 54, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _a, _b, _c, _d, _e, F3, 55, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + + _R( _e, _a, _b, _c, _d, F3, 56, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _d, _e, _a, _b, _c, F3, 57, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _c, _d, _e, _a, _b, F3, 58, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _b, _c, _d, _e, _a, F3, 59, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + + subs RNBLKS, #1; + + _R( _a, _b, _c, _d, _e, F4, 60, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _e, _a, _b, _c, _d, F4, 61, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _d, _e, _a, _b, _c, F4, 62, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _c, _d, _e, _a, _b, F4, 63, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + + beq .Lend; + + /* Transform 64-79 + Precalc 0-15 of next block. */ +#undef curK +#define curK qK1 + _R( _b, _c, _d, _e, _a, F4, 64, + WPRECALC_00_15_0, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 65, + WPRECALC_00_15_1, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _e, _a, _b, _c, _d, F4, 66, + WPRECALC_00_15_2, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _d, _e, _a, _b, _c, F4, 67, + WPRECALC_00_15_3, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _c, _d, _e, _a, _b, F4, 68, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 69, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 70, + WPRECALC_00_15_4, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _e, _a, _b, _c, _d, F4, 71, + WPRECALC_00_15_5, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _d, _e, _a, _b, _c, F4, 72, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _c, _d, _e, _a, _b, F4, 73, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 74, + WPRECALC_00_15_6, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 75, + WPRECALC_00_15_7, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _e, _a, _b, _c, _d, F4, 76, + WPRECALC_00_15_8, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _d, _e, _a, _b, _c, F4, 77, + WPRECALC_00_15_9, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _c, _d, _e, _a, _b, F4, 78, + WPRECALC_00_15_10, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 79, + WPRECALC_00_15_11, dummy, WPRECALC_00_15_12, _, _, _, _, _, _, _, _, _ ); + + /* Update the chaining variables. */ + ldm RSTATE, {RT0-RT3}; + add _a, RT0; + ldr RT0, [RSTATE, #state_h4]; + add _b, RT1; + add _c, RT2; + add _d, RT3; + add _e, RT0; + stm RSTATE, {_a-_e}; + + b .Loop; + +.Lend: + /* Transform 64-79 */ + R( _b, _c, _d, _e, _a, F4, 64 ); + R( _a, _b, _c, _d, _e, F4, 65 ); + R( _e, _a, _b, _c, _d, F4, 66 ); + R( _d, _e, _a, _b, _c, F4, 67 ); + R( _c, _d, _e, _a, _b, F4, 68 ); + R( _b, _c, _d, _e, _a, F4, 69 ); + R( _a, _b, _c, _d, _e, F4, 70 ); + R( _e, _a, _b, _c, _d, F4, 71 ); + R( _d, _e, _a, _b, _c, F4, 72 ); + R( _c, _d, _e, _a, _b, F4, 73 ); + R( _b, _c, _d, _e, _a, F4, 74 ); + R( _a, _b, _c, _d, _e, F4, 75 ); + R( _e, _a, _b, _c, _d, F4, 76 ); + R( _d, _e, _a, _b, _c, F4, 77 ); + R( _c, _d, _e, _a, _b, F4, 78 ); + R( _b, _c, _d, _e, _a, F4, 79 ); + + mov sp, ROLDSTACK; + + /* Update the chaining variables. */ + ldm RSTATE, {RT0-RT3}; + add _a, RT0; + ldr RT0, [RSTATE, #state_h4]; + add _b, RT1; + add _c, RT2; + add _d, RT3; + /*vpop {q4-q7};*/ + add _e, RT0; + stm RSTATE, {_a-_e}; + + pop {r4-r12, pc}; + +.Ldo_nothing: + bx lr +ENDPROC(sha1_transform_neon) diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index c494e579ffc3..84f2a756588b 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -23,6 +23,7 @@ #include #include #include +#include asmlinkage void sha1_block_data_order(u32 *digest, @@ -65,8 +66,8 @@ static int __sha1_update(struct sha1_state *sctx, const u8 *data, } -static int sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +int sha1_update_arm(struct shash_desc *desc, const u8 *data, + unsigned int len) { struct sha1_state *sctx = shash_desc_ctx(desc); unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; @@ -81,6 +82,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, res = __sha1_update(sctx, data, len, partial); return res; } +EXPORT_SYMBOL_GPL(sha1_update_arm); /* Add padding and return the message digest. */ @@ -135,7 +137,7 @@ static int sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_init, - .update = sha1_update, + .update = sha1_update_arm, .final = sha1_final, .export = sha1_export, .import = sha1_import, diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c new file mode 100644 index 000000000000..6f1b411b1d55 --- /dev/null +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -0,0 +1,197 @@ +/* + * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using + * ARM NEON instructions. + * + * Copyright © 2014 Jussi Kivilinna + * + * This file is based on sha1_generic.c and sha1_ssse3_glue.c: + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald + * Copyright (c) Jean-Francois Dive + * Copyright (c) Mathias Krause + * Copyright (c) Chandramouli Narayanan + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +asmlinkage void sha1_transform_neon(void *state_h, const char *data, + unsigned int rounds); + + +static int sha1_neon_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + +static int __sha1_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len, unsigned int partial) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int done = 0; + + sctx->count += len; + + if (partial) { + done = SHA1_BLOCK_SIZE - partial; + memcpy(sctx->buffer + partial, data, done); + sha1_transform_neon(sctx->state, sctx->buffer, 1); + } + + if (len - done >= SHA1_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; + + sha1_transform_neon(sctx->state, data + done, rounds); + done += rounds * SHA1_BLOCK_SIZE; + } + + memcpy(sctx->buffer, data + done, len - done); + + return 0; +} + +static int sha1_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA1_BLOCK_SIZE) { + sctx->count += len; + memcpy(sctx->buffer + partial, data, len); + + return 0; + } + + if (!may_use_simd()) { + res = sha1_update_arm(desc, data, len); + } else { + kernel_neon_begin(); + res = __sha1_neon_update(desc, data, len, partial); + kernel_neon_end(); + } + + return res; +} + + +/* Add padding and return the message digest. */ +static int sha1_neon_final(struct shash_desc *desc, u8 *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be32 *dst = (__be32 *)out; + __be64 bits; + static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->count % SHA1_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); + if (!may_use_simd()) { + sha1_update_arm(desc, padding, padlen); + sha1_update_arm(desc, (const u8 *)&bits, sizeof(bits)); + } else { + kernel_neon_begin(); + /* We need to fill a whole block for __sha1_neon_update() */ + if (padlen <= 56) { + sctx->count += padlen; + memcpy(sctx->buffer + index, padding, padlen); + } else { + __sha1_neon_update(desc, padding, padlen, index); + } + __sha1_neon_update(desc, (const u8 *)&bits, sizeof(bits), 56); + kernel_neon_end(); + } + + /* Store state in digest */ + for (i = 0; i < 5; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha1_neon_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha1_neon_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = sha1_neon_init, + .update = sha1_neon_update, + .final = sha1_neon_final, + .export = sha1_neon_export, + .import = sha1_neon_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sha1_neon_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_shash(&alg); +} + +static void __exit sha1_neon_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(sha1_neon_mod_init); +module_exit(sha1_neon_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, NEON accelerated"); +MODULE_ALIAS("sha1"); diff --git a/arch/arm/include/asm/crypto/sha1.h b/arch/arm/include/asm/crypto/sha1.h new file mode 100644 index 000000000000..75e6a417416b --- /dev/null +++ b/arch/arm/include/asm/crypto/sha1.h @@ -0,0 +1,10 @@ +#ifndef ASM_ARM_CRYPTO_SHA1_H +#define ASM_ARM_CRYPTO_SHA1_H + +#include +#include + +extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, + unsigned int len); + +#endif diff --git a/crypto/Kconfig b/crypto/Kconfig index 66808e1355e3..41265544382d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -533,6 +533,17 @@ config CRYPTO_SHA1_PPC This is the powerpc hardware accelerated implementation of the SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). +config CRYPTO_SHA1_ARM_NEON + tristate "SHA1 digest algorithm (ARM NEON)" + depends on ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN + select CRYPTO_SHA1_ARM + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using optimized ARM NEON assembly, when NEON instructions are + available. + config CRYPTO_SHA256 tristate "SHA224 and SHA256 digest algorithm" select CRYPTO_HASH -- cgit v1.2.3 From 2b996fa96e94dbf6c787fce1ec150b1146f80718 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Tue, 29 Jul 2014 17:15:24 +0100 Subject: ARM: 8120/1: crypto: sha512: add ARM NEON implementation This patch adds ARM NEON assembly implementation of SHA-512 and SHA-384 algorithms. tcrypt benchmark results on Cortex-A8, sha512-generic vs sha512-neon-asm: block-size bytes/update old-vs-new 16 16 2.99x 64 16 2.67x 64 64 3.00x 256 16 2.64x 256 64 3.06x 256 256 3.33x 1024 16 2.53x 1024 256 3.39x 1024 1024 3.52x 2048 16 2.50x 2048 256 3.41x 2048 1024 3.54x 2048 2048 3.57x 4096 16 2.49x 4096 256 3.42x 4096 1024 3.56x 4096 4096 3.59x 8192 16 2.48x 8192 256 3.42x 8192 1024 3.56x 8192 4096 3.60x 8192 8192 3.60x Acked-by: Ard Biesheuvel Tested-by: Ard Biesheuvel Signed-off-by: Jussi Kivilinna Signed-off-by: Russell King --- arch/arm/crypto/Makefile | 2 + arch/arm/crypto/sha512-armv7-neon.S | 455 ++++++++++++++++++++++++++++++++++++ arch/arm/crypto/sha512_neon_glue.c | 305 ++++++++++++++++++++++++ crypto/Kconfig | 15 ++ 4 files changed, 777 insertions(+) create mode 100644 arch/arm/crypto/sha512-armv7-neon.S create mode 100644 arch/arm/crypto/sha512_neon_glue.c diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 374956d2f896..b48fa341648d 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -6,11 +6,13 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o +obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o aes-arm-y := aes-armv4.o aes_glue.o aes-arm-bs-y := aesbs-core.o aesbs-glue.o sha1-arm-y := sha1-armv4-large.o sha1_glue.o sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o +sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $(<) > $(@) diff --git a/arch/arm/crypto/sha512-armv7-neon.S b/arch/arm/crypto/sha512-armv7-neon.S new file mode 100644 index 000000000000..fe99472e507c --- /dev/null +++ b/arch/arm/crypto/sha512-armv7-neon.S @@ -0,0 +1,455 @@ +/* sha512-armv7-neon.S - ARM/NEON assembly implementation of SHA-512 transform + * + * Copyright © 2013-2014 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include + + +.syntax unified +.code 32 +.fpu neon + +.text + +/* structure of SHA512_CONTEXT */ +#define hd_a 0 +#define hd_b ((hd_a) + 8) +#define hd_c ((hd_b) + 8) +#define hd_d ((hd_c) + 8) +#define hd_e ((hd_d) + 8) +#define hd_f ((hd_e) + 8) +#define hd_g ((hd_f) + 8) + +/* register macros */ +#define RK %r2 + +#define RA d0 +#define RB d1 +#define RC d2 +#define RD d3 +#define RE d4 +#define RF d5 +#define RG d6 +#define RH d7 + +#define RT0 d8 +#define RT1 d9 +#define RT2 d10 +#define RT3 d11 +#define RT4 d12 +#define RT5 d13 +#define RT6 d14 +#define RT7 d15 + +#define RT01q q4 +#define RT23q q5 +#define RT45q q6 +#define RT67q q7 + +#define RW0 d16 +#define RW1 d17 +#define RW2 d18 +#define RW3 d19 +#define RW4 d20 +#define RW5 d21 +#define RW6 d22 +#define RW7 d23 +#define RW8 d24 +#define RW9 d25 +#define RW10 d26 +#define RW11 d27 +#define RW12 d28 +#define RW13 d29 +#define RW14 d30 +#define RW15 d31 + +#define RW01q q8 +#define RW23q q9 +#define RW45q q10 +#define RW67q q11 +#define RW89q q12 +#define RW1011q q13 +#define RW1213q q14 +#define RW1415q q15 + +/*********************************************************************** + * ARM assembly implementation of sha512 transform + ***********************************************************************/ +#define rounds2_0_63(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, rw01q, rw2, \ + rw23q, rw1415q, rw9, rw10, interleave_op, arg1) \ + /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ + vshr.u64 RT2, re, #14; \ + vshl.u64 RT3, re, #64 - 14; \ + interleave_op(arg1); \ + vshr.u64 RT4, re, #18; \ + vshl.u64 RT5, re, #64 - 18; \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, re, #41; \ + vshl.u64 RT5, re, #64 - 41; \ + vadd.u64 RT0, RT0, rw0; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, re; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, rf, rg; \ + \ + vadd.u64 RT1, RT1, rh; \ + vshr.u64 RT2, ra, #28; \ + vshl.u64 RT3, ra, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, ra, #34; \ + vshl.u64 RT5, ra, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* h = Sum0 (a) + Maj (a, b, c); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, ra, #39; \ + vshl.u64 RT5, ra, #64 - 39; \ + veor.64 RT0, ra, rb; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rc, rb; \ + vadd.u64 rd, rd, RT1; /* d+=t1; */ \ + veor.64 rh, RT2, RT3; \ + \ + /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ + vshr.u64 RT2, rd, #14; \ + vshl.u64 RT3, rd, #64 - 14; \ + vadd.u64 rh, rh, RT0; \ + vshr.u64 RT4, rd, #18; \ + vshl.u64 RT5, rd, #64 - 18; \ + vadd.u64 rh, rh, RT1; /* h+=t1; */ \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rd, #41; \ + vshl.u64 RT5, rd, #64 - 41; \ + vadd.u64 RT0, RT0, rw1; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, rd; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, re, rf; \ + \ + vadd.u64 RT1, RT1, rg; \ + vshr.u64 RT2, rh, #28; \ + vshl.u64 RT3, rh, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, rh, #34; \ + vshl.u64 RT5, rh, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* g = Sum0 (h) + Maj (h, a, b); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rh, #39; \ + vshl.u64 RT5, rh, #64 - 39; \ + veor.64 RT0, rh, ra; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rb, ra; \ + vadd.u64 rc, rc, RT1; /* c+=t1; */ \ + veor.64 rg, RT2, RT3; \ + \ + /* w[0] += S1 (w[14]) + w[9] + S0 (w[1]); */ \ + /* w[1] += S1 (w[15]) + w[10] + S0 (w[2]); */ \ + \ + /**** S0(w[1:2]) */ \ + \ + /* w[0:1] += w[9:10] */ \ + /* RT23q = rw1:rw2 */ \ + vext.u64 RT23q, rw01q, rw23q, #1; \ + vadd.u64 rw0, rw9; \ + vadd.u64 rg, rg, RT0; \ + vadd.u64 rw1, rw10;\ + vadd.u64 rg, rg, RT1; /* g+=t1; */ \ + \ + vshr.u64 RT45q, RT23q, #1; \ + vshl.u64 RT67q, RT23q, #64 - 1; \ + vshr.u64 RT01q, RT23q, #8; \ + veor.u64 RT45q, RT45q, RT67q; \ + vshl.u64 RT67q, RT23q, #64 - 8; \ + veor.u64 RT45q, RT45q, RT01q; \ + vshr.u64 RT01q, RT23q, #7; \ + veor.u64 RT45q, RT45q, RT67q; \ + \ + /**** S1(w[14:15]) */ \ + vshr.u64 RT23q, rw1415q, #6; \ + veor.u64 RT01q, RT01q, RT45q; \ + vshr.u64 RT45q, rw1415q, #19; \ + vshl.u64 RT67q, rw1415q, #64 - 19; \ + veor.u64 RT23q, RT23q, RT45q; \ + vshr.u64 RT45q, rw1415q, #61; \ + veor.u64 RT23q, RT23q, RT67q; \ + vshl.u64 RT67q, rw1415q, #64 - 61; \ + veor.u64 RT23q, RT23q, RT45q; \ + vadd.u64 rw01q, RT01q; /* w[0:1] += S(w[1:2]) */ \ + veor.u64 RT01q, RT23q, RT67q; +#define vadd_RT01q(rw01q) \ + /* w[0:1] += S(w[14:15]) */ \ + vadd.u64 rw01q, RT01q; + +#define dummy(_) /*_*/ + +#define rounds2_64_79(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, \ + interleave_op1, arg1, interleave_op2, arg2) \ + /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ + vshr.u64 RT2, re, #14; \ + vshl.u64 RT3, re, #64 - 14; \ + interleave_op1(arg1); \ + vshr.u64 RT4, re, #18; \ + vshl.u64 RT5, re, #64 - 18; \ + interleave_op2(arg2); \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, re, #41; \ + vshl.u64 RT5, re, #64 - 41; \ + vadd.u64 RT0, RT0, rw0; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, re; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, rf, rg; \ + \ + vadd.u64 RT1, RT1, rh; \ + vshr.u64 RT2, ra, #28; \ + vshl.u64 RT3, ra, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, ra, #34; \ + vshl.u64 RT5, ra, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* h = Sum0 (a) + Maj (a, b, c); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, ra, #39; \ + vshl.u64 RT5, ra, #64 - 39; \ + veor.64 RT0, ra, rb; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rc, rb; \ + vadd.u64 rd, rd, RT1; /* d+=t1; */ \ + veor.64 rh, RT2, RT3; \ + \ + /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ + vshr.u64 RT2, rd, #14; \ + vshl.u64 RT3, rd, #64 - 14; \ + vadd.u64 rh, rh, RT0; \ + vshr.u64 RT4, rd, #18; \ + vshl.u64 RT5, rd, #64 - 18; \ + vadd.u64 rh, rh, RT1; /* h+=t1; */ \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rd, #41; \ + vshl.u64 RT5, rd, #64 - 41; \ + vadd.u64 RT0, RT0, rw1; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, rd; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, re, rf; \ + \ + vadd.u64 RT1, RT1, rg; \ + vshr.u64 RT2, rh, #28; \ + vshl.u64 RT3, rh, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, rh, #34; \ + vshl.u64 RT5, rh, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* g = Sum0 (h) + Maj (h, a, b); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rh, #39; \ + vshl.u64 RT5, rh, #64 - 39; \ + veor.64 RT0, rh, ra; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rb, ra; \ + vadd.u64 rc, rc, RT1; /* c+=t1; */ \ + veor.64 rg, RT2, RT3; +#define vadd_rg_RT0(rg) \ + vadd.u64 rg, rg, RT0; +#define vadd_rg_RT1(rg) \ + vadd.u64 rg, rg, RT1; /* g+=t1; */ + +.align 3 +ENTRY(sha512_transform_neon) + /* Input: + * %r0: SHA512_CONTEXT + * %r1: data + * %r2: u64 k[] constants + * %r3: nblks + */ + push {%lr}; + + mov %lr, #0; + + /* Load context to d0-d7 */ + vld1.64 {RA-RD}, [%r0]!; + vld1.64 {RE-RH}, [%r0]; + sub %r0, #(4*8); + + /* Load input to w[16], d16-d31 */ + /* NOTE: Assumes that on ARMv7 unaligned accesses are always allowed. */ + vld1.64 {RW0-RW3}, [%r1]!; + vld1.64 {RW4-RW7}, [%r1]!; + vld1.64 {RW8-RW11}, [%r1]!; + vld1.64 {RW12-RW15}, [%r1]!; +#ifdef __ARMEL__ + /* byteswap */ + vrev64.8 RW01q, RW01q; + vrev64.8 RW23q, RW23q; + vrev64.8 RW45q, RW45q; + vrev64.8 RW67q, RW67q; + vrev64.8 RW89q, RW89q; + vrev64.8 RW1011q, RW1011q; + vrev64.8 RW1213q, RW1213q; + vrev64.8 RW1415q, RW1415q; +#endif + + /* EABI says that d8-d15 must be preserved by callee. */ + /*vpush {RT0-RT7};*/ + +.Loop: + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, + RW23q, RW1415q, RW9, RW10, dummy, _); + b .Lenter_rounds; + +.Loop_rounds: + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, + RW23q, RW1415q, RW9, RW10, vadd_RT01q, RW1415q); +.Lenter_rounds: + rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, RW23q, RW4, + RW45q, RW01q, RW11, RW12, vadd_RT01q, RW01q); + rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, RW45q, RW6, + RW67q, RW23q, RW13, RW14, vadd_RT01q, RW23q); + rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, RW67q, RW8, + RW89q, RW45q, RW15, RW0, vadd_RT01q, RW45q); + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, RW89q, RW10, + RW1011q, RW67q, RW1, RW2, vadd_RT01q, RW67q); + rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, RW1011q, RW12, + RW1213q, RW89q, RW3, RW4, vadd_RT01q, RW89q); + add %lr, #16; + rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, RW1213q, RW14, + RW1415q, RW1011q, RW5, RW6, vadd_RT01q, RW1011q); + cmp %lr, #64; + rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, RW1415q, RW0, + RW01q, RW1213q, RW7, RW8, vadd_RT01q, RW1213q); + bne .Loop_rounds; + + subs %r3, #1; + + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, + vadd_RT01q, RW1415q, dummy, _); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); + beq .Lhandle_tail; + vld1.64 {RW0-RW3}, [%r1]!; + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); +#ifdef __ARMEL__ + vrev64.8 RW01q, RW01q; + vrev64.8 RW23q, RW23q; +#endif + vld1.64 {RW4-RW7}, [%r1]!; + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, + vadd_rg_RT0, RA, vadd_rg_RT1, RA); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); +#ifdef __ARMEL__ + vrev64.8 RW45q, RW45q; + vrev64.8 RW67q, RW67q; +#endif + vld1.64 {RW8-RW11}, [%r1]!; + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); +#ifdef __ARMEL__ + vrev64.8 RW89q, RW89q; + vrev64.8 RW1011q, RW1011q; +#endif + vld1.64 {RW12-RW15}, [%r1]!; + vadd_rg_RT0(RA); + vadd_rg_RT1(RA); + + /* Load context */ + vld1.64 {RT0-RT3}, [%r0]!; + vld1.64 {RT4-RT7}, [%r0]; + sub %r0, #(4*8); + +#ifdef __ARMEL__ + vrev64.8 RW1213q, RW1213q; + vrev64.8 RW1415q, RW1415q; +#endif + + vadd.u64 RA, RT0; + vadd.u64 RB, RT1; + vadd.u64 RC, RT2; + vadd.u64 RD, RT3; + vadd.u64 RE, RT4; + vadd.u64 RF, RT5; + vadd.u64 RG, RT6; + vadd.u64 RH, RT7; + + /* Store the first half of context */ + vst1.64 {RA-RD}, [%r0]!; + sub RK, $(8*80); + vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ + mov %lr, #0; + sub %r0, #(4*8); + + b .Loop; + +.Lhandle_tail: + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, + vadd_rg_RT0, RA, vadd_rg_RT1, RA); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); + + /* Load context to d16-d23 */ + vld1.64 {RW0-RW3}, [%r0]!; + vadd_rg_RT0(RA); + vld1.64 {RW4-RW7}, [%r0]; + vadd_rg_RT1(RA); + sub %r0, #(4*8); + + vadd.u64 RA, RW0; + vadd.u64 RB, RW1; + vadd.u64 RC, RW2; + vadd.u64 RD, RW3; + vadd.u64 RE, RW4; + vadd.u64 RF, RW5; + vadd.u64 RG, RW6; + vadd.u64 RH, RW7; + + /* Store the first half of context */ + vst1.64 {RA-RD}, [%r0]!; + + /* Clear used registers */ + /* d16-d31 */ + veor.u64 RW01q, RW01q; + veor.u64 RW23q, RW23q; + veor.u64 RW45q, RW45q; + veor.u64 RW67q, RW67q; + vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ + veor.u64 RW89q, RW89q; + veor.u64 RW1011q, RW1011q; + veor.u64 RW1213q, RW1213q; + veor.u64 RW1415q, RW1415q; + /* d8-d15 */ + /*vpop {RT0-RT7};*/ + /* d0-d7 (q0-q3) */ + veor.u64 %q0, %q0; + veor.u64 %q1, %q1; + veor.u64 %q2, %q2; + veor.u64 %q3, %q3; + + pop {%pc}; +ENDPROC(sha512_transform_neon) diff --git a/arch/arm/crypto/sha512_neon_glue.c b/arch/arm/crypto/sha512_neon_glue.c new file mode 100644 index 000000000000..0d2758ff5e12 --- /dev/null +++ b/arch/arm/crypto/sha512_neon_glue.c @@ -0,0 +1,305 @@ +/* + * Glue code for the SHA512 Secure Hash Algorithm assembly implementation + * using NEON instructions. + * + * Copyright © 2014 Jussi Kivilinna + * + * This file is based on sha512_ssse3_glue.c: + * Copyright (C) 2013 Intel Corporation + * Author: Tim Chen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static const u64 sha512_k[] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL +}; + + +asmlinkage void sha512_transform_neon(u64 *digest, const void *data, + const u64 k[], unsigned int num_blks); + + +static int sha512_neon_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA512_H0; + sctx->state[1] = SHA512_H1; + sctx->state[2] = SHA512_H2; + sctx->state[3] = SHA512_H3; + sctx->state[4] = SHA512_H4; + sctx->state[5] = SHA512_H5; + sctx->state[6] = SHA512_H6; + sctx->state[7] = SHA512_H7; + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int __sha512_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len, unsigned int partial) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int done = 0; + + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + + if (partial) { + done = SHA512_BLOCK_SIZE - partial; + memcpy(sctx->buf + partial, data, done); + sha512_transform_neon(sctx->state, sctx->buf, sha512_k, 1); + } + + if (len - done >= SHA512_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; + + sha512_transform_neon(sctx->state, data + done, sha512_k, + rounds); + + done += rounds * SHA512_BLOCK_SIZE; + } + + memcpy(sctx->buf, data + done, len - done); + + return 0; +} + +static int sha512_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA512_BLOCK_SIZE) { + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + memcpy(sctx->buf + partial, data, len); + + return 0; + } + + if (!may_use_simd()) { + res = crypto_sha512_update(desc, data, len); + } else { + kernel_neon_begin(); + res = __sha512_neon_update(desc, data, len, partial); + kernel_neon_end(); + } + + return res; +} + + +/* Add padding and return the message digest. */ +static int sha512_neon_final(struct shash_desc *desc, u8 *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be64 *dst = (__be64 *)out; + __be64 bits[2]; + static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; + + /* save number of bits */ + bits[1] = cpu_to_be64(sctx->count[0] << 3); + bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); + + /* Pad out to 112 mod 128 and append length */ + index = sctx->count[0] & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128+112) - index); + + if (!may_use_simd()) { + crypto_sha512_update(desc, padding, padlen); + crypto_sha512_update(desc, (const u8 *)&bits, sizeof(bits)); + } else { + kernel_neon_begin(); + /* We need to fill a whole block for __sha512_neon_update() */ + if (padlen <= 112) { + sctx->count[0] += padlen; + if (sctx->count[0] < padlen) + sctx->count[1]++; + memcpy(sctx->buf + index, padding, padlen); + } else { + __sha512_neon_update(desc, padding, padlen, index); + } + __sha512_neon_update(desc, (const u8 *)&bits, + sizeof(bits), 112); + kernel_neon_end(); + } + + /* Store state in digest */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be64(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha512_neon_export(struct shash_desc *desc, void *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha512_neon_import(struct shash_desc *desc, const void *in) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static int sha384_neon_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA384_H0; + sctx->state[1] = SHA384_H1; + sctx->state[2] = SHA384_H2; + sctx->state[3] = SHA384_H3; + sctx->state[4] = SHA384_H4; + sctx->state[5] = SHA384_H5; + sctx->state[6] = SHA384_H6; + sctx->state[7] = SHA384_H7; + + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int sha384_neon_final(struct shash_desc *desc, u8 *hash) +{ + u8 D[SHA512_DIGEST_SIZE]; + + sha512_neon_final(desc, D); + + memcpy(hash, D, SHA384_DIGEST_SIZE); + memset(D, 0, SHA512_DIGEST_SIZE); + + return 0; +} + +static struct shash_alg algs[] = { { + .digestsize = SHA512_DIGEST_SIZE, + .init = sha512_neon_init, + .update = sha512_neon_update, + .final = sha512_neon_final, + .export = sha512_neon_export, + .import = sha512_neon_import, + .descsize = sizeof(struct sha512_state), + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .digestsize = SHA384_DIGEST_SIZE, + .init = sha384_neon_init, + .update = sha512_neon_update, + .final = sha384_neon_final, + .export = sha512_neon_export, + .import = sha512_neon_import, + .descsize = sizeof(struct sha512_state), + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; + +static int __init sha512_neon_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} + +static void __exit sha512_neon_mod_fini(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} + +module_init(sha512_neon_mod_init); +module_exit(sha512_neon_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, NEON accelerated"); + +MODULE_ALIAS("sha512"); +MODULE_ALIAS("sha384"); diff --git a/crypto/Kconfig b/crypto/Kconfig index 41265544382d..b7cc3cb3a613 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -577,6 +577,21 @@ config CRYPTO_SHA512 This code also includes SHA-384, a 384 bit hash with 192 bits of security against collision attacks. +config CRYPTO_SHA512_ARM_NEON + tristate "SHA384 and SHA512 digest algorithm (ARM NEON)" + depends on ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-512 secure hash standard (DFIPS 180-2) implemented + using ARM NEON instructions, when available. + + This version of SHA implements a 512 bit hash with 256 bits of + security against collision attacks. + + This code also includes SHA-384, a 384 bit hash with 192 bits + of security against collision attacks. + config CRYPTO_SHA512_SPARC64 tristate "SHA384 and SHA512 digest algorithm (SPARC64)" depends on SPARC64 -- cgit v1.2.3 From 7c8865cc8b5afabec1952a172dbd79350e1844c0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 16 May 2013 11:41:48 +0200 Subject: ARM: add support for kernel mode NEON In order to safely support the use of NEON instructions in kernel mode, some precautions need to be taken: - the userland context that may be present in the registers (even if the NEON/VFP is currently disabled) must be stored under the correct task (which may not be 'current' in the UP case), - to avoid having to keep track of additional vfpstates for the kernel side, disallow the use of NEON in interrupt context and run with preemption disabled, - after use, re-enable preemption and re-enable the lazy restore machinery by disabling the NEON/VFP unit. This patch adds the functions kernel_neon_begin() and kernel_neon_end() which take care of the above. It also adds the Kconfig symbol KERNEL_MODE_NEON to enable it. Change-Id: I286f9d414e87568f094b7782762faea46c6d4831 Signed-off-by: Ard Biesheuvel Acked-by: Nicolas Pitre Signed-off-by: Chris Fries --- arch/arm/Kconfig | 7 +++++++ arch/arm/include/asm/neon.h | 36 ++++++++++++++++++++++++++++++++++ arch/arm/vfp/vfpmodule.c | 47 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 arch/arm/include/asm/neon.h diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 99887aaa04bb..29481f2b12cb 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -2221,6 +2221,13 @@ config NEON Say Y to include support code for NEON, the ARMv7 Advanced SIMD Extension. +config KERNEL_MODE_NEON + bool "Support for NEON in kernel mode" + default n + depends on NEON + help + Say Y to include support for NEON in kernel mode. + endmenu menu "Userspace binary formats" diff --git a/arch/arm/include/asm/neon.h b/arch/arm/include/asm/neon.h new file mode 100644 index 000000000000..8f730fe70093 --- /dev/null +++ b/arch/arm/include/asm/neon.h @@ -0,0 +1,36 @@ +/* + * linux/arch/arm/include/asm/neon.h + * + * Copyright (C) 2013 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +#define cpu_has_neon() (!!(elf_hwcap & HWCAP_NEON)) + +#ifdef __ARM_NEON__ + +/* + * If you are affected by the BUILD_BUG below, it probably means that you are + * using NEON code /and/ calling the kernel_neon_begin() function from the same + * compilation unit. To prevent issues that may arise from GCC reordering or + * generating(1) NEON instructions outside of these begin/end functions, the + * only supported way of using NEON code in the kernel is by isolating it in a + * separate compilation unit, and calling it from another unit from inside a + * kernel_neon_begin/kernel_neon_end pair. + * + * (1) Current GCC (4.7) might generate NEON instructions at O3 level if + * -mpfu=neon is set. + */ + +#define kernel_neon_begin() \ + BUILD_BUG_ON_MSG(1, "kernel_neon_begin() called from NEON code") + +#else +void kernel_neon_begin(void); +#endif +void kernel_neon_end(void); diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 5dfbb0b8e7f4..4cdb1b14110a 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -648,6 +649,52 @@ static int vfp_hotplug(struct notifier_block *b, unsigned long action, return NOTIFY_OK; } +#ifdef CONFIG_KERNEL_MODE_NEON + +/* + * Kernel-side NEON support functions + */ +void kernel_neon_begin(void) +{ + struct thread_info *thread = current_thread_info(); + unsigned int cpu; + u32 fpexc; + + /* + * Kernel mode NEON is only allowed outside of interrupt context + * with preemption disabled. This will make sure that the kernel + * mode NEON register contents never need to be preserved. + */ + BUG_ON(in_interrupt()); + cpu = get_cpu(); + + fpexc = fmrx(FPEXC) | FPEXC_EN; + fmxr(FPEXC, fpexc); + + /* + * Save the userland NEON/VFP state. Under UP, + * the owner could be a task other than 'current' + */ + if (vfp_state_in_hw(cpu, thread)) + vfp_save_state(&thread->vfpstate, fpexc); +#ifndef CONFIG_SMP + else if (vfp_current_hw_state[cpu] != NULL) + vfp_save_state(vfp_current_hw_state[cpu], fpexc); +#endif + vfp_current_hw_state[cpu] = NULL; +} +EXPORT_SYMBOL(kernel_neon_begin); + +void kernel_neon_end(void) +{ + /* Disable the NEON/VFP unit. */ + fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); + put_cpu(); +} +EXPORT_SYMBOL(kernel_neon_end); + +#endif /* CONFIG_KERNEL_MODE_NEON */ + /* * VFP support code initialisation. */ -- cgit v1.2.3 From 47c7a7c1f8f53e7d4f5ab38cac484718b6840c30 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sat, 21 Sep 2013 11:23:50 +0100 Subject: ARM: 7837/3: fix Thumb-2 bug in AES assembler code Patch 638591c enabled building the AES assembler code in Thumb2 mode. However, this code used arithmetic involving PC rather than adr{l} instructions to generate PC-relative references to the lookup tables, and this needs to take into account the different PC offset when running in Thumb mode. Signed-off-by: Ard Biesheuvel Acked-by: Nicolas Pitre Cc: stable@vger.kernel.org Signed-off-by: Russell King --- arch/arm/crypto/aes-armv4.S | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S index 19d6cd6f29f9..3a14ea8fe97e 100644 --- a/arch/arm/crypto/aes-armv4.S +++ b/arch/arm/crypto/aes-armv4.S @@ -148,7 +148,7 @@ AES_Te: @ const AES_KEY *key) { .align 5 ENTRY(AES_encrypt) - sub r3,pc,#8 @ AES_encrypt + adr r3,AES_encrypt stmdb sp!,{r1,r4-r12,lr} mov r12,r0 @ inp mov r11,r2 @@ -381,7 +381,7 @@ _armv4_AES_encrypt: .align 5 ENTRY(private_AES_set_encrypt_key) _armv4_AES_set_encrypt_key: - sub r3,pc,#8 @ AES_set_encrypt_key + adr r3,_armv4_AES_set_encrypt_key teq r0,#0 moveq r0,#-1 beq .Labrt @@ -843,7 +843,7 @@ AES_Td: @ const AES_KEY *key) { .align 5 ENTRY(AES_decrypt) - sub r3,pc,#8 @ AES_decrypt + adr r3,AES_decrypt stmdb sp!,{r1,r4-r12,lr} mov r12,r0 @ inp mov r11,r2 -- cgit v1.2.3 From dd74ce89359fa76f1206a336915e7ed7b4243021 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 22 May 2013 10:38:53 +0200 Subject: ARM: move VFP init to an earlier boot stage In order to use the NEON unit in the kernel, we should initialize it a bit earlier in the boot process so NEON users that like to do a quick benchmark at load time (like the xor_blocks or RAID-6 code) find the NEON/VFP unit already enabled. Replaced late_initcall() with core_initcall(). Signed-off-by: Ard Biesheuvel Acked-by: Nicolas Pitre --- arch/arm/vfp/vfpmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 4cdb1b14110a..452fb3ad68aa 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -778,4 +778,4 @@ static int __init vfp_init(void) return 0; } -late_initcall(vfp_init); +core_initcall(vfp_init); -- cgit v1.2.3 From abb0f65e46bc78fa3495f0cbca0200c2d495e1b0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 20 Sep 2013 09:57:37 +0200 Subject: ARM: pull in from asm-generic Signed-off-by: Ard Biesheuvel --- arch/arm/include/asm/Kbuild | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild index d3db39860b9c..6577b8aeb711 100644 --- a/arch/arm/include/asm/Kbuild +++ b/arch/arm/include/asm/Kbuild @@ -24,6 +24,7 @@ generic-y += sembuf.h generic-y += serial.h generic-y += shmbuf.h generic-y += siginfo.h +generic-y += simd.h generic-y += sizes.h generic-y += socket.h generic-y += sockios.h -- cgit v1.2.3 From 3b277924f2e01f9ff236fd4c0c4000d63ec42d79 Mon Sep 17 00:00:00 2001 From: Dmitry Shmidt Date: Mon, 12 Jan 2015 13:42:05 -0800 Subject: wlan: Add get_wake_irq functionality Change-Id: Ic41f06c509b2e625dc9ec4131903db6920c5fd4e Signed-off-by: Dmitry Shmidt --- include/linux/wlan_plat.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/wlan_plat.h b/include/linux/wlan_plat.h index 8ad2dbd0c296..8e8b06f1ba4a 100644 --- a/include/linux/wlan_plat.h +++ b/include/linux/wlan_plat.h @@ -23,6 +23,7 @@ struct wifi_platform_data { int (*set_carddetect)(int val); void *(*mem_prealloc)(int section, unsigned long size); int (*get_mac_addr)(unsigned char *buf); + int (*get_wake_irq)(void); void *(*get_country_code)(char *ccode, u32 flags); }; -- cgit v1.2.3 From c0d201a31f799a74efa0faa8638d738794fac46b Mon Sep 17 00:00:00 2001 From: Dmitry Shmidt Date: Tue, 13 Jan 2015 13:52:49 -0800 Subject: irq: pm: Remove unused variable Change-Id: Ie4311b554628af878cd80fd0abc03b2be294f0bf Signed-off-by: Dmitry Shmidt --- kernel/irq/pm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/irq/pm.c b/kernel/irq/pm.c index 08d0916150d5..4ea5ced207a6 100644 --- a/kernel/irq/pm.c +++ b/kernel/irq/pm.c @@ -100,7 +100,6 @@ EXPORT_SYMBOL_GPL(resume_device_irqs); int check_wakeup_irqs(void) { struct irq_desc *desc; - char suspend_abort[MAX_SUSPEND_ABORT_LEN]; int irq; for_each_irq_desc(irq, desc) { -- cgit v1.2.3 From 7d97028e4c6727decc14fa7d6e6d3d846dbb1de0 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Fri, 9 Jan 2015 19:08:03 -0800 Subject: Revert "Staging: android: binder: More offset validation." This reverts commit 3fac2c119f537d4d8fea3f0b9063d72f44857b82. Change-Id: I8840b43eceff9ef52d9bae2079d22046488a4ec2 --- drivers/staging/android/binder.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 9c821f398cad..e718c84372a6 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1307,7 +1307,6 @@ static void binder_transaction(struct binder_proc *proc, struct binder_transaction *t; struct binder_work *tcomplete; binder_size_t *offp, *off_end; - binder_size_t off_min; struct binder_proc *target_proc; struct binder_thread *target_thread = NULL; struct binder_node *target_node = NULL; @@ -1506,23 +1505,17 @@ static void binder_transaction(struct binder_proc *proc, goto err_bad_offset; } off_end = (void *)offp + tr->offsets_size; - off_min = 0; for (; offp < off_end; offp++) { struct flat_binder_object *fp; if (*offp > t->buffer->data_size - sizeof(*fp) || - *offp < off_min || t->buffer->data_size < sizeof(*fp) || !IS_ALIGNED(*offp, sizeof(u32))) { - binder_user_error("%d:%d got transaction with invalid offset, %lld (min %lld, max %lld)\n", - proc->pid, thread->pid, (u64)*offp, - (u64)off_min, - (u64)(t->buffer->data_size - - sizeof(*fp))); + binder_user_error("%d:%d got transaction with invalid offset, %lld\n", + proc->pid, thread->pid, (u64)*offp); return_error = BR_FAILED_REPLY; goto err_bad_offset; } fp = (struct flat_binder_object *)(t->buffer->data + *offp); - off_min = *offp + sizeof(struct flat_binder_object); switch (fp->type) { case BINDER_TYPE_BINDER: case BINDER_TYPE_WEAK_BINDER: { -- cgit v1.2.3 From 889cead9fa4bbb2a7f1a9dd3e715e38309497ad0 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Fri, 9 Jan 2015 19:10:24 -0800 Subject: Revert "ARM: tegra: flounder: stick to 32bit binder for now." This reverts commit 8179b7b7fe7e0ee85f670d109de6d7e3be46b2a9. Change-Id: Iba8c90cc88b39dd357ecbd7c942469966b53f123 --- drivers/staging/android/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig index c5539001f444..adcbffa5060f 100644 --- a/drivers/staging/android/Kconfig +++ b/drivers/staging/android/Kconfig @@ -21,7 +21,6 @@ config ANDROID_BINDER_IPC config ANDROID_BINDER_IPC_32BIT bool "Use old 32-bit binder api" - default y depends on !64BIT ---help--- Enable to support an old 32-bit Android user-space. Breaks the new -- cgit v1.2.3 From b4804ef90f54068bfefd6a6f8c687e7844912c1d Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Fri, 9 Jan 2015 19:10:36 -0800 Subject: Revert "Staging: android: binder: Support concurrent 32 bit and 64 bit processes." This reverts commit 2d595dc92ae19b49e4c1ebb1f8e3b461a2d06592. Change-Id: I1e4e306fa38f851b3044abb8a7c929c298c14812 --- drivers/staging/android/Kconfig | 7 - drivers/staging/android/binder.c | 253 +++++++++++++++------------------ drivers/staging/android/binder.h | 4 - drivers/staging/android/binder_trace.h | 14 +- drivers/staging/android/uapi/binder.h | 60 ++++---- 5 files changed, 147 insertions(+), 191 deletions(-) diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig index adcbffa5060f..1d6f5cf5826c 100644 --- a/drivers/staging/android/Kconfig +++ b/drivers/staging/android/Kconfig @@ -19,13 +19,6 @@ config ANDROID_BINDER_IPC Android process, using Binder to identify, invoke and pass arguments between said processes. -config ANDROID_BINDER_IPC_32BIT - bool "Use old 32-bit binder api" - depends on !64BIT - ---help--- - Enable to support an old 32-bit Android user-space. Breaks the new - Android user-space. - config ASHMEM bool "Enable the Anonymous Shared Memory Subsystem" default n diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index e718c84372a6..6e4497026b26 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -230,8 +230,8 @@ struct binder_node { int internal_strong_refs; int local_weak_refs; int local_strong_refs; - binder_uintptr_t ptr; - binder_uintptr_t cookie; + void __user *ptr; + void __user *cookie; unsigned has_strong_ref:1; unsigned pending_strong_ref:1; unsigned has_weak_ref:1; @@ -244,7 +244,7 @@ struct binder_node { struct binder_ref_death { struct binder_work work; - binder_uintptr_t cookie; + void __user *cookie; }; struct binder_ref { @@ -517,14 +517,14 @@ static void binder_insert_allocated_buffer(struct binder_proc *proc, } static struct binder_buffer *binder_buffer_lookup(struct binder_proc *proc, - uintptr_t user_ptr) + void __user *user_ptr) { struct rb_node *n = proc->allocated_buffers.rb_node; struct binder_buffer *buffer; struct binder_buffer *kern_ptr; - kern_ptr = (struct binder_buffer *)(user_ptr - proc->user_buffer_offset - - offsetof(struct binder_buffer, data)); + kern_ptr = user_ptr - proc->user_buffer_offset + - offsetof(struct binder_buffer, data); while (n) { buffer = rb_entry(n, struct binder_buffer, rb_node); @@ -858,7 +858,7 @@ static void binder_free_buf(struct binder_proc *proc, } static struct binder_node *binder_get_node(struct binder_proc *proc, - binder_uintptr_t ptr) + void __user *ptr) { struct rb_node *n = proc->nodes.rb_node; struct binder_node *node; @@ -877,8 +877,8 @@ static struct binder_node *binder_get_node(struct binder_proc *proc, } static struct binder_node *binder_new_node(struct binder_proc *proc, - binder_uintptr_t ptr, - binder_uintptr_t cookie) + void __user *ptr, + void __user *cookie) { struct rb_node **p = &proc->nodes.rb_node; struct rb_node *parent = NULL; @@ -910,9 +910,9 @@ static struct binder_node *binder_new_node(struct binder_proc *proc, INIT_LIST_HEAD(&node->work.entry); INIT_LIST_HEAD(&node->async_todo); binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%016llx c%016llx created\n", + "%d:%d node %d u%p c%p created\n", proc->pid, current->pid, node->debug_id, - (u64)node->ptr, (u64)node->cookie); + node->ptr, node->cookie); return node; } @@ -1228,9 +1228,9 @@ static void binder_send_failed_reply(struct binder_transaction *t, static void binder_transaction_buffer_release(struct binder_proc *proc, struct binder_buffer *buffer, - binder_size_t *failed_at) + size_t *failed_at) { - binder_size_t *offp, *off_end; + size_t *offp, *off_end; int debug_id = buffer->debug_id; binder_debug(BINDER_DEBUG_TRANSACTION, @@ -1241,8 +1241,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, if (buffer->target_node) binder_dec_node(buffer->target_node, 1, 0); - offp = (binder_size_t *)(buffer->data + - ALIGN(buffer->data_size, sizeof(void *))); + offp = (size_t *)(buffer->data + ALIGN(buffer->data_size, sizeof(void *))); if (failed_at) off_end = failed_at; else @@ -1252,8 +1251,8 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, if (*offp > buffer->data_size - sizeof(*fp) || buffer->data_size < sizeof(*fp) || !IS_ALIGNED(*offp, sizeof(u32))) { - pr_err("transaction release %d bad offset %lld, size %zd\n", - debug_id, (u64)*offp, buffer->data_size); + pr_err("transaction release %d bad offset %zd, size %zd\n", + debug_id, *offp, buffer->data_size); continue; } fp = (struct flat_binder_object *)(buffer->data + *offp); @@ -1262,13 +1261,13 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, case BINDER_TYPE_WEAK_BINDER: { struct binder_node *node = binder_get_node(proc, fp->binder); if (node == NULL) { - pr_err("transaction release %d bad node %016llx\n", - debug_id, (u64)fp->binder); + pr_err("transaction release %d bad node %p\n", + debug_id, fp->binder); break; } binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%016llx\n", - node->debug_id, (u64)node->ptr); + " node %d u%p\n", + node->debug_id, node->ptr); binder_dec_node(node, fp->type == BINDER_TYPE_BINDER, 0); } break; case BINDER_TYPE_HANDLE: @@ -1306,7 +1305,7 @@ static void binder_transaction(struct binder_proc *proc, { struct binder_transaction *t; struct binder_work *tcomplete; - binder_size_t *offp, *off_end; + size_t *offp, *off_end; struct binder_proc *target_proc; struct binder_thread *target_thread = NULL; struct binder_node *target_node = NULL; @@ -1439,20 +1438,18 @@ static void binder_transaction(struct binder_proc *proc, if (reply) binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d BC_REPLY %d -> %d:%d, data %016llx-%016llx size %lld-%lld\n", + "%d:%d BC_REPLY %d -> %d:%d, data %p-%p size %zd-%zd\n", proc->pid, thread->pid, t->debug_id, target_proc->pid, target_thread->pid, - (u64)tr->data.ptr.buffer, - (u64)tr->data.ptr.offsets, - (u64)tr->data_size, (u64)tr->offsets_size); + tr->data.ptr.buffer, tr->data.ptr.offsets, + tr->data_size, tr->offsets_size); else binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d BC_TRANSACTION %d -> %d - node %d, data %016llx-%016llx size %lld-%lld\n", + "%d:%d BC_TRANSACTION %d -> %d - node %d, data %p-%p size %zd-%zd\n", proc->pid, thread->pid, t->debug_id, target_proc->pid, target_node->debug_id, - (u64)tr->data.ptr.buffer, - (u64)tr->data.ptr.offsets, - (u64)tr->data_size, (u64)tr->offsets_size); + tr->data.ptr.buffer, tr->data.ptr.offsets, + tr->data_size, tr->offsets_size); if (!reply && !(tr->flags & TF_ONE_WAY)) t->from = thread; @@ -1481,26 +1478,23 @@ static void binder_transaction(struct binder_proc *proc, if (target_node) binder_inc_node(target_node, 1, 0, NULL); - offp = (binder_size_t *)(t->buffer->data + - ALIGN(tr->data_size, sizeof(void *))); + offp = (size_t *)(t->buffer->data + ALIGN(tr->data_size, sizeof(void *))); - if (copy_from_user(t->buffer->data, (const void __user *)(uintptr_t) - tr->data.ptr.buffer, tr->data_size)) { + if (copy_from_user(t->buffer->data, tr->data.ptr.buffer, tr->data_size)) { binder_user_error("%d:%d got transaction with invalid data ptr\n", proc->pid, thread->pid); return_error = BR_FAILED_REPLY; goto err_copy_data_failed; } - if (copy_from_user(offp, (const void __user *)(uintptr_t) - tr->data.ptr.offsets, tr->offsets_size)) { + if (copy_from_user(offp, tr->data.ptr.offsets, tr->offsets_size)) { binder_user_error("%d:%d got transaction with invalid offsets ptr\n", proc->pid, thread->pid); return_error = BR_FAILED_REPLY; goto err_copy_data_failed; } - if (!IS_ALIGNED(tr->offsets_size, sizeof(binder_size_t))) { - binder_user_error("%d:%d got transaction with invalid offsets size, %lld\n", - proc->pid, thread->pid, (u64)tr->offsets_size); + if (!IS_ALIGNED(tr->offsets_size, sizeof(size_t))) { + binder_user_error("%d:%d got transaction with invalid offsets size, %zd\n", + proc->pid, thread->pid, tr->offsets_size); return_error = BR_FAILED_REPLY; goto err_bad_offset; } @@ -1510,8 +1504,8 @@ static void binder_transaction(struct binder_proc *proc, if (*offp > t->buffer->data_size - sizeof(*fp) || t->buffer->data_size < sizeof(*fp) || !IS_ALIGNED(*offp, sizeof(u32))) { - binder_user_error("%d:%d got transaction with invalid offset, %lld\n", - proc->pid, thread->pid, (u64)*offp); + binder_user_error("%d:%d got transaction with invalid offset, %zd\n", + proc->pid, thread->pid, *offp); return_error = BR_FAILED_REPLY; goto err_bad_offset; } @@ -1531,10 +1525,10 @@ static void binder_transaction(struct binder_proc *proc, node->accept_fds = !!(fp->flags & FLAT_BINDER_FLAG_ACCEPTS_FDS); } if (fp->cookie != node->cookie) { - binder_user_error("%d:%d sending u%016llx node %d, cookie mismatch %016llx != %016llx\n", + binder_user_error("%d:%d sending u%p node %d, cookie mismatch %p != %p\n", proc->pid, thread->pid, - (u64)fp->binder, node->debug_id, - (u64)fp->cookie, (u64)node->cookie); + fp->binder, node->debug_id, + fp->cookie, node->cookie); goto err_binder_get_ref_for_node_failed; } if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { @@ -1556,9 +1550,9 @@ static void binder_transaction(struct binder_proc *proc, trace_binder_transaction_node_to_ref(t, node, ref); binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%016llx -> ref %d desc %d\n", - node->debug_id, (u64)node->ptr, - ref->debug_id, ref->desc); + " node %d u%p -> ref %d desc %d\n", + node->debug_id, node->ptr, ref->debug_id, + ref->desc); } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { @@ -1584,9 +1578,9 @@ static void binder_transaction(struct binder_proc *proc, binder_inc_node(ref->node, fp->type == BINDER_TYPE_BINDER, 0, NULL); trace_binder_transaction_ref_to_node(t, ref); binder_debug(BINDER_DEBUG_TRANSACTION, - " ref %d desc %d -> node %d u%016llx\n", + " ref %d desc %d -> node %d u%p\n", ref->debug_id, ref->desc, ref->node->debug_id, - (u64)ref->node->ptr); + ref->node->ptr); } else { struct binder_ref *new_ref; new_ref = binder_get_ref_for_node(target_proc, ref->node); @@ -1707,9 +1701,9 @@ err_dead_binder: err_invalid_target_handle: err_no_context_mgr_node: binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, - "%d:%d transaction failed %d, size %lld-%lld\n", + "%d:%d transaction failed %d, size %zd-%zd\n", proc->pid, thread->pid, return_error, - (u64)tr->data_size, (u64)tr->offsets_size); + tr->data_size, tr->offsets_size); { struct binder_transaction_log_entry *fe; @@ -1726,11 +1720,9 @@ err_no_context_mgr_node: } int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, - binder_uintptr_t binder_buffer, size_t size, - binder_size_t *consumed) + void __user *buffer, size_t size, size_t *consumed) { uint32_t cmd; - void __user *buffer = (void __user *)(uintptr_t)binder_buffer; void __user *ptr = buffer + *consumed; void __user *end = buffer + size; @@ -1799,33 +1791,33 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, } case BC_INCREFS_DONE: case BC_ACQUIRE_DONE: { - binder_uintptr_t node_ptr; - binder_uintptr_t cookie; + void __user *node_ptr; + void *cookie; struct binder_node *node; - if (get_user(node_ptr, (binder_uintptr_t __user *)ptr)) + if (get_user(node_ptr, (void * __user *)ptr)) return -EFAULT; - ptr += sizeof(binder_uintptr_t); - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + ptr += sizeof(void *); + if (get_user(cookie, (void * __user *)ptr)) return -EFAULT; - ptr += sizeof(binder_uintptr_t); + ptr += sizeof(void *); node = binder_get_node(proc, node_ptr); if (node == NULL) { - binder_user_error("%d:%d %s u%016llx no match\n", + binder_user_error("%d:%d %s u%p no match\n", proc->pid, thread->pid, cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", - (u64)node_ptr); + node_ptr); break; } if (cookie != node->cookie) { - binder_user_error("%d:%d %s u%016llx node %d cookie mismatch %016llx != %016llx\n", + binder_user_error("%d:%d %s u%p node %d cookie mismatch %p != %p\n", proc->pid, thread->pid, cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", - (u64)node_ptr, node->debug_id, - (u64)cookie, (u64)node->cookie); + node_ptr, node->debug_id, + cookie, node->cookie); break; } if (cmd == BC_ACQUIRE_DONE) { @@ -1861,27 +1853,27 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, return -EINVAL; case BC_FREE_BUFFER: { - binder_uintptr_t data_ptr; + void __user *data_ptr; struct binder_buffer *buffer; - if (get_user(data_ptr, (binder_uintptr_t __user *)ptr)) + if (get_user(data_ptr, (void * __user *)ptr)) return -EFAULT; - ptr += sizeof(binder_uintptr_t); + ptr += sizeof(void *); buffer = binder_buffer_lookup(proc, data_ptr); if (buffer == NULL) { - binder_user_error("%d:%d BC_FREE_BUFFER u%016llx no match\n", - proc->pid, thread->pid, (u64)data_ptr); + binder_user_error("%d:%d BC_FREE_BUFFER u%p no match\n", + proc->pid, thread->pid, data_ptr); break; } if (!buffer->allow_user_free) { - binder_user_error("%d:%d BC_FREE_BUFFER u%016llx matched unreturned buffer\n", - proc->pid, thread->pid, (u64)data_ptr); + binder_user_error("%d:%d BC_FREE_BUFFER u%p matched unreturned buffer\n", + proc->pid, thread->pid, data_ptr); break; } binder_debug(BINDER_DEBUG_FREE_BUFFER, - "%d:%d BC_FREE_BUFFER u%016llx found buffer %d for %s transaction\n", - proc->pid, thread->pid, (u64)data_ptr, buffer->debug_id, + "%d:%d BC_FREE_BUFFER u%p found buffer %d for %s transaction\n", + proc->pid, thread->pid, data_ptr, buffer->debug_id, buffer->transaction ? "active" : "finished"); if (buffer->transaction) { @@ -1951,16 +1943,16 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, case BC_REQUEST_DEATH_NOTIFICATION: case BC_CLEAR_DEATH_NOTIFICATION: { uint32_t target; - binder_uintptr_t cookie; + void __user *cookie; struct binder_ref *ref; struct binder_ref_death *death; if (get_user(target, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + if (get_user(cookie, (void __user * __user *)ptr)) return -EFAULT; - ptr += sizeof(binder_uintptr_t); + ptr += sizeof(void *); ref = binder_get_ref(proc, target); if (ref == NULL) { binder_user_error("%d:%d %s invalid ref %d\n", @@ -1973,12 +1965,12 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, } binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, - "%d:%d %s %016llx ref %d desc %d s %d w %d for node %d\n", + "%d:%d %s %p ref %d desc %d s %d w %d for node %d\n", proc->pid, thread->pid, cmd == BC_REQUEST_DEATH_NOTIFICATION ? "BC_REQUEST_DEATH_NOTIFICATION" : "BC_CLEAR_DEATH_NOTIFICATION", - (u64)cookie, ref->debug_id, ref->desc, + cookie, ref->debug_id, ref->desc, ref->strong, ref->weak, ref->node->debug_id); if (cmd == BC_REQUEST_DEATH_NOTIFICATION) { @@ -2016,9 +2008,9 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, } death = ref->death; if (death->cookie != cookie) { - binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch %016llx != %016llx\n", + binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch %p != %p\n", proc->pid, thread->pid, - (u64)death->cookie, (u64)cookie); + death->cookie, cookie); break; } ref->death = NULL; @@ -2038,9 +2030,9 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, } break; case BC_DEAD_BINDER_DONE: { struct binder_work *w; - binder_uintptr_t cookie; + void __user *cookie; struct binder_ref_death *death = NULL; - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + if (get_user(cookie, (void __user * __user *)ptr)) return -EFAULT; ptr += sizeof(void *); @@ -2052,11 +2044,11 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, } } binder_debug(BINDER_DEBUG_DEAD_BINDER, - "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n", - proc->pid, thread->pid, (u64)cookie, death); + "%d:%d BC_DEAD_BINDER_DONE %p found %p\n", + proc->pid, thread->pid, cookie, death); if (death == NULL) { - binder_user_error("%d:%d BC_DEAD_BINDER_DONE %016llx not found\n", - proc->pid, thread->pid, (u64)cookie); + binder_user_error("%d:%d BC_DEAD_BINDER_DONE %p not found\n", + proc->pid, thread->pid, cookie); break; } @@ -2108,10 +2100,9 @@ static int binder_has_thread_work(struct binder_thread *thread) static int binder_thread_read(struct binder_proc *proc, struct binder_thread *thread, - binder_uintptr_t binder_buffer, size_t size, - binder_size_t *consumed, int non_block) + void __user *buffer, size_t size, + size_t *consumed, int non_block) { - void __user *buffer = (void __user *)(uintptr_t)binder_buffer; void __user *ptr = buffer + *consumed; void __user *end = buffer + size; @@ -2256,36 +2247,32 @@ retry: if (put_user(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (put_user(node->ptr, - (binder_uintptr_t __user *)ptr)) + if (put_user(node->ptr, (void * __user *)ptr)) return -EFAULT; - ptr += sizeof(binder_uintptr_t); - if (put_user(node->cookie, - (binder_uintptr_t __user *)ptr)) + ptr += sizeof(void *); + if (put_user(node->cookie, (void * __user *)ptr)) return -EFAULT; - ptr += sizeof(binder_uintptr_t); + ptr += sizeof(void *); binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_USER_REFS, - "%d:%d %s %d u%016llx c%016llx\n", - proc->pid, thread->pid, cmd_name, - node->debug_id, - (u64)node->ptr, (u64)node->cookie); + "%d:%d %s %d u%p c%p\n", + proc->pid, thread->pid, cmd_name, node->debug_id, node->ptr, node->cookie); } else { list_del_init(&w->entry); if (!weak && !strong) { binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%016llx c%016llx deleted\n", + "%d:%d node %d u%p c%p deleted\n", proc->pid, thread->pid, node->debug_id, - (u64)node->ptr, (u64)node->cookie); + node->ptr, node->cookie); rb_erase(&node->rb_node, &proc->nodes); kfree(node); binder_stats_deleted(BINDER_STAT_NODE); } else { binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%016llx c%016llx state unchanged\n", - proc->pid, thread->pid, node->debug_id, - (u64)node->ptr, (u64)node->cookie); + "%d:%d node %d u%p c%p state unchanged\n", + proc->pid, thread->pid, node->debug_id, node->ptr, + node->cookie); } } } break; @@ -2303,18 +2290,17 @@ retry: if (put_user(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (put_user(death->cookie, - (binder_uintptr_t __user *)ptr)) + if (put_user(death->cookie, (void * __user *)ptr)) return -EFAULT; - ptr += sizeof(binder_uintptr_t); + ptr += sizeof(void *); binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, - "%d:%d %s %016llx\n", + "%d:%d %s %p\n", proc->pid, thread->pid, cmd == BR_DEAD_BINDER ? "BR_DEAD_BINDER" : "BR_CLEAR_DEATH_NOTIFICATION_DONE", - (u64)death->cookie); + death->cookie); if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION) { list_del(&w->entry); @@ -2344,8 +2330,8 @@ retry: binder_set_nice(target_node->min_priority); cmd = BR_TRANSACTION; } else { - tr.target.ptr = 0; - tr.cookie = 0; + tr.target.ptr = NULL; + tr.cookie = NULL; cmd = BR_REPLY; } tr.code = t->code; @@ -2362,9 +2348,8 @@ retry: tr.data_size = t->buffer->data_size; tr.offsets_size = t->buffer->offsets_size; - tr.data.ptr.buffer = (binder_uintptr_t)( - (uintptr_t)t->buffer->data + - proc->user_buffer_offset); + tr.data.ptr.buffer = (void *)t->buffer->data + + proc->user_buffer_offset; tr.data.ptr.offsets = tr.data.ptr.buffer + ALIGN(t->buffer->data_size, sizeof(void *)); @@ -2379,14 +2364,14 @@ retry: trace_binder_transaction_received(t); binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d %s %d %d:%d, cmd %d size %zd-%zd ptr %016llx-%016llx\n", + "%d:%d %s %d %d:%d, cmd %d size %zd-%zd ptr %p-%p\n", proc->pid, thread->pid, (cmd == BR_TRANSACTION) ? "BR_TRANSACTION" : "BR_REPLY", t->debug_id, t->from ? t->from->proc->pid : 0, t->from ? t->from->pid : 0, cmd, t->buffer->data_size, t->buffer->offsets_size, - (u64)tr.data.ptr.buffer, (u64)tr.data.ptr.offsets); + tr.data.ptr.buffer, tr.data.ptr.offsets); list_del(&t->work.entry); t->buffer->allow_user_free = 1; @@ -2456,8 +2441,8 @@ static void binder_release_work(struct list_head *list) death = container_of(w, struct binder_ref_death, work); binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, - "undelivered death notification, %016llx\n", - (u64)death->cookie); + "undelivered death notification, %p\n", + death->cookie); kfree(death); binder_stats_deleted(BINDER_STAT_DEATH); } break; @@ -2613,13 +2598,12 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) goto err; } binder_debug(BINDER_DEBUG_READ_WRITE, - "%d:%d write %lld at %016llx, read %lld at %016llx\n", - proc->pid, thread->pid, - (u64)bwr.write_size, (u64)bwr.write_buffer, - (u64)bwr.read_size, (u64)bwr.read_buffer); + "%d:%d write %zd at %016lx, read %zd at %016lx\n", + proc->pid, thread->pid, bwr.write_size, + bwr.write_buffer, bwr.read_size, bwr.read_buffer); if (bwr.write_size > 0) { - ret = binder_thread_write(proc, thread, bwr.write_buffer, bwr.write_size, &bwr.write_consumed); + ret = binder_thread_write(proc, thread, (void __user *)bwr.write_buffer, bwr.write_size, &bwr.write_consumed); trace_binder_write_done(ret); if (ret < 0) { bwr.read_consumed = 0; @@ -2629,7 +2613,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } } if (bwr.read_size > 0) { - ret = binder_thread_read(proc, thread, bwr.read_buffer, bwr.read_size, &bwr.read_consumed, filp->f_flags & O_NONBLOCK); + ret = binder_thread_read(proc, thread, (void __user *)bwr.read_buffer, bwr.read_size, &bwr.read_consumed, filp->f_flags & O_NONBLOCK); trace_binder_read_done(ret); if (!list_empty(&proc->todo)) wake_up_interruptible(&proc->wait); @@ -2640,10 +2624,9 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } } binder_debug(BINDER_DEBUG_READ_WRITE, - "%d:%d wrote %lld of %lld, read return %lld of %lld\n", - proc->pid, thread->pid, - (u64)bwr.write_consumed, (u64)bwr.write_size, - (u64)bwr.read_consumed, (u64)bwr.read_size); + "%d:%d wrote %zd of %zd, read return %zd of %zd\n", + proc->pid, thread->pid, bwr.write_consumed, bwr.write_size, + bwr.read_consumed, bwr.read_size); if (copy_to_user(ubuf, &bwr, sizeof(bwr))) { ret = -EFAULT; goto err; @@ -2675,7 +2658,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } } else binder_context_mgr_uid = current->cred->euid; - binder_context_mgr_node = binder_new_node(proc, 0, 0); + binder_context_mgr_node = binder_new_node(proc, NULL, NULL); if (binder_context_mgr_node == NULL) { ret = -ENOMEM; goto err; @@ -3170,9 +3153,8 @@ static void print_binder_work(struct seq_file *m, const char *prefix, break; case BINDER_WORK_NODE: node = container_of(w, struct binder_node, work); - seq_printf(m, "%snode work %d: u%016llx c%016llx\n", - prefix, node->debug_id, - (u64)node->ptr, (u64)node->cookie); + seq_printf(m, "%snode work %d: u%p c%p\n", + prefix, node->debug_id, node->ptr, node->cookie); break; case BINDER_WORK_DEAD_BINDER: seq_printf(m, "%shas dead binder\n", prefix); @@ -3232,8 +3214,8 @@ static void print_binder_node(struct seq_file *m, struct binder_node *node) hlist_for_each_entry(ref, &node->refs, node_entry) count++; - seq_printf(m, " node %d: u%016llx c%016llx hs %d hw %d ls %d lw %d is %d iw %d", - node->debug_id, (u64)node->ptr, (u64)node->cookie, + seq_printf(m, " node %d: u%p c%p hs %d hw %d ls %d lw %d is %d iw %d", + node->debug_id, node->ptr, node->cookie, node->has_strong_ref, node->has_weak_ref, node->local_strong_refs, node->local_weak_refs, node->internal_strong_refs, count); @@ -3535,7 +3517,6 @@ static const struct file_operations binder_fops = { .owner = THIS_MODULE, .poll = binder_poll, .unlocked_ioctl = binder_ioctl, - .compat_ioctl = binder_ioctl, .mmap = binder_mmap, .open = binder_open, .flush = binder_flush, diff --git a/drivers/staging/android/binder.h b/drivers/staging/android/binder.h index eb0834656dfe..d4101a671718 100644 --- a/drivers/staging/android/binder.h +++ b/drivers/staging/android/binder.h @@ -20,10 +20,6 @@ #ifndef _LINUX_BINDER_H #define _LINUX_BINDER_H -#ifdef CONFIG_ANDROID_BINDER_IPC_32BIT -#define BINDER_IPC_32BIT 1 -#endif - #include "uapi/binder.h" #endif /* _LINUX_BINDER_H */ diff --git a/drivers/staging/android/binder_trace.h b/drivers/staging/android/binder_trace.h index 7f20f3dc8369..82a567c2af67 100644 --- a/drivers/staging/android/binder_trace.h +++ b/drivers/staging/android/binder_trace.h @@ -152,7 +152,7 @@ TRACE_EVENT(binder_transaction_node_to_ref, TP_STRUCT__entry( __field(int, debug_id) __field(int, node_debug_id) - __field(binder_uintptr_t, node_ptr) + __field(void __user *, node_ptr) __field(int, ref_debug_id) __field(uint32_t, ref_desc) ), @@ -163,9 +163,8 @@ TRACE_EVENT(binder_transaction_node_to_ref, __entry->ref_debug_id = ref->debug_id; __entry->ref_desc = ref->desc; ), - TP_printk("transaction=%d node=%d src_ptr=0x%016llx ==> dest_ref=%d dest_desc=%d", - __entry->debug_id, __entry->node_debug_id, - (u64)__entry->node_ptr, + TP_printk("transaction=%d node=%d src_ptr=0x%p ==> dest_ref=%d dest_desc=%d", + __entry->debug_id, __entry->node_debug_id, __entry->node_ptr, __entry->ref_debug_id, __entry->ref_desc) ); @@ -178,7 +177,7 @@ TRACE_EVENT(binder_transaction_ref_to_node, __field(int, ref_debug_id) __field(uint32_t, ref_desc) __field(int, node_debug_id) - __field(binder_uintptr_t, node_ptr) + __field(void __user *, node_ptr) ), TP_fast_assign( __entry->debug_id = t->debug_id; @@ -187,10 +186,9 @@ TRACE_EVENT(binder_transaction_ref_to_node, __entry->node_debug_id = ref->node->debug_id; __entry->node_ptr = ref->node->ptr; ), - TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%016llx", + TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%p", __entry->debug_id, __entry->node_debug_id, - __entry->ref_debug_id, __entry->ref_desc, - (u64)__entry->node_ptr) + __entry->ref_debug_id, __entry->ref_desc, __entry->node_ptr) ); TRACE_EVENT(binder_transaction_ref_to_ref, diff --git a/drivers/staging/android/uapi/binder.h b/drivers/staging/android/uapi/binder.h index 4098c502fc36..008722a249bc 100644 --- a/drivers/staging/android/uapi/binder.h +++ b/drivers/staging/android/uapi/binder.h @@ -39,14 +39,6 @@ enum { FLAT_BINDER_FLAG_ACCEPTS_FDS = 0x100, }; -#ifdef BINDER_IPC_32BIT -typedef __u32 binder_size_t; -typedef __u32 binder_uintptr_t; -#else -typedef __u64 binder_size_t; -typedef __u64 binder_uintptr_t; -#endif - /* * This is the flattened representation of a Binder object for transfer * between processes. The 'offsets' supplied as part of a binder transaction @@ -61,12 +53,12 @@ struct flat_binder_object { /* 8 bytes of data. */ union { - binder_uintptr_t binder; /* local object */ - __u32 handle; /* remote object */ + void __user *binder; /* local object */ + __u32 handle; /* remote object */ }; /* extra data associated with local object */ - binder_uintptr_t cookie; + void __user *cookie; }; /* @@ -75,12 +67,12 @@ struct flat_binder_object { */ struct binder_write_read { - binder_size_t write_size; /* bytes to write */ - binder_size_t write_consumed; /* bytes consumed by driver */ - binder_uintptr_t write_buffer; - binder_size_t read_size; /* bytes to read */ - binder_size_t read_consumed; /* bytes consumed by driver */ - binder_uintptr_t read_buffer; + size_t write_size; /* bytes to write */ + size_t write_consumed; /* bytes consumed by driver */ + unsigned long write_buffer; + size_t read_size; /* bytes to read */ + size_t read_consumed; /* bytes consumed by driver */ + unsigned long read_buffer; }; /* Use with BINDER_VERSION, driver fills in fields. */ @@ -90,11 +82,7 @@ struct binder_version { }; /* This is the current protocol version. */ -#ifdef BINDER_IPC_32BIT #define BINDER_CURRENT_PROTOCOL_VERSION 7 -#else -#define BINDER_CURRENT_PROTOCOL_VERSION 8 -#endif #define BINDER_WRITE_READ _IOWR('b', 1, struct binder_write_read) #define BINDER_SET_IDLE_TIMEOUT _IOW('b', 3, __s64) @@ -132,17 +120,17 @@ struct binder_transaction_data { */ union { __u32 handle; /* target descriptor of command transaction */ - binder_uintptr_t ptr; /* target descriptor of return transaction */ + void *ptr; /* target descriptor of return transaction */ } target; - binder_uintptr_t cookie; /* target object cookie */ + void *cookie; /* target object cookie */ __u32 code; /* transaction command */ /* General information about the transaction. */ __u32 flags; pid_t sender_pid; uid_t sender_euid; - binder_size_t data_size; /* number of bytes of data */ - binder_size_t offsets_size; /* number of bytes of offsets */ + size_t data_size; /* number of bytes of data */ + size_t offsets_size; /* number of bytes of offsets */ /* If this transaction is inline, the data immediately * follows here; otherwise, it ends with a pointer to @@ -151,22 +139,22 @@ struct binder_transaction_data { union { struct { /* transaction data */ - binder_uintptr_t buffer; + const void __user *buffer; /* offsets from buffer to flat_binder_object structs */ - binder_uintptr_t offsets; + const void __user *offsets; } ptr; __u8 buf[8]; } data; }; struct binder_ptr_cookie { - binder_uintptr_t ptr; - binder_uintptr_t cookie; + void *ptr; + void *cookie; }; struct binder_handle_cookie { __u32 handle; - binder_uintptr_t cookie; + void *cookie; } __attribute__((packed)); struct binder_pri_desc { @@ -176,8 +164,8 @@ struct binder_pri_desc { struct binder_pri_ptr_cookie { __s32 priority; - binder_uintptr_t ptr; - binder_uintptr_t cookie; + void *ptr; + void *cookie; }; enum binder_driver_return_protocol { @@ -252,11 +240,11 @@ enum binder_driver_return_protocol { * stop threadpool thread */ - BR_DEAD_BINDER = _IOR('r', 15, binder_uintptr_t), + BR_DEAD_BINDER = _IOR('r', 15, void *), /* * void *: cookie */ - BR_CLEAR_DEATH_NOTIFICATION_DONE = _IOR('r', 16, binder_uintptr_t), + BR_CLEAR_DEATH_NOTIFICATION_DONE = _IOR('r', 16, void *), /* * void *: cookie */ @@ -282,7 +270,7 @@ enum binder_driver_command_protocol { * Else you have acquired a primary reference on the object. */ - BC_FREE_BUFFER = _IOW('c', 3, binder_uintptr_t), + BC_FREE_BUFFER = _IOW('c', 3, void *), /* * void *: ptr to transaction data received on a read */ @@ -337,7 +325,7 @@ enum binder_driver_command_protocol { * void *: cookie */ - BC_DEAD_BINDER_DONE = _IOW('c', 16, binder_uintptr_t), + BC_DEAD_BINDER_DONE = _IOW('c', 16, void *), /* * void *: cookie */ -- cgit v1.2.3 From d6c742effc964c65007167f42c06a7c2504b22a2 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Mon, 12 Jan 2015 13:47:28 -0800 Subject: Revert "staging: binder: Change binder mutex to rtmutex." This reverts commit 5d03bd0fdd66d3472386864cf32b4b82dda8d1a5. --- drivers/staging/android/binder.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 6e4497026b26..f3be98ef54ec 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -43,7 +42,7 @@ #include "binder.h" #include "binder_trace.h" -static DEFINE_RT_MUTEX(binder_main_lock); +static DEFINE_MUTEX(binder_main_lock); static DEFINE_MUTEX(binder_deferred_lock); static DEFINE_MUTEX(binder_mmap_lock); @@ -421,14 +420,14 @@ static long task_close_fd(struct binder_proc *proc, unsigned int fd) static inline void binder_lock(const char *tag) { trace_binder_lock(tag); - rt_mutex_lock(&binder_main_lock); + mutex_lock(&binder_main_lock); trace_binder_locked(tag); } static inline void binder_unlock(const char *tag) { trace_binder_unlock(tag); - rt_mutex_unlock(&binder_main_lock); + mutex_unlock(&binder_main_lock); } static void binder_set_nice(long nice) -- cgit v1.2.3 From 7b23ec738002fa7b16ce7742af8086ee31db5a54 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Mon, 12 Jan 2015 13:52:20 -0800 Subject: Revert "Add security hooks to binder and implement the hooks for SELinux." This reverts commit 6e6d8f546c36b161067efa5e0518f56be0200e77. Change-Id: I8f0dba7c90f2c2d285d14696277e1ec7d48978d3 --- drivers/staging/android/binder.c | 21 ------------ include/linux/security.h | 29 ---------------- security/capability.c | 24 -------------- security/security.c | 20 ----------- security/selinux/hooks.c | 66 ------------------------------------- security/selinux/include/classmap.h | 1 - 6 files changed, 161 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index f3be98ef54ec..69fd236345cb 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -37,7 +37,6 @@ #include #include #include -#include #include "binder.h" #include "binder_trace.h" @@ -1384,10 +1383,6 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_DEAD_REPLY; goto err_dead_binder; } - if (security_binder_transaction(proc->tsk, target_proc->tsk) < 0) { - return_error = BR_FAILED_REPLY; - goto err_invalid_target_handle; - } if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) { struct binder_transaction *tmp; tmp = thread->transaction_stack; @@ -1530,10 +1525,6 @@ static void binder_transaction(struct binder_proc *proc, fp->cookie, node->cookie); goto err_binder_get_ref_for_node_failed; } - if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_for_node_failed; - } ref = binder_get_ref_for_node(target_proc, node); if (ref == NULL) { return_error = BR_FAILED_REPLY; @@ -1563,10 +1554,6 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_failed; } - if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_failed; - } if (ref->node->proc == target_proc) { if (fp->type == BINDER_TYPE_HANDLE) fp->type = BINDER_TYPE_BINDER; @@ -1623,11 +1610,6 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_fget_failed; } - if (security_binder_transfer_file(proc->tsk, target_proc->tsk, file) < 0) { - fput(file); - return_error = BR_FAILED_REPLY; - goto err_get_unused_fd_failed; - } target_fd = task_get_unused_fd_flags(target_proc, O_CLOEXEC); if (target_fd < 0) { fput(file); @@ -2644,9 +2626,6 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ret = -EBUSY; goto err; } - ret = security_binder_set_context_mgr(proc->tsk); - if (ret < 0) - goto err; if (uid_valid(binder_context_mgr_uid)) { if (!uid_eq(binder_context_mgr_uid, current->cred->euid)) { pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", diff --git a/include/linux/security.h b/include/linux/security.h index 17e1888ff505..4686491852a7 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1402,11 +1402,6 @@ static inline void security_free_mnt_opts(struct security_mnt_opts *opts) struct security_operations { char name[SECURITY_NAME_MAX + 1]; - int (*binder_set_context_mgr) (struct task_struct *mgr); - int (*binder_transaction) (struct task_struct *from, struct task_struct *to); - int (*binder_transfer_binder) (struct task_struct *from, struct task_struct *to); - int (*binder_transfer_file) (struct task_struct *from, struct task_struct *to, struct file *file); - int (*ptrace_access_check) (struct task_struct *child, unsigned int mode); int (*ptrace_traceme) (struct task_struct *parent); int (*capget) (struct task_struct *target, @@ -1695,10 +1690,6 @@ extern void __init security_fixup_ops(struct security_operations *ops); /* Security operations */ -int security_binder_set_context_mgr(struct task_struct *mgr); -int security_binder_transaction(struct task_struct *from, struct task_struct *to); -int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to); -int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file); int security_ptrace_access_check(struct task_struct *child, unsigned int mode); int security_ptrace_traceme(struct task_struct *parent); int security_capget(struct task_struct *target, @@ -1878,26 +1869,6 @@ static inline int security_init(void) return 0; } -static inline int security_binder_set_context_mgr(struct task_struct *mgr) -{ - return 0; -} - -static inline int security_binder_transaction(struct task_struct *from, struct task_struct *to) -{ - return 0; -} - -static inline int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to) -{ - return 0; -} - -static inline int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) -{ - return 0; -} - static inline int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { diff --git a/security/capability.c b/security/capability.c index 6e4fc776badf..1728d4e375db 100644 --- a/security/capability.c +++ b/security/capability.c @@ -12,26 +12,6 @@ #include -static int cap_binder_set_context_mgr(struct task_struct *mgr) -{ - return 0; -} - -static int cap_binder_transaction(struct task_struct *from, struct task_struct *to) -{ - return 0; -} - -static int cap_binder_transfer_binder(struct task_struct *from, struct task_struct *to) -{ - return 0; -} - -static int cap_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) -{ - return 0; -} - static int cap_syslog(int type) { return 0; @@ -923,10 +903,6 @@ static void cap_audit_rule_free(void *lsmrule) void __init security_fixup_ops(struct security_operations *ops) { - set_to_cap_if_null(ops, binder_set_context_mgr); - set_to_cap_if_null(ops, binder_transaction); - set_to_cap_if_null(ops, binder_transfer_binder); - set_to_cap_if_null(ops, binder_transfer_file); set_to_cap_if_null(ops, ptrace_access_check); set_to_cap_if_null(ops, ptrace_traceme); set_to_cap_if_null(ops, capget); diff --git a/security/security.c b/security/security.c index d6ab2d2b363d..a3dce87d1aef 100644 --- a/security/security.c +++ b/security/security.c @@ -134,26 +134,6 @@ int __init register_security(struct security_operations *ops) /* Security operations */ -int security_binder_set_context_mgr(struct task_struct *mgr) -{ - return security_ops->binder_set_context_mgr(mgr); -} - -int security_binder_transaction(struct task_struct *from, struct task_struct *to) -{ - return security_ops->binder_transaction(from, to); -} - -int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to) -{ - return security_ops->binder_transfer_binder(from, to); -} - -int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) -{ - return security_ops->binder_transfer_file(from, to, file); -} - int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { #ifdef CONFIG_SECURITY_YAMA_STACKED diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index a8de30bd733a..8908c367425b 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1844,67 +1844,6 @@ static inline u32 open_file_to_av(struct file *file) /* Hook functions begin here. */ -static int selinux_binder_set_context_mgr(struct task_struct *mgr) -{ - u32 mysid = current_sid(); - u32 mgrsid = task_sid(mgr); - - return avc_has_perm(mysid, mgrsid, SECCLASS_BINDER, BINDER__SET_CONTEXT_MGR, NULL); -} - -static int selinux_binder_transaction(struct task_struct *from, struct task_struct *to) -{ - u32 mysid = current_sid(); - u32 fromsid = task_sid(from); - u32 tosid = task_sid(to); - int rc; - - if (mysid != fromsid) { - rc = avc_has_perm(mysid, fromsid, SECCLASS_BINDER, BINDER__IMPERSONATE, NULL); - if (rc) - return rc; - } - - return avc_has_perm(fromsid, tosid, SECCLASS_BINDER, BINDER__CALL, NULL); -} - -static int selinux_binder_transfer_binder(struct task_struct *from, struct task_struct *to) -{ - u32 fromsid = task_sid(from); - u32 tosid = task_sid(to); - return avc_has_perm(fromsid, tosid, SECCLASS_BINDER, BINDER__TRANSFER, NULL); -} - -static int selinux_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) -{ - u32 sid = task_sid(to); - struct file_security_struct *fsec = file->f_security; - struct inode *inode = file->f_path.dentry->d_inode; - struct inode_security_struct *isec = inode->i_security; - struct common_audit_data ad; - struct selinux_audit_data sad = {0,}; - int rc; - - ad.type = LSM_AUDIT_DATA_PATH; - ad.u.path = file->f_path; - ad.selinux_audit_data = &sad; - - if (sid != fsec->sid) { - rc = avc_has_perm(sid, fsec->sid, - SECCLASS_FD, - FD__USE, - &ad); - if (rc) - return rc; - } - - if (unlikely(IS_PRIVATE(inode))) - return 0; - - return avc_has_perm(sid, isec->sid, isec->sclass, file_to_av(file), - &ad); -} - static int selinux_ptrace_access_check(struct task_struct *child, unsigned int mode) { @@ -5617,11 +5556,6 @@ static int selinux_key_getsecurity(struct key *key, char **_buffer) static struct security_operations selinux_ops = { .name = "selinux", - .binder_set_context_mgr = selinux_binder_set_context_mgr, - .binder_transaction = selinux_binder_transaction, - .binder_transfer_binder = selinux_binder_transfer_binder, - .binder_transfer_file = selinux_binder_transfer_file, - .ptrace_access_check = selinux_ptrace_access_check, .ptrace_traceme = selinux_ptrace_traceme, .capget = selinux_capget, diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h index c32ff7bde81a..14d04e63b1f0 100644 --- a/security/selinux/include/classmap.h +++ b/security/selinux/include/classmap.h @@ -151,6 +151,5 @@ struct security_class_mapping secclass_map[] = { { "kernel_service", { "use_as_override", "create_files_as", NULL } }, { "tun_socket", { COMMON_SOCK_PERMS, "attach_queue", NULL } }, - { "binder", { "impersonate", "call", "set_context_mgr", "transfer", NULL } }, { NULL } }; -- cgit v1.2.3 From c963d0f41de94e59731e81b728cbb0a5dfb1e646 Mon Sep 17 00:00:00 2001 From: Bojan Prtvar Date: Mon, 2 Sep 2013 08:18:40 +0200 Subject: Staging: android: Mark local functions in binder.c as static This fixes the following sparse warnings drivers/staging/android/binder.c:1703:5: warning: symbol 'binder_thread_write' was not declared. Should it be static? drivers/staging/android/binder.c:2058:6: warning: symbol 'binder_stat_br' was not declared. Should it be static? Change-Id: Ib3fadafe30b5ffa3776270574809823e898caac3 Signed-off-by: Bojan Prtvar Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 69fd236345cb..d9f4712ad999 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1700,7 +1700,8 @@ err_no_context_mgr_node: thread->return_error = return_error; } -int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, +static int binder_thread_write(struct binder_proc *proc, + struct binder_thread *thread, void __user *buffer, size_t size, size_t *consumed) { uint32_t cmd; @@ -2055,8 +2056,8 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, return 0; } -void binder_stat_br(struct binder_proc *proc, struct binder_thread *thread, - uint32_t cmd) +static void binder_stat_br(struct binder_proc *proc, + struct binder_thread *thread, uint32_t cmd) { trace_binder_return(cmd); if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.br)) { -- cgit v1.2.3 From 268fc12dd625412dd5ece4a70a6003ca69062db3 Mon Sep 17 00:00:00 2001 From: Bojan Prtvar Date: Sun, 1 Sep 2013 20:30:38 +0200 Subject: Staging: android: add __user annotation in binder.c This fixes the following sparse error drivers/staging/android/binder.c:1795:36: error: incompatible types in comparison expression (different address spaces) Change-Id: I3824cb700d0de0e24c94771b1441e639d7d4d18b Signed-off-by: Bojan Prtvar Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index d9f4712ad999..1432d956769c 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1774,7 +1774,7 @@ static int binder_thread_write(struct binder_proc *proc, case BC_INCREFS_DONE: case BC_ACQUIRE_DONE: { void __user *node_ptr; - void *cookie; + void __user *cookie; struct binder_node *node; if (get_user(node_ptr, (void * __user *)ptr)) -- cgit v1.2.3 From dda6600546202c636b2be74a4640e929dae7cba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arve=20Hj=C3=B8nnev=C3=A5g?= Date: Fri, 21 Feb 2014 14:40:26 -0800 Subject: staging: binder: Support concurrent 32 bit and 64 bit processes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For 64bit systems we want to use the same binder interface for 32bit and 64bit processes. Thus the size and the layout of the structures passed between the kernel and the userspace has to be the same for both 32 and 64bit processes. This change replaces all the uses of void* and size_t with binder_uintptr_t and binder_size_t. These are then typedefed to specific sizes depending on the use of the interface, as follows: * __u32 - on legacy 32bit only userspace * __u64 - on mixed 32/64bit userspace where all processes use the same interface. This change also increments the BINDER_CURRENT_PROTOCOL_VERSION to 8 and hooks the compat_ioctl entry for the mixed 32/64bit Android userspace. This patch also provides a CONFIG_ANDROID_BINDER_IPC_32BIT option for compatability, which if set which enables the old protocol, setting BINDER_CURRENT_PROTOCOL_VERSION to 7, on 32 bit systems. Please note that all 64bit kernels will use the 64bit Binder ABI. Change-Id: If54f075787a6bb261012eb73295eb4f83a8c91c9 Cc: Colin Cross Cc: Arve Hjønnevåg Cc: Serban Constantinescu Cc: Android Kernel Team Signed-off-by: Arve Hjønnevåg [jstultz: Merged with upstream type changes. Various whitespace fixes and longer Kconfig description for checkpatch. Included improved commit message from Serban (with a few tweaks).] Signed-off-by: John Stultz Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/Kconfig | 12 ++ drivers/staging/android/binder.c | 268 ++++++++++++++++++--------------- drivers/staging/android/binder.h | 4 + drivers/staging/android/binder_trace.h | 14 +- drivers/staging/android/uapi/binder.h | 64 +++++--- 5 files changed, 213 insertions(+), 149 deletions(-) diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig index 1d6f5cf5826c..45e3380accbc 100644 --- a/drivers/staging/android/Kconfig +++ b/drivers/staging/android/Kconfig @@ -19,6 +19,18 @@ config ANDROID_BINDER_IPC Android process, using Binder to identify, invoke and pass arguments between said processes. +config ANDROID_BINDER_IPC_32BIT + bool "Use old 32-bit binder api" + depends on !64BIT + ---help--- + The Binder API has been changed to support both 32 and 64bit + applications in a mixed environment. + + Enable this to support an old 32-bit Android user-space (v4.4 and + earlier). + + Note that enabling this will break newer Android user-space. + config ASHMEM bool "Enable the Anonymous Shared Memory Subsystem" default n diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 1432d956769c..cfe4bc8f05cb 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -228,8 +228,8 @@ struct binder_node { int internal_strong_refs; int local_weak_refs; int local_strong_refs; - void __user *ptr; - void __user *cookie; + binder_uintptr_t ptr; + binder_uintptr_t cookie; unsigned has_strong_ref:1; unsigned pending_strong_ref:1; unsigned has_weak_ref:1; @@ -242,7 +242,7 @@ struct binder_node { struct binder_ref_death { struct binder_work work; - void __user *cookie; + binder_uintptr_t cookie; }; struct binder_ref { @@ -515,14 +515,14 @@ static void binder_insert_allocated_buffer(struct binder_proc *proc, } static struct binder_buffer *binder_buffer_lookup(struct binder_proc *proc, - void __user *user_ptr) + uintptr_t user_ptr) { struct rb_node *n = proc->allocated_buffers.rb_node; struct binder_buffer *buffer; struct binder_buffer *kern_ptr; - kern_ptr = user_ptr - proc->user_buffer_offset - - offsetof(struct binder_buffer, data); + kern_ptr = (struct binder_buffer *)(user_ptr - proc->user_buffer_offset + - offsetof(struct binder_buffer, data)); while (n) { buffer = rb_entry(n, struct binder_buffer, rb_node); @@ -856,7 +856,7 @@ static void binder_free_buf(struct binder_proc *proc, } static struct binder_node *binder_get_node(struct binder_proc *proc, - void __user *ptr) + binder_uintptr_t ptr) { struct rb_node *n = proc->nodes.rb_node; struct binder_node *node; @@ -875,8 +875,8 @@ static struct binder_node *binder_get_node(struct binder_proc *proc, } static struct binder_node *binder_new_node(struct binder_proc *proc, - void __user *ptr, - void __user *cookie) + binder_uintptr_t ptr, + binder_uintptr_t cookie) { struct rb_node **p = &proc->nodes.rb_node; struct rb_node *parent = NULL; @@ -908,9 +908,9 @@ static struct binder_node *binder_new_node(struct binder_proc *proc, INIT_LIST_HEAD(&node->work.entry); INIT_LIST_HEAD(&node->async_todo); binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%p c%p created\n", + "%d:%d node %d u%016llx c%016llx created\n", proc->pid, current->pid, node->debug_id, - node->ptr, node->cookie); + (u64)node->ptr, (u64)node->cookie); return node; } @@ -1226,9 +1226,9 @@ static void binder_send_failed_reply(struct binder_transaction *t, static void binder_transaction_buffer_release(struct binder_proc *proc, struct binder_buffer *buffer, - size_t *failed_at) + binder_size_t *failed_at) { - size_t *offp, *off_end; + binder_size_t *offp, *off_end; int debug_id = buffer->debug_id; binder_debug(BINDER_DEBUG_TRANSACTION, @@ -1239,7 +1239,8 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, if (buffer->target_node) binder_dec_node(buffer->target_node, 1, 0); - offp = (size_t *)(buffer->data + ALIGN(buffer->data_size, sizeof(void *))); + offp = (binder_size_t *)(buffer->data + + ALIGN(buffer->data_size, sizeof(void *))); if (failed_at) off_end = failed_at; else @@ -1249,8 +1250,8 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, if (*offp > buffer->data_size - sizeof(*fp) || buffer->data_size < sizeof(*fp) || !IS_ALIGNED(*offp, sizeof(u32))) { - pr_err("transaction release %d bad offset %zd, size %zd\n", - debug_id, *offp, buffer->data_size); + pr_err("transaction release %d bad offset %lld, size %zd\n", + debug_id, (u64)*offp, buffer->data_size); continue; } fp = (struct flat_binder_object *)(buffer->data + *offp); @@ -1259,13 +1260,13 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, case BINDER_TYPE_WEAK_BINDER: { struct binder_node *node = binder_get_node(proc, fp->binder); if (node == NULL) { - pr_err("transaction release %d bad node %p\n", - debug_id, fp->binder); + pr_err("transaction release %d bad node %016llx\n", + debug_id, (u64)fp->binder); break; } binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%p\n", - node->debug_id, node->ptr); + " node %d u%016llx\n", + node->debug_id, (u64)node->ptr); binder_dec_node(node, fp->type == BINDER_TYPE_BINDER, 0); } break; case BINDER_TYPE_HANDLE: @@ -1303,7 +1304,7 @@ static void binder_transaction(struct binder_proc *proc, { struct binder_transaction *t; struct binder_work *tcomplete; - size_t *offp, *off_end; + binder_size_t *offp, *off_end; struct binder_proc *target_proc; struct binder_thread *target_thread = NULL; struct binder_node *target_node = NULL; @@ -1432,18 +1433,20 @@ static void binder_transaction(struct binder_proc *proc, if (reply) binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d BC_REPLY %d -> %d:%d, data %p-%p size %zd-%zd\n", + "%d:%d BC_REPLY %d -> %d:%d, data %016llx-%016llx size %lld-%lld\n", proc->pid, thread->pid, t->debug_id, target_proc->pid, target_thread->pid, - tr->data.ptr.buffer, tr->data.ptr.offsets, - tr->data_size, tr->offsets_size); + (u64)tr->data.ptr.buffer, + (u64)tr->data.ptr.offsets, + (u64)tr->data_size, (u64)tr->offsets_size); else binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d BC_TRANSACTION %d -> %d - node %d, data %p-%p size %zd-%zd\n", + "%d:%d BC_TRANSACTION %d -> %d - node %d, data %016llx-%016llx size %lld-%lld\n", proc->pid, thread->pid, t->debug_id, target_proc->pid, target_node->debug_id, - tr->data.ptr.buffer, tr->data.ptr.offsets, - tr->data_size, tr->offsets_size); + (u64)tr->data.ptr.buffer, + (u64)tr->data.ptr.offsets, + (u64)tr->data_size, (u64)tr->offsets_size); if (!reply && !(tr->flags & TF_ONE_WAY)) t->from = thread; @@ -1472,23 +1475,26 @@ static void binder_transaction(struct binder_proc *proc, if (target_node) binder_inc_node(target_node, 1, 0, NULL); - offp = (size_t *)(t->buffer->data + ALIGN(tr->data_size, sizeof(void *))); + offp = (binder_size_t *)(t->buffer->data + + ALIGN(tr->data_size, sizeof(void *))); - if (copy_from_user(t->buffer->data, tr->data.ptr.buffer, tr->data_size)) { + if (copy_from_user(t->buffer->data, (const void __user *)(uintptr_t) + tr->data.ptr.buffer, tr->data_size)) { binder_user_error("%d:%d got transaction with invalid data ptr\n", proc->pid, thread->pid); return_error = BR_FAILED_REPLY; goto err_copy_data_failed; } - if (copy_from_user(offp, tr->data.ptr.offsets, tr->offsets_size)) { + if (copy_from_user(offp, (const void __user *)(uintptr_t) + tr->data.ptr.offsets, tr->offsets_size)) { binder_user_error("%d:%d got transaction with invalid offsets ptr\n", proc->pid, thread->pid); return_error = BR_FAILED_REPLY; goto err_copy_data_failed; } - if (!IS_ALIGNED(tr->offsets_size, sizeof(size_t))) { - binder_user_error("%d:%d got transaction with invalid offsets size, %zd\n", - proc->pid, thread->pid, tr->offsets_size); + if (!IS_ALIGNED(tr->offsets_size, sizeof(binder_size_t))) { + binder_user_error("%d:%d got transaction with invalid offsets size, %lld\n", + proc->pid, thread->pid, (u64)tr->offsets_size); return_error = BR_FAILED_REPLY; goto err_bad_offset; } @@ -1498,8 +1504,8 @@ static void binder_transaction(struct binder_proc *proc, if (*offp > t->buffer->data_size - sizeof(*fp) || t->buffer->data_size < sizeof(*fp) || !IS_ALIGNED(*offp, sizeof(u32))) { - binder_user_error("%d:%d got transaction with invalid offset, %zd\n", - proc->pid, thread->pid, *offp); + binder_user_error("%d:%d got transaction with invalid offset, %lld\n", + proc->pid, thread->pid, (u64)*offp); return_error = BR_FAILED_REPLY; goto err_bad_offset; } @@ -1519,10 +1525,10 @@ static void binder_transaction(struct binder_proc *proc, node->accept_fds = !!(fp->flags & FLAT_BINDER_FLAG_ACCEPTS_FDS); } if (fp->cookie != node->cookie) { - binder_user_error("%d:%d sending u%p node %d, cookie mismatch %p != %p\n", + binder_user_error("%d:%d sending u%016llx node %d, cookie mismatch %016llx != %016llx\n", proc->pid, thread->pid, - fp->binder, node->debug_id, - fp->cookie, node->cookie); + (u64)fp->binder, node->debug_id, + (u64)fp->cookie, (u64)node->cookie); goto err_binder_get_ref_for_node_failed; } ref = binder_get_ref_for_node(target_proc, node); @@ -1540,9 +1546,9 @@ static void binder_transaction(struct binder_proc *proc, trace_binder_transaction_node_to_ref(t, node, ref); binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%p -> ref %d desc %d\n", - node->debug_id, node->ptr, ref->debug_id, - ref->desc); + " node %d u%016llx -> ref %d desc %d\n", + node->debug_id, (u64)node->ptr, + ref->debug_id, ref->desc); } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { @@ -1564,9 +1570,9 @@ static void binder_transaction(struct binder_proc *proc, binder_inc_node(ref->node, fp->type == BINDER_TYPE_BINDER, 0, NULL); trace_binder_transaction_ref_to_node(t, ref); binder_debug(BINDER_DEBUG_TRANSACTION, - " ref %d desc %d -> node %d u%p\n", + " ref %d desc %d -> node %d u%016llx\n", ref->debug_id, ref->desc, ref->node->debug_id, - ref->node->ptr); + (u64)ref->node->ptr); } else { struct binder_ref *new_ref; new_ref = binder_get_ref_for_node(target_proc, ref->node); @@ -1682,9 +1688,9 @@ err_dead_binder: err_invalid_target_handle: err_no_context_mgr_node: binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, - "%d:%d transaction failed %d, size %zd-%zd\n", + "%d:%d transaction failed %d, size %lld-%lld\n", proc->pid, thread->pid, return_error, - tr->data_size, tr->offsets_size); + (u64)tr->data_size, (u64)tr->offsets_size); { struct binder_transaction_log_entry *fe; @@ -1702,9 +1708,11 @@ err_no_context_mgr_node: static int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, - void __user *buffer, size_t size, size_t *consumed) + binder_uintptr_t binder_buffer, size_t size, + binder_size_t *consumed) { uint32_t cmd; + void __user *buffer = (void __user *)(uintptr_t)binder_buffer; void __user *ptr = buffer + *consumed; void __user *end = buffer + size; @@ -1773,33 +1781,33 @@ static int binder_thread_write(struct binder_proc *proc, } case BC_INCREFS_DONE: case BC_ACQUIRE_DONE: { - void __user *node_ptr; - void __user *cookie; + binder_uintptr_t node_ptr; + binder_uintptr_t cookie; struct binder_node *node; - if (get_user(node_ptr, (void * __user *)ptr)) + if (get_user(node_ptr, (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); - if (get_user(cookie, (void * __user *)ptr)) + ptr += sizeof(binder_uintptr_t); + if (get_user(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); + ptr += sizeof(binder_uintptr_t); node = binder_get_node(proc, node_ptr); if (node == NULL) { - binder_user_error("%d:%d %s u%p no match\n", + binder_user_error("%d:%d %s u%016llx no match\n", proc->pid, thread->pid, cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", - node_ptr); + (u64)node_ptr); break; } if (cookie != node->cookie) { - binder_user_error("%d:%d %s u%p node %d cookie mismatch %p != %p\n", + binder_user_error("%d:%d %s u%016llx node %d cookie mismatch %016llx != %016llx\n", proc->pid, thread->pid, cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", - node_ptr, node->debug_id, - cookie, node->cookie); + (u64)node_ptr, node->debug_id, + (u64)cookie, (u64)node->cookie); break; } if (cmd == BC_ACQUIRE_DONE) { @@ -1835,27 +1843,28 @@ static int binder_thread_write(struct binder_proc *proc, return -EINVAL; case BC_FREE_BUFFER: { - void __user *data_ptr; + binder_uintptr_t data_ptr; struct binder_buffer *buffer; - if (get_user(data_ptr, (void * __user *)ptr)) + if (get_user(data_ptr, (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); + ptr += sizeof(binder_uintptr_t); buffer = binder_buffer_lookup(proc, data_ptr); if (buffer == NULL) { - binder_user_error("%d:%d BC_FREE_BUFFER u%p no match\n", - proc->pid, thread->pid, data_ptr); + binder_user_error("%d:%d BC_FREE_BUFFER u%016llx no match\n", + proc->pid, thread->pid, (u64)data_ptr); break; } if (!buffer->allow_user_free) { - binder_user_error("%d:%d BC_FREE_BUFFER u%p matched unreturned buffer\n", - proc->pid, thread->pid, data_ptr); + binder_user_error("%d:%d BC_FREE_BUFFER u%016llx matched unreturned buffer\n", + proc->pid, thread->pid, (u64)data_ptr); break; } binder_debug(BINDER_DEBUG_FREE_BUFFER, - "%d:%d BC_FREE_BUFFER u%p found buffer %d for %s transaction\n", - proc->pid, thread->pid, data_ptr, buffer->debug_id, + "%d:%d BC_FREE_BUFFER u%016llx found buffer %d for %s transaction\n", + proc->pid, thread->pid, (u64)data_ptr, + buffer->debug_id, buffer->transaction ? "active" : "finished"); if (buffer->transaction) { @@ -1925,16 +1934,16 @@ static int binder_thread_write(struct binder_proc *proc, case BC_REQUEST_DEATH_NOTIFICATION: case BC_CLEAR_DEATH_NOTIFICATION: { uint32_t target; - void __user *cookie; + binder_uintptr_t cookie; struct binder_ref *ref; struct binder_ref_death *death; if (get_user(target, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (get_user(cookie, (void __user * __user *)ptr)) + if (get_user(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); + ptr += sizeof(binder_uintptr_t); ref = binder_get_ref(proc, target); if (ref == NULL) { binder_user_error("%d:%d %s invalid ref %d\n", @@ -1947,12 +1956,12 @@ static int binder_thread_write(struct binder_proc *proc, } binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, - "%d:%d %s %p ref %d desc %d s %d w %d for node %d\n", + "%d:%d %s %016llx ref %d desc %d s %d w %d for node %d\n", proc->pid, thread->pid, cmd == BC_REQUEST_DEATH_NOTIFICATION ? "BC_REQUEST_DEATH_NOTIFICATION" : "BC_CLEAR_DEATH_NOTIFICATION", - cookie, ref->debug_id, ref->desc, + (u64)cookie, ref->debug_id, ref->desc, ref->strong, ref->weak, ref->node->debug_id); if (cmd == BC_REQUEST_DEATH_NOTIFICATION) { @@ -1990,9 +1999,10 @@ static int binder_thread_write(struct binder_proc *proc, } death = ref->death; if (death->cookie != cookie) { - binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch %p != %p\n", + binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch %016llx != %016llx\n", proc->pid, thread->pid, - death->cookie, cookie); + (u64)death->cookie, + (u64)cookie); break; } ref->death = NULL; @@ -2012,9 +2022,9 @@ static int binder_thread_write(struct binder_proc *proc, } break; case BC_DEAD_BINDER_DONE: { struct binder_work *w; - void __user *cookie; + binder_uintptr_t cookie; struct binder_ref_death *death = NULL; - if (get_user(cookie, (void __user * __user *)ptr)) + if (get_user(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(void *); @@ -2026,11 +2036,12 @@ static int binder_thread_write(struct binder_proc *proc, } } binder_debug(BINDER_DEBUG_DEAD_BINDER, - "%d:%d BC_DEAD_BINDER_DONE %p found %p\n", - proc->pid, thread->pid, cookie, death); + "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n", + proc->pid, thread->pid, (u64)cookie, + death); if (death == NULL) { - binder_user_error("%d:%d BC_DEAD_BINDER_DONE %p not found\n", - proc->pid, thread->pid, cookie); + binder_user_error("%d:%d BC_DEAD_BINDER_DONE %016llx not found\n", + proc->pid, thread->pid, (u64)cookie); break; } @@ -2082,9 +2093,10 @@ static int binder_has_thread_work(struct binder_thread *thread) static int binder_thread_read(struct binder_proc *proc, struct binder_thread *thread, - void __user *buffer, size_t size, - size_t *consumed, int non_block) + binder_uintptr_t binder_buffer, size_t size, + binder_size_t *consumed, int non_block) { + void __user *buffer = (void __user *)(uintptr_t)binder_buffer; void __user *ptr = buffer + *consumed; void __user *end = buffer + size; @@ -2229,32 +2241,40 @@ retry: if (put_user(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (put_user(node->ptr, (void * __user *)ptr)) + if (put_user(node->ptr, + (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); - if (put_user(node->cookie, (void * __user *)ptr)) + ptr += sizeof(binder_uintptr_t); + if (put_user(node->cookie, + (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); + ptr += sizeof(binder_uintptr_t); binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_USER_REFS, - "%d:%d %s %d u%p c%p\n", - proc->pid, thread->pid, cmd_name, node->debug_id, node->ptr, node->cookie); + "%d:%d %s %d u%016llx c%016llx\n", + proc->pid, thread->pid, cmd_name, + node->debug_id, + (u64)node->ptr, (u64)node->cookie); } else { list_del_init(&w->entry); if (!weak && !strong) { binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%p c%p deleted\n", - proc->pid, thread->pid, node->debug_id, - node->ptr, node->cookie); + "%d:%d node %d u%016llx c%016llx deleted\n", + proc->pid, thread->pid, + node->debug_id, + (u64)node->ptr, + (u64)node->cookie); rb_erase(&node->rb_node, &proc->nodes); kfree(node); binder_stats_deleted(BINDER_STAT_NODE); } else { binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%p c%p state unchanged\n", - proc->pid, thread->pid, node->debug_id, node->ptr, - node->cookie); + "%d:%d node %d u%016llx c%016llx state unchanged\n", + proc->pid, thread->pid, + node->debug_id, + (u64)node->ptr, + (u64)node->cookie); } } } break; @@ -2272,17 +2292,18 @@ retry: if (put_user(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); - if (put_user(death->cookie, (void * __user *)ptr)) + if (put_user(death->cookie, + (binder_uintptr_t __user *)ptr)) return -EFAULT; - ptr += sizeof(void *); + ptr += sizeof(binder_uintptr_t); binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, - "%d:%d %s %p\n", + "%d:%d %s %016llx\n", proc->pid, thread->pid, cmd == BR_DEAD_BINDER ? "BR_DEAD_BINDER" : "BR_CLEAR_DEATH_NOTIFICATION_DONE", - death->cookie); + (u64)death->cookie); if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION) { list_del(&w->entry); @@ -2312,8 +2333,8 @@ retry: binder_set_nice(target_node->min_priority); cmd = BR_TRANSACTION; } else { - tr.target.ptr = NULL; - tr.cookie = NULL; + tr.target.ptr = 0; + tr.cookie = 0; cmd = BR_REPLY; } tr.code = t->code; @@ -2330,8 +2351,9 @@ retry: tr.data_size = t->buffer->data_size; tr.offsets_size = t->buffer->offsets_size; - tr.data.ptr.buffer = (void *)t->buffer->data + - proc->user_buffer_offset; + tr.data.ptr.buffer = (binder_uintptr_t)( + (uintptr_t)t->buffer->data + + proc->user_buffer_offset); tr.data.ptr.offsets = tr.data.ptr.buffer + ALIGN(t->buffer->data_size, sizeof(void *)); @@ -2346,14 +2368,14 @@ retry: trace_binder_transaction_received(t); binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d %s %d %d:%d, cmd %d size %zd-%zd ptr %p-%p\n", + "%d:%d %s %d %d:%d, cmd %d size %zd-%zd ptr %016llx-%016llx\n", proc->pid, thread->pid, (cmd == BR_TRANSACTION) ? "BR_TRANSACTION" : "BR_REPLY", t->debug_id, t->from ? t->from->proc->pid : 0, t->from ? t->from->pid : 0, cmd, t->buffer->data_size, t->buffer->offsets_size, - tr.data.ptr.buffer, tr.data.ptr.offsets); + (u64)tr.data.ptr.buffer, (u64)tr.data.ptr.offsets); list_del(&t->work.entry); t->buffer->allow_user_free = 1; @@ -2423,8 +2445,8 @@ static void binder_release_work(struct list_head *list) death = container_of(w, struct binder_ref_death, work); binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, - "undelivered death notification, %p\n", - death->cookie); + "undelivered death notification, %016llx\n", + (u64)death->cookie); kfree(death); binder_stats_deleted(BINDER_STAT_DEATH); } break; @@ -2580,12 +2602,16 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) goto err; } binder_debug(BINDER_DEBUG_READ_WRITE, - "%d:%d write %zd at %016lx, read %zd at %016lx\n", - proc->pid, thread->pid, bwr.write_size, - bwr.write_buffer, bwr.read_size, bwr.read_buffer); + "%d:%d write %lld at %016llx, read %lld at %016llx\n", + proc->pid, thread->pid, + (u64)bwr.write_size, (u64)bwr.write_buffer, + (u64)bwr.read_size, (u64)bwr.read_buffer); if (bwr.write_size > 0) { - ret = binder_thread_write(proc, thread, (void __user *)bwr.write_buffer, bwr.write_size, &bwr.write_consumed); + ret = binder_thread_write(proc, thread, + bwr.write_buffer, + bwr.write_size, + &bwr.write_consumed); trace_binder_write_done(ret); if (ret < 0) { bwr.read_consumed = 0; @@ -2595,7 +2621,10 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } } if (bwr.read_size > 0) { - ret = binder_thread_read(proc, thread, (void __user *)bwr.read_buffer, bwr.read_size, &bwr.read_consumed, filp->f_flags & O_NONBLOCK); + ret = binder_thread_read(proc, thread, bwr.read_buffer, + bwr.read_size, + &bwr.read_consumed, + filp->f_flags & O_NONBLOCK); trace_binder_read_done(ret); if (!list_empty(&proc->todo)) wake_up_interruptible(&proc->wait); @@ -2606,9 +2635,10 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } } binder_debug(BINDER_DEBUG_READ_WRITE, - "%d:%d wrote %zd of %zd, read return %zd of %zd\n", - proc->pid, thread->pid, bwr.write_consumed, bwr.write_size, - bwr.read_consumed, bwr.read_size); + "%d:%d wrote %lld of %lld, read return %lld of %lld\n", + proc->pid, thread->pid, + (u64)bwr.write_consumed, (u64)bwr.write_size, + (u64)bwr.read_consumed, (u64)bwr.read_size); if (copy_to_user(ubuf, &bwr, sizeof(bwr))) { ret = -EFAULT; goto err; @@ -2637,7 +2667,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } } else binder_context_mgr_uid = current->cred->euid; - binder_context_mgr_node = binder_new_node(proc, NULL, NULL); + binder_context_mgr_node = binder_new_node(proc, 0, 0); if (binder_context_mgr_node == NULL) { ret = -ENOMEM; goto err; @@ -3132,8 +3162,9 @@ static void print_binder_work(struct seq_file *m, const char *prefix, break; case BINDER_WORK_NODE: node = container_of(w, struct binder_node, work); - seq_printf(m, "%snode work %d: u%p c%p\n", - prefix, node->debug_id, node->ptr, node->cookie); + seq_printf(m, "%snode work %d: u%016llx c%016llx\n", + prefix, node->debug_id, + (u64)node->ptr, (u64)node->cookie); break; case BINDER_WORK_DEAD_BINDER: seq_printf(m, "%shas dead binder\n", prefix); @@ -3193,8 +3224,8 @@ static void print_binder_node(struct seq_file *m, struct binder_node *node) hlist_for_each_entry(ref, &node->refs, node_entry) count++; - seq_printf(m, " node %d: u%p c%p hs %d hw %d ls %d lw %d is %d iw %d", - node->debug_id, node->ptr, node->cookie, + seq_printf(m, " node %d: u%016llx c%016llx hs %d hw %d ls %d lw %d is %d iw %d", + node->debug_id, (u64)node->ptr, (u64)node->cookie, node->has_strong_ref, node->has_weak_ref, node->local_strong_refs, node->local_weak_refs, node->internal_strong_refs, count); @@ -3496,6 +3527,7 @@ static const struct file_operations binder_fops = { .owner = THIS_MODULE, .poll = binder_poll, .unlocked_ioctl = binder_ioctl, + .compat_ioctl = binder_ioctl, .mmap = binder_mmap, .open = binder_open, .flush = binder_flush, diff --git a/drivers/staging/android/binder.h b/drivers/staging/android/binder.h index d4101a671718..eb0834656dfe 100644 --- a/drivers/staging/android/binder.h +++ b/drivers/staging/android/binder.h @@ -20,6 +20,10 @@ #ifndef _LINUX_BINDER_H #define _LINUX_BINDER_H +#ifdef CONFIG_ANDROID_BINDER_IPC_32BIT +#define BINDER_IPC_32BIT 1 +#endif + #include "uapi/binder.h" #endif /* _LINUX_BINDER_H */ diff --git a/drivers/staging/android/binder_trace.h b/drivers/staging/android/binder_trace.h index 82a567c2af67..7f20f3dc8369 100644 --- a/drivers/staging/android/binder_trace.h +++ b/drivers/staging/android/binder_trace.h @@ -152,7 +152,7 @@ TRACE_EVENT(binder_transaction_node_to_ref, TP_STRUCT__entry( __field(int, debug_id) __field(int, node_debug_id) - __field(void __user *, node_ptr) + __field(binder_uintptr_t, node_ptr) __field(int, ref_debug_id) __field(uint32_t, ref_desc) ), @@ -163,8 +163,9 @@ TRACE_EVENT(binder_transaction_node_to_ref, __entry->ref_debug_id = ref->debug_id; __entry->ref_desc = ref->desc; ), - TP_printk("transaction=%d node=%d src_ptr=0x%p ==> dest_ref=%d dest_desc=%d", - __entry->debug_id, __entry->node_debug_id, __entry->node_ptr, + TP_printk("transaction=%d node=%d src_ptr=0x%016llx ==> dest_ref=%d dest_desc=%d", + __entry->debug_id, __entry->node_debug_id, + (u64)__entry->node_ptr, __entry->ref_debug_id, __entry->ref_desc) ); @@ -177,7 +178,7 @@ TRACE_EVENT(binder_transaction_ref_to_node, __field(int, ref_debug_id) __field(uint32_t, ref_desc) __field(int, node_debug_id) - __field(void __user *, node_ptr) + __field(binder_uintptr_t, node_ptr) ), TP_fast_assign( __entry->debug_id = t->debug_id; @@ -186,9 +187,10 @@ TRACE_EVENT(binder_transaction_ref_to_node, __entry->node_debug_id = ref->node->debug_id; __entry->node_ptr = ref->node->ptr; ), - TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%p", + TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%016llx", __entry->debug_id, __entry->node_debug_id, - __entry->ref_debug_id, __entry->ref_desc, __entry->node_ptr) + __entry->ref_debug_id, __entry->ref_desc, + (u64)__entry->node_ptr) ); TRACE_EVENT(binder_transaction_ref_to_ref, diff --git a/drivers/staging/android/uapi/binder.h b/drivers/staging/android/uapi/binder.h index 008722a249bc..a86a1d8e76c4 100644 --- a/drivers/staging/android/uapi/binder.h +++ b/drivers/staging/android/uapi/binder.h @@ -39,6 +39,14 @@ enum { FLAT_BINDER_FLAG_ACCEPTS_FDS = 0x100, }; +#ifdef BINDER_IPC_32BIT +typedef __u32 binder_size_t; +typedef __u32 binder_uintptr_t; +#else +typedef __u64 binder_size_t; +typedef __u64 binder_uintptr_t; +#endif + /* * This is the flattened representation of a Binder object for transfer * between processes. The 'offsets' supplied as part of a binder transaction @@ -53,12 +61,12 @@ struct flat_binder_object { /* 8 bytes of data. */ union { - void __user *binder; /* local object */ - __u32 handle; /* remote object */ + binder_uintptr_t binder; /* local object */ + __u32 handle; /* remote object */ }; /* extra data associated with local object */ - void __user *cookie; + binder_uintptr_t cookie; }; /* @@ -67,12 +75,12 @@ struct flat_binder_object { */ struct binder_write_read { - size_t write_size; /* bytes to write */ - size_t write_consumed; /* bytes consumed by driver */ - unsigned long write_buffer; - size_t read_size; /* bytes to read */ - size_t read_consumed; /* bytes consumed by driver */ - unsigned long read_buffer; + binder_size_t write_size; /* bytes to write */ + binder_size_t write_consumed; /* bytes consumed by driver */ + binder_uintptr_t write_buffer; + binder_size_t read_size; /* bytes to read */ + binder_size_t read_consumed; /* bytes consumed by driver */ + binder_uintptr_t read_buffer; }; /* Use with BINDER_VERSION, driver fills in fields. */ @@ -82,7 +90,11 @@ struct binder_version { }; /* This is the current protocol version. */ +#ifdef BINDER_IPC_32BIT #define BINDER_CURRENT_PROTOCOL_VERSION 7 +#else +#define BINDER_CURRENT_PROTOCOL_VERSION 8 +#endif #define BINDER_WRITE_READ _IOWR('b', 1, struct binder_write_read) #define BINDER_SET_IDLE_TIMEOUT _IOW('b', 3, __s64) @@ -119,18 +131,20 @@ struct binder_transaction_data { * identifying the target and contents of the transaction. */ union { - __u32 handle; /* target descriptor of command transaction */ - void *ptr; /* target descriptor of return transaction */ + /* target descriptor of command transaction */ + __u32 handle; + /* target descriptor of return transaction */ + binder_uintptr_t ptr; } target; - void *cookie; /* target object cookie */ + binder_uintptr_t cookie; /* target object cookie */ __u32 code; /* transaction command */ /* General information about the transaction. */ __u32 flags; pid_t sender_pid; uid_t sender_euid; - size_t data_size; /* number of bytes of data */ - size_t offsets_size; /* number of bytes of offsets */ + binder_size_t data_size; /* number of bytes of data */ + binder_size_t offsets_size; /* number of bytes of offsets */ /* If this transaction is inline, the data immediately * follows here; otherwise, it ends with a pointer to @@ -139,22 +153,22 @@ struct binder_transaction_data { union { struct { /* transaction data */ - const void __user *buffer; + binder_uintptr_t buffer; /* offsets from buffer to flat_binder_object structs */ - const void __user *offsets; + binder_uintptr_t offsets; } ptr; __u8 buf[8]; } data; }; struct binder_ptr_cookie { - void *ptr; - void *cookie; + binder_uintptr_t ptr; + binder_uintptr_t cookie; }; struct binder_handle_cookie { __u32 handle; - void *cookie; + binder_uintptr_t cookie; } __attribute__((packed)); struct binder_pri_desc { @@ -164,8 +178,8 @@ struct binder_pri_desc { struct binder_pri_ptr_cookie { __s32 priority; - void *ptr; - void *cookie; + binder_uintptr_t ptr; + binder_uintptr_t cookie; }; enum binder_driver_return_protocol { @@ -240,11 +254,11 @@ enum binder_driver_return_protocol { * stop threadpool thread */ - BR_DEAD_BINDER = _IOR('r', 15, void *), + BR_DEAD_BINDER = _IOR('r', 15, binder_uintptr_t), /* * void *: cookie */ - BR_CLEAR_DEATH_NOTIFICATION_DONE = _IOR('r', 16, void *), + BR_CLEAR_DEATH_NOTIFICATION_DONE = _IOR('r', 16, binder_uintptr_t), /* * void *: cookie */ @@ -270,7 +284,7 @@ enum binder_driver_command_protocol { * Else you have acquired a primary reference on the object. */ - BC_FREE_BUFFER = _IOW('c', 3, void *), + BC_FREE_BUFFER = _IOW('c', 3, binder_uintptr_t), /* * void *: ptr to transaction data received on a read */ @@ -325,7 +339,7 @@ enum binder_driver_command_protocol { * void *: cookie */ - BC_DEAD_BINDER_DONE = _IOW('c', 16, void *), + BC_DEAD_BINDER_DONE = _IOW('c', 16, binder_uintptr_t), /* * void *: cookie */ -- cgit v1.2.3 From 9c5197b7a50ae6a323b0281826fb36ab6389d74e Mon Sep 17 00:00:00 2001 From: Mathieu Maret Date: Tue, 15 Apr 2014 12:03:05 +0200 Subject: staging: binder: add __user annotation in binder.c Add __user to binder_version to correct sparse warning. Reduce line size to fit to coding style. Change-Id: I8694fb5a082721c69d1596b2853c5d4899f6536b Signed-off-by: Mathieu Maret Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index cfe4bc8f05cb..1f5e2499ac1b 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -2683,16 +2683,20 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) binder_free_thread(proc, thread); thread = NULL; break; - case BINDER_VERSION: + case BINDER_VERSION: { + struct binder_version __user *ver = ubuf; + if (size != sizeof(struct binder_version)) { ret = -EINVAL; goto err; } - if (put_user(BINDER_CURRENT_PROTOCOL_VERSION, &((struct binder_version *)ubuf)->protocol_version)) { + if (put_user(BINDER_CURRENT_PROTOCOL_VERSION, + &ver->protocol_version)) { ret = -EINVAL; goto err; } break; + } default: ret = -EINVAL; goto err; -- cgit v1.2.3 From d0a7d1865c7ff9054a8eeb722e4f7ae0d80fcdb3 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Thu, 1 May 2014 01:30:23 +0900 Subject: staging: android: fix missing a blank line after declarations This patch fixes "Missing a blank line after declarations" warnings. Change-Id: Iede8a80f003eba36fd1f8d3ec8135d9d35c16ee9 Signed-off-by: Seunghun Lee Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/alarm-dev.c | 1 + drivers/staging/android/binder.c | 34 +++++++++++++++++++++++++++ drivers/staging/android/ion/ion.c | 10 ++++++++ drivers/staging/android/ion/ion_heap.c | 2 ++ drivers/staging/android/ion/ion_priv.h | 1 + drivers/staging/android/ion/ion_system_heap.c | 5 ++++ drivers/staging/android/logger.c | 3 +++ drivers/staging/android/sw_sync.c | 2 ++ drivers/staging/android/sync.c | 8 +++++++ drivers/staging/android/timed_gpio.c | 1 + 10 files changed, 67 insertions(+) diff --git a/drivers/staging/android/alarm-dev.c b/drivers/staging/android/alarm-dev.c index 6dc27dac679d..053233c5dec1 100644 --- a/drivers/staging/android/alarm-dev.c +++ b/drivers/staging/android/alarm-dev.c @@ -326,6 +326,7 @@ static int alarm_release(struct inode *inode, struct file *file) if (file->private_data) { for (i = 0; i < ANDROID_ALARM_TYPE_COUNT; i++) { uint32_t alarm_type_mask = 1U << i; + if (alarm_enabled & alarm_type_mask) { alarm_dbg(INFO, "%s: clear alarm, pending %d\n", diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 1f5e2499ac1b..35c00abc3dd1 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -118,6 +118,7 @@ static int binder_set_stop_on_user_error(const char *val, struct kernel_param *kp) { int ret; + ret = param_set_int(val, kp); if (binder_stop_on_user_error < 2) wake_up(&binder_user_error_wait); @@ -194,6 +195,7 @@ static struct binder_transaction_log_entry *binder_transaction_log_add( struct binder_transaction_log *log) { struct binder_transaction_log_entry *e; + e = &log->entry[log->next]; memset(e, 0, sizeof(*e)); log->next++; @@ -432,6 +434,7 @@ static inline void binder_unlock(const char *tag) static void binder_set_nice(long nice) { long min_nice; + if (can_nice(current, nice)) { set_user_nice(current, nice); return; @@ -584,6 +587,7 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { int ret; struct page **page_array_ptr; + page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE]; BUG_ON(*page); @@ -726,6 +730,7 @@ static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, binder_insert_allocated_buffer(proc, buffer); if (buffer_size != size) { struct binder_buffer *new_buffer = (void *)buffer->data + size; + list_add(&new_buffer->entry, &buffer->entry); new_buffer->free = 1; binder_insert_free_buffer(proc, new_buffer); @@ -838,6 +843,7 @@ static void binder_free_buf(struct binder_proc *proc, if (!list_is_last(&buffer->entry, &proc->buffers)) { struct binder_buffer *next = list_entry(buffer->entry.next, struct binder_buffer, entry); + if (next->free) { rb_erase(&next->rb_node, &proc->free_buffers); binder_delete_free_buffer(proc, next); @@ -846,6 +852,7 @@ static void binder_free_buf(struct binder_proc *proc, if (proc->buffers.next != &buffer->entry) { struct binder_buffer *prev = list_entry(buffer->entry.prev, struct binder_buffer, entry); + if (prev->free) { binder_delete_free_buffer(proc, buffer); rb_erase(&prev->rb_node, &proc->free_buffers); @@ -1107,6 +1114,7 @@ static int binder_inc_ref(struct binder_ref *ref, int strong, struct list_head *target_list) { int ret; + if (strong) { if (ref->strong == 0) { ret = binder_inc_node(ref->node, 1, 1, target_list); @@ -1138,6 +1146,7 @@ static int binder_dec_ref(struct binder_ref *ref, int strong) ref->strong--; if (ref->strong == 0) { int ret; + ret = binder_dec_node(ref->node, strong, 1); if (ret) return ret; @@ -1177,6 +1186,7 @@ static void binder_send_failed_reply(struct binder_transaction *t, uint32_t error_code) { struct binder_thread *target_thread; + BUG_ON(t->flags & TF_ONE_WAY); while (1) { target_thread = t->from; @@ -1247,6 +1257,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, off_end = (void *)offp + buffer->offsets_size; for (; offp < off_end; offp++) { struct flat_binder_object *fp; + if (*offp > buffer->data_size - sizeof(*fp) || buffer->data_size < sizeof(*fp) || !IS_ALIGNED(*offp, sizeof(u32))) { @@ -1259,6 +1270,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, case BINDER_TYPE_BINDER: case BINDER_TYPE_WEAK_BINDER: { struct binder_node *node = binder_get_node(proc, fp->binder); + if (node == NULL) { pr_err("transaction release %d bad node %016llx\n", debug_id, (u64)fp->binder); @@ -1272,6 +1284,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { struct binder_ref *ref = binder_get_ref(proc, fp->handle); + if (ref == NULL) { pr_err("transaction release %d bad handle %d\n", debug_id, fp->handle); @@ -1363,6 +1376,7 @@ static void binder_transaction(struct binder_proc *proc, } else { if (tr->target.handle) { struct binder_ref *ref; + ref = binder_get_ref(proc, tr->target.handle); if (ref == NULL) { binder_user_error("%d:%d got transaction to invalid handle\n", @@ -1386,6 +1400,7 @@ static void binder_transaction(struct binder_proc *proc, } if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) { struct binder_transaction *tmp; + tmp = thread->transaction_stack; if (tmp->to_thread != thread) { binder_user_error("%d:%d got new transaction with bad transaction stack, transaction %d has target %d:%d\n", @@ -1501,6 +1516,7 @@ static void binder_transaction(struct binder_proc *proc, off_end = (void *)offp + tr->offsets_size; for (; offp < off_end; offp++) { struct flat_binder_object *fp; + if (*offp > t->buffer->data_size - sizeof(*fp) || t->buffer->data_size < sizeof(*fp) || !IS_ALIGNED(*offp, sizeof(u32))) { @@ -1515,6 +1531,7 @@ static void binder_transaction(struct binder_proc *proc, case BINDER_TYPE_WEAK_BINDER: { struct binder_ref *ref; struct binder_node *node = binder_get_node(proc, fp->binder); + if (node == NULL) { node = binder_new_node(proc, fp->binder, fp->cookie); if (node == NULL) { @@ -1553,6 +1570,7 @@ static void binder_transaction(struct binder_proc *proc, case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { struct binder_ref *ref = binder_get_ref(proc, fp->handle); + if (ref == NULL) { binder_user_error("%d:%d got transaction with invalid handle, %d\n", proc->pid, @@ -1575,6 +1593,7 @@ static void binder_transaction(struct binder_proc *proc, (u64)ref->node->ptr); } else { struct binder_ref *new_ref; + new_ref = binder_get_ref_for_node(target_proc, ref->node); if (new_ref == NULL) { return_error = BR_FAILED_REPLY; @@ -1694,6 +1713,7 @@ err_no_context_mgr_node: { struct binder_transaction_log_entry *fe; + fe = binder_transaction_log_add(&binder_transaction_log_failed); *fe = *e; } @@ -2024,12 +2044,14 @@ static int binder_thread_write(struct binder_proc *proc, struct binder_work *w; binder_uintptr_t cookie; struct binder_ref_death *death = NULL; + if (get_user(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(void *); list_for_each_entry(w, &proc->delivered_death, entry) { struct binder_ref_death *tmp_death = container_of(w, struct binder_ref_death, work); + if (tmp_death->cookie == cookie) { death = tmp_death; break; @@ -2216,6 +2238,7 @@ retry: const char *cmd_name; int strong = node->internal_strong_refs || node->local_strong_refs; int weak = !hlist_empty(&node->refs) || node->local_weak_refs || strong; + if (weak && !node->has_weak_ref) { cmd = BR_INCREFS; cmd_name = "BR_INCREFS"; @@ -2322,6 +2345,7 @@ retry: BUG_ON(t->buffer == NULL); if (t->buffer->target_node) { struct binder_node *target_node = t->buffer->target_node; + tr.target.ptr = target_node->ptr; tr.cookie = target_node->cookie; t->saved_priority = task_nice(current); @@ -2343,6 +2367,7 @@ retry: if (t->from) { struct task_struct *sender = t->from->proc->tsk; + tr.sender_pid = task_tgid_nr_ns(sender, task_active_pid_ns(current)); } else { @@ -2413,6 +2438,7 @@ done: static void binder_release_work(struct list_head *list) { struct binder_work *w; + while (!list_empty(list)) { w = list_first_entry(list, struct binder_work, entry); list_del_init(&w->entry); @@ -2593,6 +2619,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) switch (cmd) { case BINDER_WRITE_READ: { struct binder_write_read bwr; + if (size != sizeof(struct binder_write_read)) { ret = -EINVAL; goto err; @@ -2717,6 +2744,7 @@ err_unlocked: static void binder_vma_open(struct vm_area_struct *vma) { struct binder_proc *proc = vma->vm_private_data; + binder_debug(BINDER_DEBUG_OPEN_CLOSE, "%d open vm area %lx-%lx (%ld K) vma %lx pagep %lx\n", proc->pid, vma->vm_start, vma->vm_end, @@ -2727,6 +2755,7 @@ static void binder_vma_open(struct vm_area_struct *vma) static void binder_vma_close(struct vm_area_struct *vma) { struct binder_proc *proc = vma->vm_private_data; + binder_debug(BINDER_DEBUG_OPEN_CLOSE, "%d close vm area %lx-%lx (%ld K) vma %lx pagep %lx\n", proc->pid, vma->vm_start, vma->vm_end, @@ -2869,6 +2898,7 @@ static int binder_open(struct inode *nodp, struct file *filp) if (binder_debugfs_dir_entry_proc) { char strbuf[11]; + snprintf(strbuf, sizeof(strbuf), "%u", proc->pid); proc->debugfs_entry = debugfs_create_file(strbuf, S_IRUGO, binder_debugfs_dir_entry_proc, proc, &binder_proc_fops); @@ -2890,8 +2920,10 @@ static void binder_deferred_flush(struct binder_proc *proc) { struct rb_node *n; int wake_count = 0; + for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) { struct binder_thread *thread = rb_entry(n, struct binder_thread, rb_node); + thread->looper |= BINDER_LOOPER_STATE_NEED_RETURN; if (thread->looper & BINDER_LOOPER_STATE_WAITING) { wake_up_interruptible(&thread->wait); @@ -2908,6 +2940,7 @@ static void binder_deferred_flush(struct binder_proc *proc) static int binder_release(struct inode *nodp, struct file *filp) { struct binder_proc *proc = filp->private_data; + debugfs_remove(proc->debugfs_entry); binder_defer_work(proc, BINDER_DEFERRED_RELEASE); @@ -3069,6 +3102,7 @@ static void binder_deferred_func(struct work_struct *work) struct files_struct *files; int defer; + do { binder_lock(__func__); mutex_lock(&binder_deferred_lock); diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c index 48774e3974aa..698e363be551 100644 --- a/drivers/staging/android/ion/ion.c +++ b/drivers/staging/android/ion/ion.c @@ -407,6 +407,7 @@ static struct ion_handle *ion_handle_lookup(struct ion_client *client, while (n) { struct ion_handle *entry = rb_entry(n, struct ion_handle, node); + if (buffer < entry->buffer) n = n->rb_left; else if (buffer > entry->buffer) @@ -719,9 +720,11 @@ static int ion_get_client_serial(const struct rb_root *root, { int serial = -1; struct rb_node *node; + for (node = rb_first(root); node; node = rb_next(node)) { struct ion_client *client = rb_entry(node, struct ion_client, node); + if (strcmp(client->name, name)) continue; serial = max(serial, client->display_serial); @@ -1034,12 +1037,14 @@ static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) static void ion_dma_buf_release(struct dma_buf *dmabuf) { struct ion_buffer *buffer = dmabuf->priv; + ion_buffer_put(buffer); } static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset) { struct ion_buffer *buffer = dmabuf->priv; + return buffer->vaddr + offset * PAGE_SIZE; } @@ -1293,6 +1298,7 @@ static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) case ION_IOC_IMPORT: { struct ion_handle *handle; + handle = ion_import_dma_buf(client, data.fd.fd); if (IS_ERR(handle)) ret = PTR_ERR(handle); @@ -1394,6 +1400,7 @@ static int ion_debug_heap_show(struct seq_file *s, void *unused) struct ion_client *client = rb_entry(n, struct ion_client, node); size_t size = ion_debug_heap_total(client, heap->id); + if (!size) continue; if (client->task) { @@ -1518,6 +1525,7 @@ void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap) if (!debug_file) { char buf[256], *path; + path = dentry_path(dev->heaps_debug_root, buf, 256); pr_err("Failed to create heap debugfs at %s/%s\n", path, heap->name); @@ -1533,6 +1541,7 @@ void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap) &debug_shrink_fops); if (!debug_file) { char buf[256], *path; + path = dentry_path(dev->heaps_debug_root, buf, 256); pr_err("Failed to create heap shrinker debugfs at %s/%s\n", path, debug_name); @@ -1608,6 +1617,7 @@ void __init ion_reserve(struct ion_platform_data *data) if (data->heaps[i].base == 0) { phys_addr_t paddr; + paddr = memblock_alloc_base(data->heaps[i].size, data->heaps[i].align, MEMBLOCK_ALLOC_ANYWHERE); diff --git a/drivers/staging/android/ion/ion_heap.c b/drivers/staging/android/ion/ion_heap.c index 750b76af0cf3..e1bbd8b1b4f4 100644 --- a/drivers/staging/android/ion/ion_heap.c +++ b/drivers/staging/android/ion/ion_heap.c @@ -48,6 +48,7 @@ void *ion_heap_map_kernel(struct ion_heap *heap, for_each_sg(table->sgl, sg, table->nents, i) { int npages_this_entry = PAGE_ALIGN(sg->length) / PAGE_SIZE; struct page *page = sg_page(sg); + BUG_ON(i >= npages); for (j = 0; j < npages_this_entry; j++) *(tmp++) = page++; @@ -105,6 +106,7 @@ int ion_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer, static int ion_heap_clear_pages(struct page **pages, int num, pgprot_t pgprot) { void *addr = vm_map_ram(pages, num, -1, pgprot); + if (!addr) return -ENOMEM; memset(addr, 0, PAGE_SIZE * num); diff --git a/drivers/staging/android/ion/ion_priv.h b/drivers/staging/android/ion/ion_priv.h index 18ef365e7552..7c06e2e4fc01 100644 --- a/drivers/staging/android/ion/ion_priv.h +++ b/drivers/staging/android/ion/ion_priv.h @@ -178,6 +178,7 @@ struct ion_heap { spinlock_t free_lock; wait_queue_head_t waitqueue; struct task_struct *task; + int (*debug_show)(struct ion_heap *heap, struct seq_file *, void *); }; diff --git a/drivers/staging/android/ion/ion_system_heap.c b/drivers/staging/android/ion/ion_system_heap.c index a052418d2efc..5945445ee5f3 100644 --- a/drivers/staging/android/ion/ion_system_heap.c +++ b/drivers/staging/android/ion/ion_system_heap.c @@ -34,6 +34,7 @@ static const int num_orders = ARRAY_SIZE(orders); static int order_to_index(unsigned int order) { int i; + for (i = 0; i < num_orders; i++) if (order == orders[i]) return i; @@ -92,6 +93,7 @@ static void free_buffer_page(struct ion_system_heap *heap, if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) { struct ion_page_pool *pool = heap->pools[order_to_index(order)]; + ion_page_pool_free(pool, page); } else { __free_pages(page, order); @@ -242,6 +244,7 @@ static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, for (i = 0; i < num_orders; i++) { struct ion_page_pool *pool = sys_heap->pools[i]; + nr_total += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan); } @@ -267,8 +270,10 @@ static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s, struct ion_system_heap, heap); int i; + for (i = 0; i < num_orders; i++) { struct ion_page_pool *pool = sys_heap->pools[i]; + seq_printf(s, "%d order %u highmem pages in pool = %lu total\n", pool->high_count, pool->order, (1 << pool->order) * PAGE_SIZE * pool->high_count); diff --git a/drivers/staging/android/logger.c b/drivers/staging/android/logger.c index 9bd874789ce5..c000c66004f6 100644 --- a/drivers/staging/android/logger.c +++ b/drivers/staging/android/logger.c @@ -108,6 +108,7 @@ static inline struct logger_log *file_get_log(struct file *file) { if (file->f_mode & FMODE_READ) { struct logger_reader *reader = file->private_data; + return reader->log; } else return file->private_data; @@ -124,6 +125,7 @@ static struct logger_entry *get_entry_header(struct logger_log *log, size_t off, struct logger_entry *scratch) { size_t len = min(sizeof(struct logger_entry), log->size - off); + if (len != sizeof(struct logger_entry)) { memcpy(((void *) scratch), log->buffer + off, len); memcpy(((void *) scratch) + len, log->buffer, @@ -640,6 +642,7 @@ static unsigned int logger_poll(struct file *file, poll_table *wait) static long logger_set_version(struct logger_reader *reader, void __user *arg) { int version; + if (copy_from_user(&version, arg, sizeof(int))) return -EFAULT; diff --git a/drivers/staging/android/sw_sync.c b/drivers/staging/android/sw_sync.c index 4928f93bdf3d..820af5cfb830 100644 --- a/drivers/staging/android/sw_sync.c +++ b/drivers/staging/android/sw_sync.c @@ -97,6 +97,7 @@ static void sw_sync_pt_value_str(struct sync_pt *sync_pt, char *str, int size) { struct sw_sync_pt *pt = (struct sw_sync_pt *)sync_pt; + snprintf(str, size, "%d", pt->value); } @@ -156,6 +157,7 @@ static int sw_sync_open(struct inode *inode, struct file *file) static int sw_sync_release(struct inode *inode, struct file *file) { struct sw_sync_timeline *obj = file->private_data; + sync_timeline_destroy(&obj->obj); return 0; } diff --git a/drivers/staging/android/sync.c b/drivers/staging/android/sync.c index d38305b40930..61e624905916 100644 --- a/drivers/staging/android/sync.c +++ b/drivers/staging/android/sync.c @@ -384,6 +384,7 @@ static void sync_fence_detach_pts(struct sync_fence *fence) list_for_each_safe(pos, n, &fence->pt_list_head) { struct sync_pt *pt = container_of(pos, struct sync_pt, pt_list); + sync_timeline_remove_pt(pt); } } @@ -394,6 +395,7 @@ static void sync_fence_free_pts(struct sync_fence *fence) list_for_each_safe(pos, n, &fence->pt_list_head) { struct sync_pt *pt = container_of(pos, struct sync_pt, pt_list); + sync_pt_free(pt); } } @@ -827,6 +829,7 @@ static long sync_fence_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct sync_fence *fence = file->private_data; + switch (cmd) { case SYNC_IOC_WAIT: return sync_fence_ioctl_wait(fence, arg); @@ -856,18 +859,21 @@ static const char *sync_status_str(int status) static void sync_print_pt(struct seq_file *s, struct sync_pt *pt, bool fence) { int status = pt->status; + seq_printf(s, " %s%spt %s", fence ? pt->parent->name : "", fence ? "_" : "", sync_status_str(status)); if (pt->status) { struct timeval tv = ktime_to_timeval(pt->timestamp); + seq_printf(s, "@%ld.%06ld", tv.tv_sec, tv.tv_usec); } if (pt->parent->ops->timeline_value_str && pt->parent->ops->pt_value_str) { char value[64]; + pt->parent->ops->pt_value_str(pt, value, sizeof(value)); seq_printf(s, ": %s", value); if (fence) { @@ -892,6 +898,7 @@ static void sync_print_obj(struct seq_file *s, struct sync_timeline *obj) if (obj->ops->timeline_value_str) { char value[64]; + obj->ops->timeline_value_str(obj, value, sizeof(value)); seq_printf(s, ": %s", value); } else if (obj->ops->print_obj) { @@ -1001,6 +1008,7 @@ void sync_dump(void) for (i = 0; i < s.count; i += DUMP_CHUNK) { if ((s.count - i) > DUMP_CHUNK) { char c = s.buf[i + DUMP_CHUNK]; + s.buf[i + DUMP_CHUNK] = 0; pr_cont("%s", s.buf + i); s.buf[i + DUMP_CHUNK] = c; diff --git a/drivers/staging/android/timed_gpio.c b/drivers/staging/android/timed_gpio.c index e81451425c01..ae9966d1f7cc 100644 --- a/drivers/staging/android/timed_gpio.c +++ b/drivers/staging/android/timed_gpio.c @@ -51,6 +51,7 @@ static int gpio_get_time(struct timed_output_dev *dev) if (hrtimer_active(&data->timer)) { ktime_t r = hrtimer_get_remaining(&data->timer); struct timeval t = ktime_to_timeval(r); + return t.tv_sec * 1000 + t.tv_usec / 1000; } else return 0; -- cgit v1.2.3 From 12b727a8f9ba257c8d5882a65661e5a81521e571 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Wed, 30 Apr 2014 23:58:55 -0700 Subject: staging: binder: cleanup dereference of noderef expressions Clean up sparse warnings for cred struct dereference. Change-Id: I78059976c0488abfe9eb4e9f0b0f8ac10c7ef4f9 Signed-off-by: Jerry Snitselaar Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 35c00abc3dd1..c78411a22562 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1326,6 +1326,7 @@ static void binder_transaction(struct binder_proc *proc, struct binder_transaction *in_reply_to = NULL; struct binder_transaction_log_entry *e; uint32_t return_error; + const struct cred *cred = __task_cred(proc->tsk); e = binder_transaction_log_add(&binder_transaction_log); e->call_type = reply ? 2 : !!(tr->flags & TF_ONE_WAY); @@ -1467,7 +1468,7 @@ static void binder_transaction(struct binder_proc *proc, t->from = thread; else t->from = NULL; - t->sender_euid = proc->tsk->cred->euid; + t->sender_euid = cred->euid; t->to_proc = target_proc; t->to_thread = target_thread; t->code = tr->code; @@ -2600,6 +2601,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) struct binder_thread *thread; unsigned int size = _IOC_SIZE(cmd); void __user *ubuf = (void __user *)arg; + const struct cred *cred = current_cred(); /*pr_info("binder_ioctl: %d:%d %x %lx\n", proc->pid, current->pid, cmd, arg);*/ @@ -2685,15 +2687,15 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) goto err; } if (uid_valid(binder_context_mgr_uid)) { - if (!uid_eq(binder_context_mgr_uid, current->cred->euid)) { + if (!uid_eq(binder_context_mgr_uid, cred->euid)) { pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", - from_kuid(&init_user_ns, current->cred->euid), + from_kuid(&init_user_ns, cred->euid), from_kuid(&init_user_ns, binder_context_mgr_uid)); ret = -EPERM; goto err; } } else - binder_context_mgr_uid = current->cred->euid; + binder_context_mgr_uid = cred->euid; binder_context_mgr_node = binder_new_node(proc, 0, 0); if (binder_context_mgr_node == NULL) { ret = -ENOMEM; -- cgit v1.2.3 From 045788ea680a4e204aa832ba3985ee1f6a87abc4 Mon Sep 17 00:00:00 2001 From: Christian Engelmayer Date: Wed, 7 May 2014 21:44:53 +0200 Subject: staging: binder: fix usage of uninit scalar in binder_transaction() Fix the error path when a cookie mismatch is detected. In that case the function jumps to the exit label without setting the uninitialized, local variable 'return_error'. Detected by Coverity - CID 201453. Change-Id: I6c960b7d3ad0adb28fad106a9a0b8cb934013987 Signed-off-by: Christian Engelmayer Acked-by: Arve Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index c78411a22562..e8dd7ddcba41 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1547,6 +1547,7 @@ static void binder_transaction(struct binder_proc *proc, proc->pid, thread->pid, (u64)fp->binder, node->debug_id, (u64)fp->cookie, (u64)node->cookie); + return_error = BR_FAILED_REPLY; goto err_binder_get_ref_for_node_failed; } ref = binder_get_ref_for_node(target_proc, node); -- cgit v1.2.3 From 2d47ab7fed4155c334f881548612290833b1d930 Mon Sep 17 00:00:00 2001 From: Tair Rzayev Date: Sat, 31 May 2014 22:43:34 +0300 Subject: staging: android: binder.c: Use more appropriate functions for euid retrieval Instead of getting the reference to whole credential structure, use task_euid() and current_euid() to get it. Change-Id: Id5b9d0305b5f90023415863e569988023aaae290 Signed-off-by: Tair Rzayev Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index e8dd7ddcba41..989f809f323f 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1326,7 +1326,6 @@ static void binder_transaction(struct binder_proc *proc, struct binder_transaction *in_reply_to = NULL; struct binder_transaction_log_entry *e; uint32_t return_error; - const struct cred *cred = __task_cred(proc->tsk); e = binder_transaction_log_add(&binder_transaction_log); e->call_type = reply ? 2 : !!(tr->flags & TF_ONE_WAY); @@ -1468,7 +1467,7 @@ static void binder_transaction(struct binder_proc *proc, t->from = thread; else t->from = NULL; - t->sender_euid = cred->euid; + t->sender_euid = task_euid(proc->tsk); t->to_proc = target_proc; t->to_thread = target_thread; t->code = tr->code; @@ -2602,7 +2601,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) struct binder_thread *thread; unsigned int size = _IOC_SIZE(cmd); void __user *ubuf = (void __user *)arg; - const struct cred *cred = current_cred(); + kuid_t curr_euid = current_euid(); /*pr_info("binder_ioctl: %d:%d %x %lx\n", proc->pid, current->pid, cmd, arg);*/ @@ -2688,15 +2687,16 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) goto err; } if (uid_valid(binder_context_mgr_uid)) { - if (!uid_eq(binder_context_mgr_uid, cred->euid)) { + if (!uid_eq(binder_context_mgr_uid, curr_euid)) { pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", - from_kuid(&init_user_ns, cred->euid), + from_kuid(&init_user_ns, curr_euid), from_kuid(&init_user_ns, binder_context_mgr_uid)); ret = -EPERM; goto err; } - } else - binder_context_mgr_uid = cred->euid; + } else { + binder_context_mgr_uid = curr_euid; + } binder_context_mgr_node = binder_new_node(proc, 0, 0); if (binder_context_mgr_node == NULL) { ret = -ENOMEM; -- cgit v1.2.3 From a82ef7179a96b30c86258f4744c83cce9bfae441 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Sat, 21 Jun 2014 20:23:16 +0530 Subject: Staging: Android: removed an unnecessary else statement As per checkpatch warning, removed an unnecessary else statement proceeding an if statement with a return. Change-Id: I718d9ece6c8bac128c10ecaf904721410d701b60 Signed-off-by: Karthik Nayak Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 989f809f323f..9455998d037c 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -454,9 +454,8 @@ static size_t binder_buffer_size(struct binder_proc *proc, { if (list_is_last(&buffer->entry, &proc->buffers)) return proc->buffer + proc->buffer_size - (void *)buffer->data; - else - return (size_t)list_entry(buffer->entry.next, - struct binder_buffer, entry) - (size_t)buffer->data; + return (size_t)list_entry(buffer->entry.next, + struct binder_buffer, entry) - (size_t)buffer->data; } static void binder_insert_free_buffer(struct binder_proc *proc, -- cgit v1.2.3 From eab972a63c2171ff7fec78e3ed3f86cb795edebe Mon Sep 17 00:00:00 2001 From: Vinayak Menon Date: Mon, 2 Jun 2014 18:17:59 +0530 Subject: staging: binder: add vm_fault handler An issue was observed when a userspace task exits. The page which hits error here is the zero page. In binder mmap, the whole of vma is not mapped. On a task crash, when debuggerd reads the binder regions, the unmapped areas fall to do_anonymous_page in handle_pte_fault, due to the absence of a vm_fault handler. This results in zero page being mapped. Later in zap_pte_range, vm_normal_page returns zero page in the case of VM_MIXEDMAP and it results in the error. BUG: Bad page map in process mediaserver pte:9dff379f pmd:9bfbd831 page:c0ed8e60 count:1 mapcount:-1 mapping: (null) index:0x0 page flags: 0x404(referenced|reserved) addr:40c3f000 vm_flags:10220051 anon_vma: (null) mapping:d9fe0764 index:fd vma->vm_ops->fault: (null) vma->vm_file->f_op->mmap: binder_mmap+0x0/0x274 CPU: 0 PID: 1463 Comm: mediaserver Tainted: G W 3.10.17+ #1 [] (unwind_backtrace+0x0/0x11c) from [] (show_stack+0x10/0x14) [] (show_stack+0x10/0x14) from [] (print_bad_pte+0x158/0x190) [] (print_bad_pte+0x158/0x190) from [] (unmap_single_vma+0x2e4/0x598) [] (unmap_single_vma+0x2e4/0x598) from [] (unmap_vmas+0x34/0x50) [] (unmap_vmas+0x34/0x50) from [] (exit_mmap+0xc8/0x1e8) [] (exit_mmap+0xc8/0x1e8) from [] (mmput+0x54/0xd0) [] (mmput+0x54/0xd0) from [] (do_exit+0x360/0x990) [] (do_exit+0x360/0x990) from [] (do_group_exit+0x84/0xc0) [] (do_group_exit+0x84/0xc0) from [] (get_signal_to_deliver+0x4d4/0x548) [] (get_signal_to_deliver+0x4d4/0x548) from [] (do_signal+0xa8/0x3b8) Add a vm_fault handler which returns VM_FAULT_SIGBUS, and prevents the wrong fallback to do_anonymous_page. Change-Id: I43c227e489f74f4907f199caf99f571b61883064 Signed-off-by: Vinayak Menon Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 9455998d037c..570595651f3d 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -2768,9 +2768,15 @@ static void binder_vma_close(struct vm_area_struct *vma) binder_defer_work(proc, BINDER_DEFERRED_PUT_FILES); } +static int binder_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + return VM_FAULT_SIGBUS; +} + static struct vm_operations_struct binder_vm_ops = { .open = binder_vma_open, .close = binder_vma_close, + .fault = binder_vm_fault, }; static int binder_mmap(struct file *filp, struct vm_area_struct *vma) -- cgit v1.2.3 From 8802f02dfcf2cf5fb3569fd2b888543ca6a327a6 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Mon, 12 Jan 2015 14:01:03 -0800 Subject: staging: android: binder.c: binder_ioctl() cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit binder_ioctl() is quite huge and checkpatch dirty - mostly because of the amount of code for the BINDER_WRITE_READ and BINDER_SET_CONTEXT_MGR. Moved that code into the new binder_ioctl_write_read() and binder_ioctl_set_ctx_mgr() Change-Id: I9c5cea46a570ca91768e5aa7b11c7178bdbb667d Signed-off-by: Tair Rzayev Cc: Arve HjønnevÃ¥g Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 186 ++++++++++++++++++++++----------------- 1 file changed, 107 insertions(+), 79 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 570595651f3d..265547b8c5da 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -2593,6 +2593,106 @@ static unsigned int binder_poll(struct file *filp, return 0; } +static int binder_ioctl_write_read(struct file *filp, + unsigned int cmd, unsigned long arg, + struct binder_thread *thread) +{ + int ret = 0; + struct binder_proc *proc = filp->private_data; + unsigned int size = _IOC_SIZE(cmd); + void __user *ubuf = (void __user *)arg; + struct binder_write_read bwr; + + if (size != sizeof(struct binder_write_read)) { + ret = -EINVAL; + goto out; + } + if (copy_from_user(&bwr, ubuf, sizeof(bwr))) { + ret = -EFAULT; + goto out; + } + binder_debug(BINDER_DEBUG_READ_WRITE, + "%d:%d write %lld at %016llx, read %lld at %016llx\n", + proc->pid, thread->pid, + (u64)bwr.write_size, (u64)bwr.write_buffer, + (u64)bwr.read_size, (u64)bwr.read_buffer); + + if (bwr.write_size > 0) { + ret = binder_thread_write(proc, thread, + bwr.write_buffer, + bwr.write_size, + &bwr.write_consumed); + trace_binder_write_done(ret); + if (ret < 0) { + bwr.read_consumed = 0; + if (copy_to_user(ubuf, &bwr, sizeof(bwr))) + ret = -EFAULT; + goto out; + } + } + if (bwr.read_size > 0) { + ret = binder_thread_read(proc, thread, bwr.read_buffer, + bwr.read_size, + &bwr.read_consumed, + filp->f_flags & O_NONBLOCK); + trace_binder_read_done(ret); + if (!list_empty(&proc->todo)) + wake_up_interruptible(&proc->wait); + if (ret < 0) { + if (copy_to_user(ubuf, &bwr, sizeof(bwr))) + ret = -EFAULT; + goto out; + } + } + binder_debug(BINDER_DEBUG_READ_WRITE, + "%d:%d wrote %lld of %lld, read return %lld of %lld\n", + proc->pid, thread->pid, + (u64)bwr.write_consumed, (u64)bwr.write_size, + (u64)bwr.read_consumed, (u64)bwr.read_size); + if (copy_to_user(ubuf, &bwr, sizeof(bwr))) { + ret = -EFAULT; + goto out; + } +out: + return ret; +} + +static int binder_ioctl_set_ctx_mgr(struct file *filp) +{ + int ret = 0; + struct binder_proc *proc = filp->private_data; + kuid_t curr_euid = current_euid(); + + if (binder_context_mgr_node != NULL) { + pr_err("BINDER_SET_CONTEXT_MGR already set\n"); + ret = -EBUSY; + goto out; + } + if (uid_valid(binder_context_mgr_uid)) { + if (!uid_eq(binder_context_mgr_uid, curr_euid)) { + pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", + from_kuid(&init_user_ns, curr_euid), + from_kuid(&init_user_ns, + binder_context_mgr_uid)); + ret = -EPERM; + goto out; + } + } else { + binder_context_mgr_uid = curr_euid; + } + binder_context_mgr_node = binder_new_node(proc, 0, 0); + if (binder_context_mgr_node == NULL) { + ret = -ENOMEM; + goto out; + } + binder_context_mgr_node->local_weak_refs++; + binder_context_mgr_node->local_strong_refs++; + binder_context_mgr_node->has_strong_ref = 1; + binder_context_mgr_node->has_weak_ref = 1; +out: + return ret; +} + static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { int ret; @@ -2600,9 +2700,9 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) struct binder_thread *thread; unsigned int size = _IOC_SIZE(cmd); void __user *ubuf = (void __user *)arg; - kuid_t curr_euid = current_euid(); - /*pr_info("binder_ioctl: %d:%d %x %lx\n", proc->pid, current->pid, cmd, arg);*/ + /*pr_info("binder_ioctl: %d:%d %x %lx\n", + proc->pid, current->pid, cmd, arg);*/ trace_binder_ioctl(cmd, arg); @@ -2618,61 +2718,11 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } switch (cmd) { - case BINDER_WRITE_READ: { - struct binder_write_read bwr; - - if (size != sizeof(struct binder_write_read)) { - ret = -EINVAL; + case BINDER_WRITE_READ: + ret = binder_ioctl_write_read(filp, cmd, arg, thread); + if (ret) goto err; - } - if (copy_from_user(&bwr, ubuf, sizeof(bwr))) { - ret = -EFAULT; - goto err; - } - binder_debug(BINDER_DEBUG_READ_WRITE, - "%d:%d write %lld at %016llx, read %lld at %016llx\n", - proc->pid, thread->pid, - (u64)bwr.write_size, (u64)bwr.write_buffer, - (u64)bwr.read_size, (u64)bwr.read_buffer); - - if (bwr.write_size > 0) { - ret = binder_thread_write(proc, thread, - bwr.write_buffer, - bwr.write_size, - &bwr.write_consumed); - trace_binder_write_done(ret); - if (ret < 0) { - bwr.read_consumed = 0; - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) - ret = -EFAULT; - goto err; - } - } - if (bwr.read_size > 0) { - ret = binder_thread_read(proc, thread, bwr.read_buffer, - bwr.read_size, - &bwr.read_consumed, - filp->f_flags & O_NONBLOCK); - trace_binder_read_done(ret); - if (!list_empty(&proc->todo)) - wake_up_interruptible(&proc->wait); - if (ret < 0) { - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) - ret = -EFAULT; - goto err; - } - } - binder_debug(BINDER_DEBUG_READ_WRITE, - "%d:%d wrote %lld of %lld, read return %lld of %lld\n", - proc->pid, thread->pid, - (u64)bwr.write_consumed, (u64)bwr.write_size, - (u64)bwr.read_consumed, (u64)bwr.read_size); - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) { - ret = -EFAULT; - goto err; - } break; - } case BINDER_SET_MAX_THREADS: if (copy_from_user(&proc->max_threads, ubuf, sizeof(proc->max_threads))) { ret = -EINVAL; @@ -2680,31 +2730,9 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } break; case BINDER_SET_CONTEXT_MGR: - if (binder_context_mgr_node != NULL) { - pr_err("BINDER_SET_CONTEXT_MGR already set\n"); - ret = -EBUSY; + ret = binder_ioctl_set_ctx_mgr(filp); + if (ret) goto err; - } - if (uid_valid(binder_context_mgr_uid)) { - if (!uid_eq(binder_context_mgr_uid, curr_euid)) { - pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", - from_kuid(&init_user_ns, curr_euid), - from_kuid(&init_user_ns, binder_context_mgr_uid)); - ret = -EPERM; - goto err; - } - } else { - binder_context_mgr_uid = curr_euid; - } - binder_context_mgr_node = binder_new_node(proc, 0, 0); - if (binder_context_mgr_node == NULL) { - ret = -ENOMEM; - goto err; - } - binder_context_mgr_node->local_weak_refs++; - binder_context_mgr_node->local_strong_refs++; - binder_context_mgr_node->has_strong_ref = 1; - binder_context_mgr_node->has_weak_ref = 1; break; case BINDER_THREAD_EXIT: binder_debug(BINDER_DEBUG_THREADS, "%d:%d exit\n", -- cgit v1.2.3 From 0137e89dee95eff290c919c3fafb5b1e7d9fad6a Mon Sep 17 00:00:00 2001 From: Lucas Tanure Date: Sun, 13 Jul 2014 21:31:05 -0300 Subject: staging: android: Clean up else statement from binder_send_failed_reply Kernel coding style. Remove useless else statement after return. Changes from v1 and v2: Fix warning for mixed declarations and code. Declaration of "struct binder_transaction *next" made outside of while. Changes from v3: Removed initialization to NULL for next variable. Change-Id: I1ec8b512130595b90098126acf562e58eeca8458 Signed-off-by: Lucas Tanure Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 265547b8c5da..cc3f5a151bd6 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1185,6 +1185,7 @@ static void binder_send_failed_reply(struct binder_transaction *t, uint32_t error_code) { struct binder_thread *target_thread; + struct binder_transaction *next; BUG_ON(t->flags & TF_ONE_WAY); while (1) { @@ -1212,24 +1213,23 @@ static void binder_send_failed_reply(struct binder_transaction *t, target_thread->return_error); } return; - } else { - struct binder_transaction *next = t->from_parent; + } + next = t->from_parent; - binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, - "send failed reply for transaction %d, target dead\n", - t->debug_id); + binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, + "send failed reply for transaction %d, target dead\n", + t->debug_id); - binder_pop_transaction(target_thread, t); - if (next == NULL) { - binder_debug(BINDER_DEBUG_DEAD_BINDER, - "reply failed, no target thread at root\n"); - return; - } - t = next; + binder_pop_transaction(target_thread, t); + if (next == NULL) { binder_debug(BINDER_DEBUG_DEAD_BINDER, - "reply failed, no target thread -- retry %d\n", - t->debug_id); + "reply failed, no target thread at root\n"); + return; } + t = next; + binder_debug(BINDER_DEBUG_DEAD_BINDER, + "reply failed, no target thread -- retry %d\n", + t->debug_id); } } -- cgit v1.2.3 From 36ac7f7143bf8fc813012ab9638d1face082f903 Mon Sep 17 00:00:00 2001 From: Purnendu Kapadia Date: Fri, 15 Aug 2014 18:20:30 +0100 Subject: staging: android: fix attribute as suggested by checkpatch we should use __packed attribute Change-Id: I9c74b57799bc6787d54cf6e2563adc96dff666d2 Signed-off-by: Purnendu Kapadia Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/uapi/binder.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/staging/android/uapi/binder.h b/drivers/staging/android/uapi/binder.h index a86a1d8e76c4..c653800b3a1a 100644 --- a/drivers/staging/android/uapi/binder.h +++ b/drivers/staging/android/uapi/binder.h @@ -169,7 +169,7 @@ struct binder_ptr_cookie { struct binder_handle_cookie { __u32 handle; binder_uintptr_t cookie; -} __attribute__((packed)); +} __packed; struct binder_pri_desc { __s32 priority; -- cgit v1.2.3 From dd0f11419f8fe1a64082575fba2132fe8558c6fe Mon Sep 17 00:00:00 2001 From: William Panlener Date: Wed, 3 Sep 2014 22:44:03 -0500 Subject: staging: android: Break up a long line in binder_send_failed_reply Kernel coding style. Breaking long lines and strings. Change-Id: Ie1af1f3bcfef547ab69f873f2e86ee37c8c23caf Signed-off-by: William Panlener Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index cc3f5a151bd6..124518f53618 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1200,7 +1200,8 @@ static void binder_send_failed_reply(struct binder_transaction *t, if (target_thread->return_error == BR_OK) { binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, "send failed reply for transaction %d to %d:%d\n", - t->debug_id, target_thread->proc->pid, + t->debug_id, + target_thread->proc->pid, target_thread->pid); binder_pop_transaction(target_thread, t); -- cgit v1.2.3 From 0a2be85ca49d19a251aa4ebc232df7b99dcafaa2 Mon Sep 17 00:00:00 2001 From: Dmitry Voytik Date: Mon, 8 Sep 2014 18:16:34 +0400 Subject: staging: binder: fix coding style issues Fix coding style issues: * put braces in all if-else branches; * limit the length of changed lines to 80 columns. checkpatch.pl warning count reduces by 3. Change-Id: I1796588ef0f358780445203c5afa87361ab2bf73 Signed-off-by: Dmitry Voytik Signed-off-by: Greg Kroah-Hartman --- drivers/staging/android/binder.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 124518f53618..b44b9398810c 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -2201,12 +2201,16 @@ retry: struct binder_work *w; struct binder_transaction *t = NULL; - if (!list_empty(&thread->todo)) - w = list_first_entry(&thread->todo, struct binder_work, entry); - else if (!list_empty(&proc->todo) && wait_for_proc_work) - w = list_first_entry(&proc->todo, struct binder_work, entry); - else { - if (ptr - buffer == 4 && !(thread->looper & BINDER_LOOPER_STATE_NEED_RETURN)) /* no data added */ + if (!list_empty(&thread->todo)) { + w = list_first_entry(&thread->todo, struct binder_work, + entry); + } else if (!list_empty(&proc->todo) && wait_for_proc_work) { + w = list_first_entry(&proc->todo, struct binder_work, + entry); + } else { + /* no data added */ + if (ptr - buffer == 4 && + !(thread->looper & BINDER_LOOPER_STATE_NEED_RETURN)) goto retry; break; } -- cgit v1.2.3 From a8363b44d11a212b1d44d7823fa1795de6887185 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 16 Oct 2014 14:40:38 +0200 Subject: staging: android: binder: move to the "real" part of the kernel The Android binder code has been "stable" for many years now. No matter what comes in the future, we are going to have to support this API, so might as well move it to the "real" part of the kernel as there's no real work that needs to be done to the existing code. Change-Id: I36d5c6fc05aff26dd01a227201be18e86c9f9994 Signed-off-by: Greg Kroah-Hartman --- drivers/Kconfig | 2 + drivers/Makefile | 1 + drivers/android/Kconfig | 37 + drivers/android/Makefile | 3 + drivers/android/binder.c | 3675 ++++++++++++++++++++++++++++++++ drivers/android/binder.h | 30 + drivers/android/binder_trace.h | 329 +++ drivers/staging/android/Kconfig | 23 - drivers/staging/android/Makefile | 1 - drivers/staging/android/binder.c | 3675 -------------------------------- drivers/staging/android/binder.h | 30 - drivers/staging/android/binder_trace.h | 329 --- drivers/staging/android/uapi/binder.h | 349 --- include/uapi/linux/Kbuild | 1 + include/uapi/linux/android/Kbuild | 2 + include/uapi/linux/android/binder.h | 349 +++ 16 files changed, 4429 insertions(+), 4407 deletions(-) create mode 100644 drivers/android/Kconfig create mode 100644 drivers/android/Makefile create mode 100644 drivers/android/binder.c create mode 100644 drivers/android/binder.h create mode 100644 drivers/android/binder_trace.h delete mode 100644 drivers/staging/android/binder.c delete mode 100644 drivers/staging/android/binder.h delete mode 100644 drivers/staging/android/binder_trace.h delete mode 100644 drivers/staging/android/uapi/binder.h create mode 100644 include/uapi/linux/android/Kbuild create mode 100644 include/uapi/linux/android/binder.h diff --git a/drivers/Kconfig b/drivers/Kconfig index e40c5e6d541e..f1a95063c257 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -168,4 +168,6 @@ source "drivers/ipack/Kconfig" source "drivers/reset/Kconfig" +source "drivers/android/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index 2bb844b79a69..fe5799aa5091 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -153,3 +153,4 @@ obj-$(CONFIG_IIO) += iio/ obj-$(CONFIG_VME_BUS) += vme/ obj-$(CONFIG_IPACK_BUS) += ipack/ obj-$(CONFIG_NTB) += ntb/ +obj-$(CONFIG_ANDROID) += android/ diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig new file mode 100644 index 000000000000..bdfc6c6f4f5a --- /dev/null +++ b/drivers/android/Kconfig @@ -0,0 +1,37 @@ +menu "Android" + +config ANDROID + bool "Android Drivers" + ---help--- + Enable support for various drivers needed on the Android platform + +if ANDROID + +config ANDROID_BINDER_IPC + bool "Android Binder IPC Driver" + depends on MMU + default n + ---help--- + Binder is used in Android for both communication between processes, + and remote method invocation. + + This means one Android process can call a method/routine in another + Android process, using Binder to identify, invoke and pass arguments + between said processes. + +config ANDROID_BINDER_IPC_32BIT + bool + depends on !64BIT && ANDROID_BINDER_IPC + default y + ---help--- + The Binder API has been changed to support both 32 and 64bit + applications in a mixed environment. + + Enable this to support an old 32-bit Android user-space (v4.4 and + earlier). + + Note that enabling this will break newer Android user-space. + +endif # if ANDROID + +endmenu diff --git a/drivers/android/Makefile b/drivers/android/Makefile new file mode 100644 index 000000000000..3b7e4b072c58 --- /dev/null +++ b/drivers/android/Makefile @@ -0,0 +1,3 @@ +ccflags-y += -I$(src) # needed for trace events + +obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o diff --git a/drivers/android/binder.c b/drivers/android/binder.c new file mode 100644 index 000000000000..b44b9398810c --- /dev/null +++ b/drivers/android/binder.c @@ -0,0 +1,3675 @@ +/* binder.c + * + * Android IPC Subsystem + * + * Copyright (C) 2007-2008 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "binder.h" +#include "binder_trace.h" + +static DEFINE_MUTEX(binder_main_lock); +static DEFINE_MUTEX(binder_deferred_lock); +static DEFINE_MUTEX(binder_mmap_lock); + +static HLIST_HEAD(binder_procs); +static HLIST_HEAD(binder_deferred_list); +static HLIST_HEAD(binder_dead_nodes); + +static struct dentry *binder_debugfs_dir_entry_root; +static struct dentry *binder_debugfs_dir_entry_proc; +static struct binder_node *binder_context_mgr_node; +static kuid_t binder_context_mgr_uid = INVALID_UID; +static int binder_last_id; +static struct workqueue_struct *binder_deferred_workqueue; + +#define BINDER_DEBUG_ENTRY(name) \ +static int binder_##name##_open(struct inode *inode, struct file *file) \ +{ \ + return single_open(file, binder_##name##_show, inode->i_private); \ +} \ +\ +static const struct file_operations binder_##name##_fops = { \ + .owner = THIS_MODULE, \ + .open = binder_##name##_open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = single_release, \ +} + +static int binder_proc_show(struct seq_file *m, void *unused); +BINDER_DEBUG_ENTRY(proc); + +/* This is only defined in include/asm-arm/sizes.h */ +#ifndef SZ_1K +#define SZ_1K 0x400 +#endif + +#ifndef SZ_4M +#define SZ_4M 0x400000 +#endif + +#define FORBIDDEN_MMAP_FLAGS (VM_WRITE) + +#define BINDER_SMALL_BUF_SIZE (PAGE_SIZE * 64) + +enum { + BINDER_DEBUG_USER_ERROR = 1U << 0, + BINDER_DEBUG_FAILED_TRANSACTION = 1U << 1, + BINDER_DEBUG_DEAD_TRANSACTION = 1U << 2, + BINDER_DEBUG_OPEN_CLOSE = 1U << 3, + BINDER_DEBUG_DEAD_BINDER = 1U << 4, + BINDER_DEBUG_DEATH_NOTIFICATION = 1U << 5, + BINDER_DEBUG_READ_WRITE = 1U << 6, + BINDER_DEBUG_USER_REFS = 1U << 7, + BINDER_DEBUG_THREADS = 1U << 8, + BINDER_DEBUG_TRANSACTION = 1U << 9, + BINDER_DEBUG_TRANSACTION_COMPLETE = 1U << 10, + BINDER_DEBUG_FREE_BUFFER = 1U << 11, + BINDER_DEBUG_INTERNAL_REFS = 1U << 12, + BINDER_DEBUG_BUFFER_ALLOC = 1U << 13, + BINDER_DEBUG_PRIORITY_CAP = 1U << 14, + BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 15, +}; +static uint32_t binder_debug_mask = BINDER_DEBUG_USER_ERROR | + BINDER_DEBUG_FAILED_TRANSACTION | BINDER_DEBUG_DEAD_TRANSACTION; +module_param_named(debug_mask, binder_debug_mask, uint, S_IWUSR | S_IRUGO); + +static bool binder_debug_no_lock; +module_param_named(proc_no_lock, binder_debug_no_lock, bool, S_IWUSR | S_IRUGO); + +static DECLARE_WAIT_QUEUE_HEAD(binder_user_error_wait); +static int binder_stop_on_user_error; + +static int binder_set_stop_on_user_error(const char *val, + struct kernel_param *kp) +{ + int ret; + + ret = param_set_int(val, kp); + if (binder_stop_on_user_error < 2) + wake_up(&binder_user_error_wait); + return ret; +} +module_param_call(stop_on_user_error, binder_set_stop_on_user_error, + param_get_int, &binder_stop_on_user_error, S_IWUSR | S_IRUGO); + +#define binder_debug(mask, x...) \ + do { \ + if (binder_debug_mask & mask) \ + pr_info(x); \ + } while (0) + +#define binder_user_error(x...) \ + do { \ + if (binder_debug_mask & BINDER_DEBUG_USER_ERROR) \ + pr_info(x); \ + if (binder_stop_on_user_error) \ + binder_stop_on_user_error = 2; \ + } while (0) + +enum binder_stat_types { + BINDER_STAT_PROC, + BINDER_STAT_THREAD, + BINDER_STAT_NODE, + BINDER_STAT_REF, + BINDER_STAT_DEATH, + BINDER_STAT_TRANSACTION, + BINDER_STAT_TRANSACTION_COMPLETE, + BINDER_STAT_COUNT +}; + +struct binder_stats { + int br[_IOC_NR(BR_FAILED_REPLY) + 1]; + int bc[_IOC_NR(BC_DEAD_BINDER_DONE) + 1]; + int obj_created[BINDER_STAT_COUNT]; + int obj_deleted[BINDER_STAT_COUNT]; +}; + +static struct binder_stats binder_stats; + +static inline void binder_stats_deleted(enum binder_stat_types type) +{ + binder_stats.obj_deleted[type]++; +} + +static inline void binder_stats_created(enum binder_stat_types type) +{ + binder_stats.obj_created[type]++; +} + +struct binder_transaction_log_entry { + int debug_id; + int call_type; + int from_proc; + int from_thread; + int target_handle; + int to_proc; + int to_thread; + int to_node; + int data_size; + int offsets_size; +}; +struct binder_transaction_log { + int next; + int full; + struct binder_transaction_log_entry entry[32]; +}; +static struct binder_transaction_log binder_transaction_log; +static struct binder_transaction_log binder_transaction_log_failed; + +static struct binder_transaction_log_entry *binder_transaction_log_add( + struct binder_transaction_log *log) +{ + struct binder_transaction_log_entry *e; + + e = &log->entry[log->next]; + memset(e, 0, sizeof(*e)); + log->next++; + if (log->next == ARRAY_SIZE(log->entry)) { + log->next = 0; + log->full = 1; + } + return e; +} + +struct binder_work { + struct list_head entry; + enum { + BINDER_WORK_TRANSACTION = 1, + BINDER_WORK_TRANSACTION_COMPLETE, + BINDER_WORK_NODE, + BINDER_WORK_DEAD_BINDER, + BINDER_WORK_DEAD_BINDER_AND_CLEAR, + BINDER_WORK_CLEAR_DEATH_NOTIFICATION, + } type; +}; + +struct binder_node { + int debug_id; + struct binder_work work; + union { + struct rb_node rb_node; + struct hlist_node dead_node; + }; + struct binder_proc *proc; + struct hlist_head refs; + int internal_strong_refs; + int local_weak_refs; + int local_strong_refs; + binder_uintptr_t ptr; + binder_uintptr_t cookie; + unsigned has_strong_ref:1; + unsigned pending_strong_ref:1; + unsigned has_weak_ref:1; + unsigned pending_weak_ref:1; + unsigned has_async_transaction:1; + unsigned accept_fds:1; + unsigned min_priority:8; + struct list_head async_todo; +}; + +struct binder_ref_death { + struct binder_work work; + binder_uintptr_t cookie; +}; + +struct binder_ref { + /* Lookups needed: */ + /* node + proc => ref (transaction) */ + /* desc + proc => ref (transaction, inc/dec ref) */ + /* node => refs + procs (proc exit) */ + int debug_id; + struct rb_node rb_node_desc; + struct rb_node rb_node_node; + struct hlist_node node_entry; + struct binder_proc *proc; + struct binder_node *node; + uint32_t desc; + int strong; + int weak; + struct binder_ref_death *death; +}; + +struct binder_buffer { + struct list_head entry; /* free and allocated entries by address */ + struct rb_node rb_node; /* free entry by size or allocated entry */ + /* by address */ + unsigned free:1; + unsigned allow_user_free:1; + unsigned async_transaction:1; + unsigned debug_id:29; + + struct binder_transaction *transaction; + + struct binder_node *target_node; + size_t data_size; + size_t offsets_size; + uint8_t data[0]; +}; + +enum binder_deferred_state { + BINDER_DEFERRED_PUT_FILES = 0x01, + BINDER_DEFERRED_FLUSH = 0x02, + BINDER_DEFERRED_RELEASE = 0x04, +}; + +struct binder_proc { + struct hlist_node proc_node; + struct rb_root threads; + struct rb_root nodes; + struct rb_root refs_by_desc; + struct rb_root refs_by_node; + int pid; + struct vm_area_struct *vma; + struct mm_struct *vma_vm_mm; + struct task_struct *tsk; + struct files_struct *files; + struct hlist_node deferred_work_node; + int deferred_work; + void *buffer; + ptrdiff_t user_buffer_offset; + + struct list_head buffers; + struct rb_root free_buffers; + struct rb_root allocated_buffers; + size_t free_async_space; + + struct page **pages; + size_t buffer_size; + uint32_t buffer_free; + struct list_head todo; + wait_queue_head_t wait; + struct binder_stats stats; + struct list_head delivered_death; + int max_threads; + int requested_threads; + int requested_threads_started; + int ready_threads; + long default_priority; + struct dentry *debugfs_entry; +}; + +enum { + BINDER_LOOPER_STATE_REGISTERED = 0x01, + BINDER_LOOPER_STATE_ENTERED = 0x02, + BINDER_LOOPER_STATE_EXITED = 0x04, + BINDER_LOOPER_STATE_INVALID = 0x08, + BINDER_LOOPER_STATE_WAITING = 0x10, + BINDER_LOOPER_STATE_NEED_RETURN = 0x20 +}; + +struct binder_thread { + struct binder_proc *proc; + struct rb_node rb_node; + int pid; + int looper; + struct binder_transaction *transaction_stack; + struct list_head todo; + uint32_t return_error; /* Write failed, return error code in read buf */ + uint32_t return_error2; /* Write failed, return error code in read */ + /* buffer. Used when sending a reply to a dead process that */ + /* we are also waiting on */ + wait_queue_head_t wait; + struct binder_stats stats; +}; + +struct binder_transaction { + int debug_id; + struct binder_work work; + struct binder_thread *from; + struct binder_transaction *from_parent; + struct binder_proc *to_proc; + struct binder_thread *to_thread; + struct binder_transaction *to_parent; + unsigned need_reply:1; + /* unsigned is_dead:1; */ /* not used at the moment */ + + struct binder_buffer *buffer; + unsigned int code; + unsigned int flags; + long priority; + long saved_priority; + kuid_t sender_euid; +}; + +static void +binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer); + +static int task_get_unused_fd_flags(struct binder_proc *proc, int flags) +{ + struct files_struct *files = proc->files; + unsigned long rlim_cur; + unsigned long irqs; + + if (files == NULL) + return -ESRCH; + + if (!lock_task_sighand(proc->tsk, &irqs)) + return -EMFILE; + + rlim_cur = task_rlimit(proc->tsk, RLIMIT_NOFILE); + unlock_task_sighand(proc->tsk, &irqs); + + return __alloc_fd(files, 0, rlim_cur, flags); +} + +/* + * copied from fd_install + */ +static void task_fd_install( + struct binder_proc *proc, unsigned int fd, struct file *file) +{ + if (proc->files) + __fd_install(proc->files, fd, file); +} + +/* + * copied from sys_close + */ +static long task_close_fd(struct binder_proc *proc, unsigned int fd) +{ + int retval; + + if (proc->files == NULL) + return -ESRCH; + + retval = __close_fd(proc->files, fd); + /* can't restart close syscall because file table entry was cleared */ + if (unlikely(retval == -ERESTARTSYS || + retval == -ERESTARTNOINTR || + retval == -ERESTARTNOHAND || + retval == -ERESTART_RESTARTBLOCK)) + retval = -EINTR; + + return retval; +} + +static inline void binder_lock(const char *tag) +{ + trace_binder_lock(tag); + mutex_lock(&binder_main_lock); + trace_binder_locked(tag); +} + +static inline void binder_unlock(const char *tag) +{ + trace_binder_unlock(tag); + mutex_unlock(&binder_main_lock); +} + +static void binder_set_nice(long nice) +{ + long min_nice; + + if (can_nice(current, nice)) { + set_user_nice(current, nice); + return; + } + min_nice = 20 - current->signal->rlim[RLIMIT_NICE].rlim_cur; + binder_debug(BINDER_DEBUG_PRIORITY_CAP, + "%d: nice value %ld not allowed use %ld instead\n", + current->pid, nice, min_nice); + set_user_nice(current, min_nice); + if (min_nice < 20) + return; + binder_user_error("%d RLIMIT_NICE not set\n", current->pid); +} + +static size_t binder_buffer_size(struct binder_proc *proc, + struct binder_buffer *buffer) +{ + if (list_is_last(&buffer->entry, &proc->buffers)) + return proc->buffer + proc->buffer_size - (void *)buffer->data; + return (size_t)list_entry(buffer->entry.next, + struct binder_buffer, entry) - (size_t)buffer->data; +} + +static void binder_insert_free_buffer(struct binder_proc *proc, + struct binder_buffer *new_buffer) +{ + struct rb_node **p = &proc->free_buffers.rb_node; + struct rb_node *parent = NULL; + struct binder_buffer *buffer; + size_t buffer_size; + size_t new_buffer_size; + + BUG_ON(!new_buffer->free); + + new_buffer_size = binder_buffer_size(proc, new_buffer); + + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: add free buffer, size %zd, at %p\n", + proc->pid, new_buffer_size, new_buffer); + + while (*p) { + parent = *p; + buffer = rb_entry(parent, struct binder_buffer, rb_node); + BUG_ON(!buffer->free); + + buffer_size = binder_buffer_size(proc, buffer); + + if (new_buffer_size < buffer_size) + p = &parent->rb_left; + else + p = &parent->rb_right; + } + rb_link_node(&new_buffer->rb_node, parent, p); + rb_insert_color(&new_buffer->rb_node, &proc->free_buffers); +} + +static void binder_insert_allocated_buffer(struct binder_proc *proc, + struct binder_buffer *new_buffer) +{ + struct rb_node **p = &proc->allocated_buffers.rb_node; + struct rb_node *parent = NULL; + struct binder_buffer *buffer; + + BUG_ON(new_buffer->free); + + while (*p) { + parent = *p; + buffer = rb_entry(parent, struct binder_buffer, rb_node); + BUG_ON(buffer->free); + + if (new_buffer < buffer) + p = &parent->rb_left; + else if (new_buffer > buffer) + p = &parent->rb_right; + else + BUG(); + } + rb_link_node(&new_buffer->rb_node, parent, p); + rb_insert_color(&new_buffer->rb_node, &proc->allocated_buffers); +} + +static struct binder_buffer *binder_buffer_lookup(struct binder_proc *proc, + uintptr_t user_ptr) +{ + struct rb_node *n = proc->allocated_buffers.rb_node; + struct binder_buffer *buffer; + struct binder_buffer *kern_ptr; + + kern_ptr = (struct binder_buffer *)(user_ptr - proc->user_buffer_offset + - offsetof(struct binder_buffer, data)); + + while (n) { + buffer = rb_entry(n, struct binder_buffer, rb_node); + BUG_ON(buffer->free); + + if (kern_ptr < buffer) + n = n->rb_left; + else if (kern_ptr > buffer) + n = n->rb_right; + else + return buffer; + } + return NULL; +} + +static int binder_update_page_range(struct binder_proc *proc, int allocate, + void *start, void *end, + struct vm_area_struct *vma) +{ + void *page_addr; + unsigned long user_page_addr; + struct vm_struct tmp_area; + struct page **page; + struct mm_struct *mm; + + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: %s pages %p-%p\n", proc->pid, + allocate ? "allocate" : "free", start, end); + + if (end <= start) + return 0; + + trace_binder_update_page_range(proc, allocate, start, end); + + if (vma) + mm = NULL; + else + mm = get_task_mm(proc->tsk); + + if (mm) { + down_write(&mm->mmap_sem); + vma = proc->vma; + if (vma && mm != proc->vma_vm_mm) { + pr_err("%d: vma mm and task mm mismatch\n", + proc->pid); + vma = NULL; + } + } + + if (allocate == 0) + goto free_range; + + if (vma == NULL) { + pr_err("%d: binder_alloc_buf failed to map pages in userspace, no vma\n", + proc->pid); + goto err_no_vma; + } + + for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { + int ret; + struct page **page_array_ptr; + + page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE]; + + BUG_ON(*page); + *page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); + if (*page == NULL) { + pr_err("%d: binder_alloc_buf failed for page at %p\n", + proc->pid, page_addr); + goto err_alloc_page_failed; + } + tmp_area.addr = page_addr; + tmp_area.size = PAGE_SIZE + PAGE_SIZE /* guard page? */; + page_array_ptr = page; + ret = map_vm_area(&tmp_area, PAGE_KERNEL, &page_array_ptr); + if (ret) { + pr_err("%d: binder_alloc_buf failed to map page at %p in kernel\n", + proc->pid, page_addr); + goto err_map_kernel_failed; + } + user_page_addr = + (uintptr_t)page_addr + proc->user_buffer_offset; + ret = vm_insert_page(vma, user_page_addr, page[0]); + if (ret) { + pr_err("%d: binder_alloc_buf failed to map page at %lx in userspace\n", + proc->pid, user_page_addr); + goto err_vm_insert_page_failed; + } + /* vm_insert_page does not seem to increment the refcount */ + } + if (mm) { + up_write(&mm->mmap_sem); + mmput(mm); + } + return 0; + +free_range: + for (page_addr = end - PAGE_SIZE; page_addr >= start; + page_addr -= PAGE_SIZE) { + page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE]; + if (vma) + zap_page_range(vma, (uintptr_t)page_addr + + proc->user_buffer_offset, PAGE_SIZE, NULL); +err_vm_insert_page_failed: + unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE); +err_map_kernel_failed: + __free_page(*page); + *page = NULL; +err_alloc_page_failed: + ; + } +err_no_vma: + if (mm) { + up_write(&mm->mmap_sem); + mmput(mm); + } + return -ENOMEM; +} + +static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, + size_t data_size, + size_t offsets_size, int is_async) +{ + struct rb_node *n = proc->free_buffers.rb_node; + struct binder_buffer *buffer; + size_t buffer_size; + struct rb_node *best_fit = NULL; + void *has_page_addr; + void *end_page_addr; + size_t size; + + if (proc->vma == NULL) { + pr_err("%d: binder_alloc_buf, no vma\n", + proc->pid); + return NULL; + } + + size = ALIGN(data_size, sizeof(void *)) + + ALIGN(offsets_size, sizeof(void *)); + + if (size < data_size || size < offsets_size) { + binder_user_error("%d: got transaction with invalid size %zd-%zd\n", + proc->pid, data_size, offsets_size); + return NULL; + } + + if (is_async && + proc->free_async_space < size + sizeof(struct binder_buffer)) { + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: binder_alloc_buf size %zd failed, no async space left\n", + proc->pid, size); + return NULL; + } + + while (n) { + buffer = rb_entry(n, struct binder_buffer, rb_node); + BUG_ON(!buffer->free); + buffer_size = binder_buffer_size(proc, buffer); + + if (size < buffer_size) { + best_fit = n; + n = n->rb_left; + } else if (size > buffer_size) + n = n->rb_right; + else { + best_fit = n; + break; + } + } + if (best_fit == NULL) { + pr_err("%d: binder_alloc_buf size %zd failed, no address space\n", + proc->pid, size); + return NULL; + } + if (n == NULL) { + buffer = rb_entry(best_fit, struct binder_buffer, rb_node); + buffer_size = binder_buffer_size(proc, buffer); + } + + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: binder_alloc_buf size %zd got buffer %p size %zd\n", + proc->pid, size, buffer, buffer_size); + + has_page_addr = + (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK); + if (n == NULL) { + if (size + sizeof(struct binder_buffer) + 4 >= buffer_size) + buffer_size = size; /* no room for other buffers */ + else + buffer_size = size + sizeof(struct binder_buffer); + } + end_page_addr = + (void *)PAGE_ALIGN((uintptr_t)buffer->data + buffer_size); + if (end_page_addr > has_page_addr) + end_page_addr = has_page_addr; + if (binder_update_page_range(proc, 1, + (void *)PAGE_ALIGN((uintptr_t)buffer->data), end_page_addr, NULL)) + return NULL; + + rb_erase(best_fit, &proc->free_buffers); + buffer->free = 0; + binder_insert_allocated_buffer(proc, buffer); + if (buffer_size != size) { + struct binder_buffer *new_buffer = (void *)buffer->data + size; + + list_add(&new_buffer->entry, &buffer->entry); + new_buffer->free = 1; + binder_insert_free_buffer(proc, new_buffer); + } + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: binder_alloc_buf size %zd got %p\n", + proc->pid, size, buffer); + buffer->data_size = data_size; + buffer->offsets_size = offsets_size; + buffer->async_transaction = is_async; + if (is_async) { + proc->free_async_space -= size + sizeof(struct binder_buffer); + binder_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, + "%d: binder_alloc_buf size %zd async free %zd\n", + proc->pid, size, proc->free_async_space); + } + + return buffer; +} + +static void *buffer_start_page(struct binder_buffer *buffer) +{ + return (void *)((uintptr_t)buffer & PAGE_MASK); +} + +static void *buffer_end_page(struct binder_buffer *buffer) +{ + return (void *)(((uintptr_t)(buffer + 1) - 1) & PAGE_MASK); +} + +static void binder_delete_free_buffer(struct binder_proc *proc, + struct binder_buffer *buffer) +{ + struct binder_buffer *prev, *next = NULL; + int free_page_end = 1; + int free_page_start = 1; + + BUG_ON(proc->buffers.next == &buffer->entry); + prev = list_entry(buffer->entry.prev, struct binder_buffer, entry); + BUG_ON(!prev->free); + if (buffer_end_page(prev) == buffer_start_page(buffer)) { + free_page_start = 0; + if (buffer_end_page(prev) == buffer_end_page(buffer)) + free_page_end = 0; + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: merge free, buffer %p share page with %p\n", + proc->pid, buffer, prev); + } + + if (!list_is_last(&buffer->entry, &proc->buffers)) { + next = list_entry(buffer->entry.next, + struct binder_buffer, entry); + if (buffer_start_page(next) == buffer_end_page(buffer)) { + free_page_end = 0; + if (buffer_start_page(next) == + buffer_start_page(buffer)) + free_page_start = 0; + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: merge free, buffer %p share page with %p\n", + proc->pid, buffer, prev); + } + } + list_del(&buffer->entry); + if (free_page_start || free_page_end) { + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: merge free, buffer %p do not share page%s%s with %p or %p\n", + proc->pid, buffer, free_page_start ? "" : " end", + free_page_end ? "" : " start", prev, next); + binder_update_page_range(proc, 0, free_page_start ? + buffer_start_page(buffer) : buffer_end_page(buffer), + (free_page_end ? buffer_end_page(buffer) : + buffer_start_page(buffer)) + PAGE_SIZE, NULL); + } +} + +static void binder_free_buf(struct binder_proc *proc, + struct binder_buffer *buffer) +{ + size_t size, buffer_size; + + buffer_size = binder_buffer_size(proc, buffer); + + size = ALIGN(buffer->data_size, sizeof(void *)) + + ALIGN(buffer->offsets_size, sizeof(void *)); + + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%d: binder_free_buf %p size %zd buffer_size %zd\n", + proc->pid, buffer, size, buffer_size); + + BUG_ON(buffer->free); + BUG_ON(size > buffer_size); + BUG_ON(buffer->transaction != NULL); + BUG_ON((void *)buffer < proc->buffer); + BUG_ON((void *)buffer > proc->buffer + proc->buffer_size); + + if (buffer->async_transaction) { + proc->free_async_space += size + sizeof(struct binder_buffer); + + binder_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, + "%d: binder_free_buf size %zd async free %zd\n", + proc->pid, size, proc->free_async_space); + } + + binder_update_page_range(proc, 0, + (void *)PAGE_ALIGN((uintptr_t)buffer->data), + (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK), + NULL); + rb_erase(&buffer->rb_node, &proc->allocated_buffers); + buffer->free = 1; + if (!list_is_last(&buffer->entry, &proc->buffers)) { + struct binder_buffer *next = list_entry(buffer->entry.next, + struct binder_buffer, entry); + + if (next->free) { + rb_erase(&next->rb_node, &proc->free_buffers); + binder_delete_free_buffer(proc, next); + } + } + if (proc->buffers.next != &buffer->entry) { + struct binder_buffer *prev = list_entry(buffer->entry.prev, + struct binder_buffer, entry); + + if (prev->free) { + binder_delete_free_buffer(proc, buffer); + rb_erase(&prev->rb_node, &proc->free_buffers); + buffer = prev; + } + } + binder_insert_free_buffer(proc, buffer); +} + +static struct binder_node *binder_get_node(struct binder_proc *proc, + binder_uintptr_t ptr) +{ + struct rb_node *n = proc->nodes.rb_node; + struct binder_node *node; + + while (n) { + node = rb_entry(n, struct binder_node, rb_node); + + if (ptr < node->ptr) + n = n->rb_left; + else if (ptr > node->ptr) + n = n->rb_right; + else + return node; + } + return NULL; +} + +static struct binder_node *binder_new_node(struct binder_proc *proc, + binder_uintptr_t ptr, + binder_uintptr_t cookie) +{ + struct rb_node **p = &proc->nodes.rb_node; + struct rb_node *parent = NULL; + struct binder_node *node; + + while (*p) { + parent = *p; + node = rb_entry(parent, struct binder_node, rb_node); + + if (ptr < node->ptr) + p = &(*p)->rb_left; + else if (ptr > node->ptr) + p = &(*p)->rb_right; + else + return NULL; + } + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (node == NULL) + return NULL; + binder_stats_created(BINDER_STAT_NODE); + rb_link_node(&node->rb_node, parent, p); + rb_insert_color(&node->rb_node, &proc->nodes); + node->debug_id = ++binder_last_id; + node->proc = proc; + node->ptr = ptr; + node->cookie = cookie; + node->work.type = BINDER_WORK_NODE; + INIT_LIST_HEAD(&node->work.entry); + INIT_LIST_HEAD(&node->async_todo); + binder_debug(BINDER_DEBUG_INTERNAL_REFS, + "%d:%d node %d u%016llx c%016llx created\n", + proc->pid, current->pid, node->debug_id, + (u64)node->ptr, (u64)node->cookie); + return node; +} + +static int binder_inc_node(struct binder_node *node, int strong, int internal, + struct list_head *target_list) +{ + if (strong) { + if (internal) { + if (target_list == NULL && + node->internal_strong_refs == 0 && + !(node == binder_context_mgr_node && + node->has_strong_ref)) { + pr_err("invalid inc strong node for %d\n", + node->debug_id); + return -EINVAL; + } + node->internal_strong_refs++; + } else + node->local_strong_refs++; + if (!node->has_strong_ref && target_list) { + list_del_init(&node->work.entry); + list_add_tail(&node->work.entry, target_list); + } + } else { + if (!internal) + node->local_weak_refs++; + if (!node->has_weak_ref && list_empty(&node->work.entry)) { + if (target_list == NULL) { + pr_err("invalid inc weak node for %d\n", + node->debug_id); + return -EINVAL; + } + list_add_tail(&node->work.entry, target_list); + } + } + return 0; +} + +static int binder_dec_node(struct binder_node *node, int strong, int internal) +{ + if (strong) { + if (internal) + node->internal_strong_refs--; + else + node->local_strong_refs--; + if (node->local_strong_refs || node->internal_strong_refs) + return 0; + } else { + if (!internal) + node->local_weak_refs--; + if (node->local_weak_refs || !hlist_empty(&node->refs)) + return 0; + } + if (node->proc && (node->has_strong_ref || node->has_weak_ref)) { + if (list_empty(&node->work.entry)) { + list_add_tail(&node->work.entry, &node->proc->todo); + wake_up_interruptible(&node->proc->wait); + } + } else { + if (hlist_empty(&node->refs) && !node->local_strong_refs && + !node->local_weak_refs) { + list_del_init(&node->work.entry); + if (node->proc) { + rb_erase(&node->rb_node, &node->proc->nodes); + binder_debug(BINDER_DEBUG_INTERNAL_REFS, + "refless node %d deleted\n", + node->debug_id); + } else { + hlist_del(&node->dead_node); + binder_debug(BINDER_DEBUG_INTERNAL_REFS, + "dead node %d deleted\n", + node->debug_id); + } + kfree(node); + binder_stats_deleted(BINDER_STAT_NODE); + } + } + + return 0; +} + + +static struct binder_ref *binder_get_ref(struct binder_proc *proc, + uint32_t desc) +{ + struct rb_node *n = proc->refs_by_desc.rb_node; + struct binder_ref *ref; + + while (n) { + ref = rb_entry(n, struct binder_ref, rb_node_desc); + + if (desc < ref->desc) + n = n->rb_left; + else if (desc > ref->desc) + n = n->rb_right; + else + return ref; + } + return NULL; +} + +static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc, + struct binder_node *node) +{ + struct rb_node *n; + struct rb_node **p = &proc->refs_by_node.rb_node; + struct rb_node *parent = NULL; + struct binder_ref *ref, *new_ref; + + while (*p) { + parent = *p; + ref = rb_entry(parent, struct binder_ref, rb_node_node); + + if (node < ref->node) + p = &(*p)->rb_left; + else if (node > ref->node) + p = &(*p)->rb_right; + else + return ref; + } + new_ref = kzalloc(sizeof(*ref), GFP_KERNEL); + if (new_ref == NULL) + return NULL; + binder_stats_created(BINDER_STAT_REF); + new_ref->debug_id = ++binder_last_id; + new_ref->proc = proc; + new_ref->node = node; + rb_link_node(&new_ref->rb_node_node, parent, p); + rb_insert_color(&new_ref->rb_node_node, &proc->refs_by_node); + + new_ref->desc = (node == binder_context_mgr_node) ? 0 : 1; + for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) { + ref = rb_entry(n, struct binder_ref, rb_node_desc); + if (ref->desc > new_ref->desc) + break; + new_ref->desc = ref->desc + 1; + } + + p = &proc->refs_by_desc.rb_node; + while (*p) { + parent = *p; + ref = rb_entry(parent, struct binder_ref, rb_node_desc); + + if (new_ref->desc < ref->desc) + p = &(*p)->rb_left; + else if (new_ref->desc > ref->desc) + p = &(*p)->rb_right; + else + BUG(); + } + rb_link_node(&new_ref->rb_node_desc, parent, p); + rb_insert_color(&new_ref->rb_node_desc, &proc->refs_by_desc); + if (node) { + hlist_add_head(&new_ref->node_entry, &node->refs); + + binder_debug(BINDER_DEBUG_INTERNAL_REFS, + "%d new ref %d desc %d for node %d\n", + proc->pid, new_ref->debug_id, new_ref->desc, + node->debug_id); + } else { + binder_debug(BINDER_DEBUG_INTERNAL_REFS, + "%d new ref %d desc %d for dead node\n", + proc->pid, new_ref->debug_id, new_ref->desc); + } + return new_ref; +} + +static void binder_delete_ref(struct binder_ref *ref) +{ + binder_debug(BINDER_DEBUG_INTERNAL_REFS, + "%d delete ref %d desc %d for node %d\n", + ref->proc->pid, ref->debug_id, ref->desc, + ref->node->debug_id); + + rb_erase(&ref->rb_node_desc, &ref->proc->refs_by_desc); + rb_erase(&ref->rb_node_node, &ref->proc->refs_by_node); + if (ref->strong) + binder_dec_node(ref->node, 1, 1); + hlist_del(&ref->node_entry); + binder_dec_node(ref->node, 0, 1); + if (ref->death) { + binder_debug(BINDER_DEBUG_DEAD_BINDER, + "%d delete ref %d desc %d has death notification\n", + ref->proc->pid, ref->debug_id, ref->desc); + list_del(&ref->death->work.entry); + kfree(ref->death); + binder_stats_deleted(BINDER_STAT_DEATH); + } + kfree(ref); + binder_stats_deleted(BINDER_STAT_REF); +} + +static int binder_inc_ref(struct binder_ref *ref, int strong, + struct list_head *target_list) +{ + int ret; + + if (strong) { + if (ref->strong == 0) { + ret = binder_inc_node(ref->node, 1, 1, target_list); + if (ret) + return ret; + } + ref->strong++; + } else { + if (ref->weak == 0) { + ret = binder_inc_node(ref->node, 0, 1, target_list); + if (ret) + return ret; + } + ref->weak++; + } + return 0; +} + + +static int binder_dec_ref(struct binder_ref *ref, int strong) +{ + if (strong) { + if (ref->strong == 0) { + binder_user_error("%d invalid dec strong, ref %d desc %d s %d w %d\n", + ref->proc->pid, ref->debug_id, + ref->desc, ref->strong, ref->weak); + return -EINVAL; + } + ref->strong--; + if (ref->strong == 0) { + int ret; + + ret = binder_dec_node(ref->node, strong, 1); + if (ret) + return ret; + } + } else { + if (ref->weak == 0) { + binder_user_error("%d invalid dec weak, ref %d desc %d s %d w %d\n", + ref->proc->pid, ref->debug_id, + ref->desc, ref->strong, ref->weak); + return -EINVAL; + } + ref->weak--; + } + if (ref->strong == 0 && ref->weak == 0) + binder_delete_ref(ref); + return 0; +} + +static void binder_pop_transaction(struct binder_thread *target_thread, + struct binder_transaction *t) +{ + if (target_thread) { + BUG_ON(target_thread->transaction_stack != t); + BUG_ON(target_thread->transaction_stack->from != target_thread); + target_thread->transaction_stack = + target_thread->transaction_stack->from_parent; + t->from = NULL; + } + t->need_reply = 0; + if (t->buffer) + t->buffer->transaction = NULL; + kfree(t); + binder_stats_deleted(BINDER_STAT_TRANSACTION); +} + +static void binder_send_failed_reply(struct binder_transaction *t, + uint32_t error_code) +{ + struct binder_thread *target_thread; + struct binder_transaction *next; + + BUG_ON(t->flags & TF_ONE_WAY); + while (1) { + target_thread = t->from; + if (target_thread) { + if (target_thread->return_error != BR_OK && + target_thread->return_error2 == BR_OK) { + target_thread->return_error2 = + target_thread->return_error; + target_thread->return_error = BR_OK; + } + if (target_thread->return_error == BR_OK) { + binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, + "send failed reply for transaction %d to %d:%d\n", + t->debug_id, + target_thread->proc->pid, + target_thread->pid); + + binder_pop_transaction(target_thread, t); + target_thread->return_error = error_code; + wake_up_interruptible(&target_thread->wait); + } else { + pr_err("reply failed, target thread, %d:%d, has error code %d already\n", + target_thread->proc->pid, + target_thread->pid, + target_thread->return_error); + } + return; + } + next = t->from_parent; + + binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, + "send failed reply for transaction %d, target dead\n", + t->debug_id); + + binder_pop_transaction(target_thread, t); + if (next == NULL) { + binder_debug(BINDER_DEBUG_DEAD_BINDER, + "reply failed, no target thread at root\n"); + return; + } + t = next; + binder_debug(BINDER_DEBUG_DEAD_BINDER, + "reply failed, no target thread -- retry %d\n", + t->debug_id); + } +} + +static void binder_transaction_buffer_release(struct binder_proc *proc, + struct binder_buffer *buffer, + binder_size_t *failed_at) +{ + binder_size_t *offp, *off_end; + int debug_id = buffer->debug_id; + + binder_debug(BINDER_DEBUG_TRANSACTION, + "%d buffer release %d, size %zd-%zd, failed at %p\n", + proc->pid, buffer->debug_id, + buffer->data_size, buffer->offsets_size, failed_at); + + if (buffer->target_node) + binder_dec_node(buffer->target_node, 1, 0); + + offp = (binder_size_t *)(buffer->data + + ALIGN(buffer->data_size, sizeof(void *))); + if (failed_at) + off_end = failed_at; + else + off_end = (void *)offp + buffer->offsets_size; + for (; offp < off_end; offp++) { + struct flat_binder_object *fp; + + if (*offp > buffer->data_size - sizeof(*fp) || + buffer->data_size < sizeof(*fp) || + !IS_ALIGNED(*offp, sizeof(u32))) { + pr_err("transaction release %d bad offset %lld, size %zd\n", + debug_id, (u64)*offp, buffer->data_size); + continue; + } + fp = (struct flat_binder_object *)(buffer->data + *offp); + switch (fp->type) { + case BINDER_TYPE_BINDER: + case BINDER_TYPE_WEAK_BINDER: { + struct binder_node *node = binder_get_node(proc, fp->binder); + + if (node == NULL) { + pr_err("transaction release %d bad node %016llx\n", + debug_id, (u64)fp->binder); + break; + } + binder_debug(BINDER_DEBUG_TRANSACTION, + " node %d u%016llx\n", + node->debug_id, (u64)node->ptr); + binder_dec_node(node, fp->type == BINDER_TYPE_BINDER, 0); + } break; + case BINDER_TYPE_HANDLE: + case BINDER_TYPE_WEAK_HANDLE: { + struct binder_ref *ref = binder_get_ref(proc, fp->handle); + + if (ref == NULL) { + pr_err("transaction release %d bad handle %d\n", + debug_id, fp->handle); + break; + } + binder_debug(BINDER_DEBUG_TRANSACTION, + " ref %d desc %d (node %d)\n", + ref->debug_id, ref->desc, ref->node->debug_id); + binder_dec_ref(ref, fp->type == BINDER_TYPE_HANDLE); + } break; + + case BINDER_TYPE_FD: + binder_debug(BINDER_DEBUG_TRANSACTION, + " fd %d\n", fp->handle); + if (failed_at) + task_close_fd(proc, fp->handle); + break; + + default: + pr_err("transaction release %d bad object type %x\n", + debug_id, fp->type); + break; + } + } +} + +static void binder_transaction(struct binder_proc *proc, + struct binder_thread *thread, + struct binder_transaction_data *tr, int reply) +{ + struct binder_transaction *t; + struct binder_work *tcomplete; + binder_size_t *offp, *off_end; + struct binder_proc *target_proc; + struct binder_thread *target_thread = NULL; + struct binder_node *target_node = NULL; + struct list_head *target_list; + wait_queue_head_t *target_wait; + struct binder_transaction *in_reply_to = NULL; + struct binder_transaction_log_entry *e; + uint32_t return_error; + + e = binder_transaction_log_add(&binder_transaction_log); + e->call_type = reply ? 2 : !!(tr->flags & TF_ONE_WAY); + e->from_proc = proc->pid; + e->from_thread = thread->pid; + e->target_handle = tr->target.handle; + e->data_size = tr->data_size; + e->offsets_size = tr->offsets_size; + + if (reply) { + in_reply_to = thread->transaction_stack; + if (in_reply_to == NULL) { + binder_user_error("%d:%d got reply transaction with no transaction stack\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + goto err_empty_call_stack; + } + binder_set_nice(in_reply_to->saved_priority); + if (in_reply_to->to_thread != thread) { + binder_user_error("%d:%d got reply transaction with bad transaction stack, transaction %d has target %d:%d\n", + proc->pid, thread->pid, in_reply_to->debug_id, + in_reply_to->to_proc ? + in_reply_to->to_proc->pid : 0, + in_reply_to->to_thread ? + in_reply_to->to_thread->pid : 0); + return_error = BR_FAILED_REPLY; + in_reply_to = NULL; + goto err_bad_call_stack; + } + thread->transaction_stack = in_reply_to->to_parent; + target_thread = in_reply_to->from; + if (target_thread == NULL) { + return_error = BR_DEAD_REPLY; + goto err_dead_binder; + } + if (target_thread->transaction_stack != in_reply_to) { + binder_user_error("%d:%d got reply transaction with bad target transaction stack %d, expected %d\n", + proc->pid, thread->pid, + target_thread->transaction_stack ? + target_thread->transaction_stack->debug_id : 0, + in_reply_to->debug_id); + return_error = BR_FAILED_REPLY; + in_reply_to = NULL; + target_thread = NULL; + goto err_dead_binder; + } + target_proc = target_thread->proc; + } else { + if (tr->target.handle) { + struct binder_ref *ref; + + ref = binder_get_ref(proc, tr->target.handle); + if (ref == NULL) { + binder_user_error("%d:%d got transaction to invalid handle\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + goto err_invalid_target_handle; + } + target_node = ref->node; + } else { + target_node = binder_context_mgr_node; + if (target_node == NULL) { + return_error = BR_DEAD_REPLY; + goto err_no_context_mgr_node; + } + } + e->to_node = target_node->debug_id; + target_proc = target_node->proc; + if (target_proc == NULL) { + return_error = BR_DEAD_REPLY; + goto err_dead_binder; + } + if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) { + struct binder_transaction *tmp; + + tmp = thread->transaction_stack; + if (tmp->to_thread != thread) { + binder_user_error("%d:%d got new transaction with bad transaction stack, transaction %d has target %d:%d\n", + proc->pid, thread->pid, tmp->debug_id, + tmp->to_proc ? tmp->to_proc->pid : 0, + tmp->to_thread ? + tmp->to_thread->pid : 0); + return_error = BR_FAILED_REPLY; + goto err_bad_call_stack; + } + while (tmp) { + if (tmp->from && tmp->from->proc == target_proc) + target_thread = tmp->from; + tmp = tmp->from_parent; + } + } + } + if (target_thread) { + e->to_thread = target_thread->pid; + target_list = &target_thread->todo; + target_wait = &target_thread->wait; + } else { + target_list = &target_proc->todo; + target_wait = &target_proc->wait; + } + e->to_proc = target_proc->pid; + + /* TODO: reuse incoming transaction for reply */ + t = kzalloc(sizeof(*t), GFP_KERNEL); + if (t == NULL) { + return_error = BR_FAILED_REPLY; + goto err_alloc_t_failed; + } + binder_stats_created(BINDER_STAT_TRANSACTION); + + tcomplete = kzalloc(sizeof(*tcomplete), GFP_KERNEL); + if (tcomplete == NULL) { + return_error = BR_FAILED_REPLY; + goto err_alloc_tcomplete_failed; + } + binder_stats_created(BINDER_STAT_TRANSACTION_COMPLETE); + + t->debug_id = ++binder_last_id; + e->debug_id = t->debug_id; + + if (reply) + binder_debug(BINDER_DEBUG_TRANSACTION, + "%d:%d BC_REPLY %d -> %d:%d, data %016llx-%016llx size %lld-%lld\n", + proc->pid, thread->pid, t->debug_id, + target_proc->pid, target_thread->pid, + (u64)tr->data.ptr.buffer, + (u64)tr->data.ptr.offsets, + (u64)tr->data_size, (u64)tr->offsets_size); + else + binder_debug(BINDER_DEBUG_TRANSACTION, + "%d:%d BC_TRANSACTION %d -> %d - node %d, data %016llx-%016llx size %lld-%lld\n", + proc->pid, thread->pid, t->debug_id, + target_proc->pid, target_node->debug_id, + (u64)tr->data.ptr.buffer, + (u64)tr->data.ptr.offsets, + (u64)tr->data_size, (u64)tr->offsets_size); + + if (!reply && !(tr->flags & TF_ONE_WAY)) + t->from = thread; + else + t->from = NULL; + t->sender_euid = task_euid(proc->tsk); + t->to_proc = target_proc; + t->to_thread = target_thread; + t->code = tr->code; + t->flags = tr->flags; + t->priority = task_nice(current); + + trace_binder_transaction(reply, t, target_node); + + t->buffer = binder_alloc_buf(target_proc, tr->data_size, + tr->offsets_size, !reply && (t->flags & TF_ONE_WAY)); + if (t->buffer == NULL) { + return_error = BR_FAILED_REPLY; + goto err_binder_alloc_buf_failed; + } + t->buffer->allow_user_free = 0; + t->buffer->debug_id = t->debug_id; + t->buffer->transaction = t; + t->buffer->target_node = target_node; + trace_binder_transaction_alloc_buf(t->buffer); + if (target_node) + binder_inc_node(target_node, 1, 0, NULL); + + offp = (binder_size_t *)(t->buffer->data + + ALIGN(tr->data_size, sizeof(void *))); + + if (copy_from_user(t->buffer->data, (const void __user *)(uintptr_t) + tr->data.ptr.buffer, tr->data_size)) { + binder_user_error("%d:%d got transaction with invalid data ptr\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + goto err_copy_data_failed; + } + if (copy_from_user(offp, (const void __user *)(uintptr_t) + tr->data.ptr.offsets, tr->offsets_size)) { + binder_user_error("%d:%d got transaction with invalid offsets ptr\n", + proc->pid, thread->pid); + return_error = BR_FAILED_REPLY; + goto err_copy_data_failed; + } + if (!IS_ALIGNED(tr->offsets_size, sizeof(binder_size_t))) { + binder_user_error("%d:%d got transaction with invalid offsets size, %lld\n", + proc->pid, thread->pid, (u64)tr->offsets_size); + return_error = BR_FAILED_REPLY; + goto err_bad_offset; + } + off_end = (void *)offp + tr->offsets_size; + for (; offp < off_end; offp++) { + struct flat_binder_object *fp; + + if (*offp > t->buffer->data_size - sizeof(*fp) || + t->buffer->data_size < sizeof(*fp) || + !IS_ALIGNED(*offp, sizeof(u32))) { + binder_user_error("%d:%d got transaction with invalid offset, %lld\n", + proc->pid, thread->pid, (u64)*offp); + return_error = BR_FAILED_REPLY; + goto err_bad_offset; + } + fp = (struct flat_binder_object *)(t->buffer->data + *offp); + switch (fp->type) { + case BINDER_TYPE_BINDER: + case BINDER_TYPE_WEAK_BINDER: { + struct binder_ref *ref; + struct binder_node *node = binder_get_node(proc, fp->binder); + + if (node == NULL) { + node = binder_new_node(proc, fp->binder, fp->cookie); + if (node == NULL) { + return_error = BR_FAILED_REPLY; + goto err_binder_new_node_failed; + } + node->min_priority = fp->flags & FLAT_BINDER_FLAG_PRIORITY_MASK; + node->accept_fds = !!(fp->flags & FLAT_BINDER_FLAG_ACCEPTS_FDS); + } + if (fp->cookie != node->cookie) { + binder_user_error("%d:%d sending u%016llx node %d, cookie mismatch %016llx != %016llx\n", + proc->pid, thread->pid, + (u64)fp->binder, node->debug_id, + (u64)fp->cookie, (u64)node->cookie); + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_for_node_failed; + } + ref = binder_get_ref_for_node(target_proc, node); + if (ref == NULL) { + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_for_node_failed; + } + if (fp->type == BINDER_TYPE_BINDER) + fp->type = BINDER_TYPE_HANDLE; + else + fp->type = BINDER_TYPE_WEAK_HANDLE; + fp->handle = ref->desc; + binder_inc_ref(ref, fp->type == BINDER_TYPE_HANDLE, + &thread->todo); + + trace_binder_transaction_node_to_ref(t, node, ref); + binder_debug(BINDER_DEBUG_TRANSACTION, + " node %d u%016llx -> ref %d desc %d\n", + node->debug_id, (u64)node->ptr, + ref->debug_id, ref->desc); + } break; + case BINDER_TYPE_HANDLE: + case BINDER_TYPE_WEAK_HANDLE: { + struct binder_ref *ref = binder_get_ref(proc, fp->handle); + + if (ref == NULL) { + binder_user_error("%d:%d got transaction with invalid handle, %d\n", + proc->pid, + thread->pid, fp->handle); + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_failed; + } + if (ref->node->proc == target_proc) { + if (fp->type == BINDER_TYPE_HANDLE) + fp->type = BINDER_TYPE_BINDER; + else + fp->type = BINDER_TYPE_WEAK_BINDER; + fp->binder = ref->node->ptr; + fp->cookie = ref->node->cookie; + binder_inc_node(ref->node, fp->type == BINDER_TYPE_BINDER, 0, NULL); + trace_binder_transaction_ref_to_node(t, ref); + binder_debug(BINDER_DEBUG_TRANSACTION, + " ref %d desc %d -> node %d u%016llx\n", + ref->debug_id, ref->desc, ref->node->debug_id, + (u64)ref->node->ptr); + } else { + struct binder_ref *new_ref; + + new_ref = binder_get_ref_for_node(target_proc, ref->node); + if (new_ref == NULL) { + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_for_node_failed; + } + fp->handle = new_ref->desc; + binder_inc_ref(new_ref, fp->type == BINDER_TYPE_HANDLE, NULL); + trace_binder_transaction_ref_to_ref(t, ref, + new_ref); + binder_debug(BINDER_DEBUG_TRANSACTION, + " ref %d desc %d -> ref %d desc %d (node %d)\n", + ref->debug_id, ref->desc, new_ref->debug_id, + new_ref->desc, ref->node->debug_id); + } + } break; + + case BINDER_TYPE_FD: { + int target_fd; + struct file *file; + + if (reply) { + if (!(in_reply_to->flags & TF_ACCEPT_FDS)) { + binder_user_error("%d:%d got reply with fd, %d, but target does not allow fds\n", + proc->pid, thread->pid, fp->handle); + return_error = BR_FAILED_REPLY; + goto err_fd_not_allowed; + } + } else if (!target_node->accept_fds) { + binder_user_error("%d:%d got transaction with fd, %d, but target does not allow fds\n", + proc->pid, thread->pid, fp->handle); + return_error = BR_FAILED_REPLY; + goto err_fd_not_allowed; + } + + file = fget(fp->handle); + if (file == NULL) { + binder_user_error("%d:%d got transaction with invalid fd, %d\n", + proc->pid, thread->pid, fp->handle); + return_error = BR_FAILED_REPLY; + goto err_fget_failed; + } + target_fd = task_get_unused_fd_flags(target_proc, O_CLOEXEC); + if (target_fd < 0) { + fput(file); + return_error = BR_FAILED_REPLY; + goto err_get_unused_fd_failed; + } + task_fd_install(target_proc, target_fd, file); + trace_binder_transaction_fd(t, fp->handle, target_fd); + binder_debug(BINDER_DEBUG_TRANSACTION, + " fd %d -> %d\n", fp->handle, target_fd); + /* TODO: fput? */ + fp->handle = target_fd; + } break; + + default: + binder_user_error("%d:%d got transaction with invalid object type, %x\n", + proc->pid, thread->pid, fp->type); + return_error = BR_FAILED_REPLY; + goto err_bad_object_type; + } + } + if (reply) { + BUG_ON(t->buffer->async_transaction != 0); + binder_pop_transaction(target_thread, in_reply_to); + } else if (!(t->flags & TF_ONE_WAY)) { + BUG_ON(t->buffer->async_transaction != 0); + t->need_reply = 1; + t->from_parent = thread->transaction_stack; + thread->transaction_stack = t; + } else { + BUG_ON(target_node == NULL); + BUG_ON(t->buffer->async_transaction != 1); + if (target_node->has_async_transaction) { + target_list = &target_node->async_todo; + target_wait = NULL; + } else + target_node->has_async_transaction = 1; + } + t->work.type = BINDER_WORK_TRANSACTION; + list_add_tail(&t->work.entry, target_list); + tcomplete->type = BINDER_WORK_TRANSACTION_COMPLETE; + list_add_tail(&tcomplete->entry, &thread->todo); + if (target_wait) + wake_up_interruptible(target_wait); + return; + +err_get_unused_fd_failed: +err_fget_failed: +err_fd_not_allowed: +err_binder_get_ref_for_node_failed: +err_binder_get_ref_failed: +err_binder_new_node_failed: +err_bad_object_type: +err_bad_offset: +err_copy_data_failed: + trace_binder_transaction_failed_buffer_release(t->buffer); + binder_transaction_buffer_release(target_proc, t->buffer, offp); + t->buffer->transaction = NULL; + binder_free_buf(target_proc, t->buffer); +err_binder_alloc_buf_failed: + kfree(tcomplete); + binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE); +err_alloc_tcomplete_failed: + kfree(t); + binder_stats_deleted(BINDER_STAT_TRANSACTION); +err_alloc_t_failed: +err_bad_call_stack: +err_empty_call_stack: +err_dead_binder: +err_invalid_target_handle: +err_no_context_mgr_node: + binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, + "%d:%d transaction failed %d, size %lld-%lld\n", + proc->pid, thread->pid, return_error, + (u64)tr->data_size, (u64)tr->offsets_size); + + { + struct binder_transaction_log_entry *fe; + + fe = binder_transaction_log_add(&binder_transaction_log_failed); + *fe = *e; + } + + BUG_ON(thread->return_error != BR_OK); + if (in_reply_to) { + thread->return_error = BR_TRANSACTION_COMPLETE; + binder_send_failed_reply(in_reply_to, return_error); + } else + thread->return_error = return_error; +} + +static int binder_thread_write(struct binder_proc *proc, + struct binder_thread *thread, + binder_uintptr_t binder_buffer, size_t size, + binder_size_t *consumed) +{ + uint32_t cmd; + void __user *buffer = (void __user *)(uintptr_t)binder_buffer; + void __user *ptr = buffer + *consumed; + void __user *end = buffer + size; + + while (ptr < end && thread->return_error == BR_OK) { + if (get_user(cmd, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + trace_binder_command(cmd); + if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.bc)) { + binder_stats.bc[_IOC_NR(cmd)]++; + proc->stats.bc[_IOC_NR(cmd)]++; + thread->stats.bc[_IOC_NR(cmd)]++; + } + switch (cmd) { + case BC_INCREFS: + case BC_ACQUIRE: + case BC_RELEASE: + case BC_DECREFS: { + uint32_t target; + struct binder_ref *ref; + const char *debug_string; + + if (get_user(target, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + if (target == 0 && binder_context_mgr_node && + (cmd == BC_INCREFS || cmd == BC_ACQUIRE)) { + ref = binder_get_ref_for_node(proc, + binder_context_mgr_node); + if (ref->desc != target) { + binder_user_error("%d:%d tried to acquire reference to desc 0, got %d instead\n", + proc->pid, thread->pid, + ref->desc); + } + } else + ref = binder_get_ref(proc, target); + if (ref == NULL) { + binder_user_error("%d:%d refcount change on invalid ref %d\n", + proc->pid, thread->pid, target); + break; + } + switch (cmd) { + case BC_INCREFS: + debug_string = "IncRefs"; + binder_inc_ref(ref, 0, NULL); + break; + case BC_ACQUIRE: + debug_string = "Acquire"; + binder_inc_ref(ref, 1, NULL); + break; + case BC_RELEASE: + debug_string = "Release"; + binder_dec_ref(ref, 1); + break; + case BC_DECREFS: + default: + debug_string = "DecRefs"; + binder_dec_ref(ref, 0); + break; + } + binder_debug(BINDER_DEBUG_USER_REFS, + "%d:%d %s ref %d desc %d s %d w %d for node %d\n", + proc->pid, thread->pid, debug_string, ref->debug_id, + ref->desc, ref->strong, ref->weak, ref->node->debug_id); + break; + } + case BC_INCREFS_DONE: + case BC_ACQUIRE_DONE: { + binder_uintptr_t node_ptr; + binder_uintptr_t cookie; + struct binder_node *node; + + if (get_user(node_ptr, (binder_uintptr_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(binder_uintptr_t); + if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(binder_uintptr_t); + node = binder_get_node(proc, node_ptr); + if (node == NULL) { + binder_user_error("%d:%d %s u%016llx no match\n", + proc->pid, thread->pid, + cmd == BC_INCREFS_DONE ? + "BC_INCREFS_DONE" : + "BC_ACQUIRE_DONE", + (u64)node_ptr); + break; + } + if (cookie != node->cookie) { + binder_user_error("%d:%d %s u%016llx node %d cookie mismatch %016llx != %016llx\n", + proc->pid, thread->pid, + cmd == BC_INCREFS_DONE ? + "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", + (u64)node_ptr, node->debug_id, + (u64)cookie, (u64)node->cookie); + break; + } + if (cmd == BC_ACQUIRE_DONE) { + if (node->pending_strong_ref == 0) { + binder_user_error("%d:%d BC_ACQUIRE_DONE node %d has no pending acquire request\n", + proc->pid, thread->pid, + node->debug_id); + break; + } + node->pending_strong_ref = 0; + } else { + if (node->pending_weak_ref == 0) { + binder_user_error("%d:%d BC_INCREFS_DONE node %d has no pending increfs request\n", + proc->pid, thread->pid, + node->debug_id); + break; + } + node->pending_weak_ref = 0; + } + binder_dec_node(node, cmd == BC_ACQUIRE_DONE, 0); + binder_debug(BINDER_DEBUG_USER_REFS, + "%d:%d %s node %d ls %d lw %d\n", + proc->pid, thread->pid, + cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", + node->debug_id, node->local_strong_refs, node->local_weak_refs); + break; + } + case BC_ATTEMPT_ACQUIRE: + pr_err("BC_ATTEMPT_ACQUIRE not supported\n"); + return -EINVAL; + case BC_ACQUIRE_RESULT: + pr_err("BC_ACQUIRE_RESULT not supported\n"); + return -EINVAL; + + case BC_FREE_BUFFER: { + binder_uintptr_t data_ptr; + struct binder_buffer *buffer; + + if (get_user(data_ptr, (binder_uintptr_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(binder_uintptr_t); + + buffer = binder_buffer_lookup(proc, data_ptr); + if (buffer == NULL) { + binder_user_error("%d:%d BC_FREE_BUFFER u%016llx no match\n", + proc->pid, thread->pid, (u64)data_ptr); + break; + } + if (!buffer->allow_user_free) { + binder_user_error("%d:%d BC_FREE_BUFFER u%016llx matched unreturned buffer\n", + proc->pid, thread->pid, (u64)data_ptr); + break; + } + binder_debug(BINDER_DEBUG_FREE_BUFFER, + "%d:%d BC_FREE_BUFFER u%016llx found buffer %d for %s transaction\n", + proc->pid, thread->pid, (u64)data_ptr, + buffer->debug_id, + buffer->transaction ? "active" : "finished"); + + if (buffer->transaction) { + buffer->transaction->buffer = NULL; + buffer->transaction = NULL; + } + if (buffer->async_transaction && buffer->target_node) { + BUG_ON(!buffer->target_node->has_async_transaction); + if (list_empty(&buffer->target_node->async_todo)) + buffer->target_node->has_async_transaction = 0; + else + list_move_tail(buffer->target_node->async_todo.next, &thread->todo); + } + trace_binder_transaction_buffer_release(buffer); + binder_transaction_buffer_release(proc, buffer, NULL); + binder_free_buf(proc, buffer); + break; + } + + case BC_TRANSACTION: + case BC_REPLY: { + struct binder_transaction_data tr; + + if (copy_from_user(&tr, ptr, sizeof(tr))) + return -EFAULT; + ptr += sizeof(tr); + binder_transaction(proc, thread, &tr, cmd == BC_REPLY); + break; + } + + case BC_REGISTER_LOOPER: + binder_debug(BINDER_DEBUG_THREADS, + "%d:%d BC_REGISTER_LOOPER\n", + proc->pid, thread->pid); + if (thread->looper & BINDER_LOOPER_STATE_ENTERED) { + thread->looper |= BINDER_LOOPER_STATE_INVALID; + binder_user_error("%d:%d ERROR: BC_REGISTER_LOOPER called after BC_ENTER_LOOPER\n", + proc->pid, thread->pid); + } else if (proc->requested_threads == 0) { + thread->looper |= BINDER_LOOPER_STATE_INVALID; + binder_user_error("%d:%d ERROR: BC_REGISTER_LOOPER called without request\n", + proc->pid, thread->pid); + } else { + proc->requested_threads--; + proc->requested_threads_started++; + } + thread->looper |= BINDER_LOOPER_STATE_REGISTERED; + break; + case BC_ENTER_LOOPER: + binder_debug(BINDER_DEBUG_THREADS, + "%d:%d BC_ENTER_LOOPER\n", + proc->pid, thread->pid); + if (thread->looper & BINDER_LOOPER_STATE_REGISTERED) { + thread->looper |= BINDER_LOOPER_STATE_INVALID; + binder_user_error("%d:%d ERROR: BC_ENTER_LOOPER called after BC_REGISTER_LOOPER\n", + proc->pid, thread->pid); + } + thread->looper |= BINDER_LOOPER_STATE_ENTERED; + break; + case BC_EXIT_LOOPER: + binder_debug(BINDER_DEBUG_THREADS, + "%d:%d BC_EXIT_LOOPER\n", + proc->pid, thread->pid); + thread->looper |= BINDER_LOOPER_STATE_EXITED; + break; + + case BC_REQUEST_DEATH_NOTIFICATION: + case BC_CLEAR_DEATH_NOTIFICATION: { + uint32_t target; + binder_uintptr_t cookie; + struct binder_ref *ref; + struct binder_ref_death *death; + + if (get_user(target, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(binder_uintptr_t); + ref = binder_get_ref(proc, target); + if (ref == NULL) { + binder_user_error("%d:%d %s invalid ref %d\n", + proc->pid, thread->pid, + cmd == BC_REQUEST_DEATH_NOTIFICATION ? + "BC_REQUEST_DEATH_NOTIFICATION" : + "BC_CLEAR_DEATH_NOTIFICATION", + target); + break; + } + + binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, + "%d:%d %s %016llx ref %d desc %d s %d w %d for node %d\n", + proc->pid, thread->pid, + cmd == BC_REQUEST_DEATH_NOTIFICATION ? + "BC_REQUEST_DEATH_NOTIFICATION" : + "BC_CLEAR_DEATH_NOTIFICATION", + (u64)cookie, ref->debug_id, ref->desc, + ref->strong, ref->weak, ref->node->debug_id); + + if (cmd == BC_REQUEST_DEATH_NOTIFICATION) { + if (ref->death) { + binder_user_error("%d:%d BC_REQUEST_DEATH_NOTIFICATION death notification already set\n", + proc->pid, thread->pid); + break; + } + death = kzalloc(sizeof(*death), GFP_KERNEL); + if (death == NULL) { + thread->return_error = BR_ERROR; + binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, + "%d:%d BC_REQUEST_DEATH_NOTIFICATION failed\n", + proc->pid, thread->pid); + break; + } + binder_stats_created(BINDER_STAT_DEATH); + INIT_LIST_HEAD(&death->work.entry); + death->cookie = cookie; + ref->death = death; + if (ref->node->proc == NULL) { + ref->death->work.type = BINDER_WORK_DEAD_BINDER; + if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) { + list_add_tail(&ref->death->work.entry, &thread->todo); + } else { + list_add_tail(&ref->death->work.entry, &proc->todo); + wake_up_interruptible(&proc->wait); + } + } + } else { + if (ref->death == NULL) { + binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification not active\n", + proc->pid, thread->pid); + break; + } + death = ref->death; + if (death->cookie != cookie) { + binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch %016llx != %016llx\n", + proc->pid, thread->pid, + (u64)death->cookie, + (u64)cookie); + break; + } + ref->death = NULL; + if (list_empty(&death->work.entry)) { + death->work.type = BINDER_WORK_CLEAR_DEATH_NOTIFICATION; + if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) { + list_add_tail(&death->work.entry, &thread->todo); + } else { + list_add_tail(&death->work.entry, &proc->todo); + wake_up_interruptible(&proc->wait); + } + } else { + BUG_ON(death->work.type != BINDER_WORK_DEAD_BINDER); + death->work.type = BINDER_WORK_DEAD_BINDER_AND_CLEAR; + } + } + } break; + case BC_DEAD_BINDER_DONE: { + struct binder_work *w; + binder_uintptr_t cookie; + struct binder_ref_death *death = NULL; + + if (get_user(cookie, (binder_uintptr_t __user *)ptr)) + return -EFAULT; + + ptr += sizeof(void *); + list_for_each_entry(w, &proc->delivered_death, entry) { + struct binder_ref_death *tmp_death = container_of(w, struct binder_ref_death, work); + + if (tmp_death->cookie == cookie) { + death = tmp_death; + break; + } + } + binder_debug(BINDER_DEBUG_DEAD_BINDER, + "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n", + proc->pid, thread->pid, (u64)cookie, + death); + if (death == NULL) { + binder_user_error("%d:%d BC_DEAD_BINDER_DONE %016llx not found\n", + proc->pid, thread->pid, (u64)cookie); + break; + } + + list_del_init(&death->work.entry); + if (death->work.type == BINDER_WORK_DEAD_BINDER_AND_CLEAR) { + death->work.type = BINDER_WORK_CLEAR_DEATH_NOTIFICATION; + if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) { + list_add_tail(&death->work.entry, &thread->todo); + } else { + list_add_tail(&death->work.entry, &proc->todo); + wake_up_interruptible(&proc->wait); + } + } + } break; + + default: + pr_err("%d:%d unknown command %d\n", + proc->pid, thread->pid, cmd); + return -EINVAL; + } + *consumed = ptr - buffer; + } + return 0; +} + +static void binder_stat_br(struct binder_proc *proc, + struct binder_thread *thread, uint32_t cmd) +{ + trace_binder_return(cmd); + if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.br)) { + binder_stats.br[_IOC_NR(cmd)]++; + proc->stats.br[_IOC_NR(cmd)]++; + thread->stats.br[_IOC_NR(cmd)]++; + } +} + +static int binder_has_proc_work(struct binder_proc *proc, + struct binder_thread *thread) +{ + return !list_empty(&proc->todo) || + (thread->looper & BINDER_LOOPER_STATE_NEED_RETURN); +} + +static int binder_has_thread_work(struct binder_thread *thread) +{ + return !list_empty(&thread->todo) || thread->return_error != BR_OK || + (thread->looper & BINDER_LOOPER_STATE_NEED_RETURN); +} + +static int binder_thread_read(struct binder_proc *proc, + struct binder_thread *thread, + binder_uintptr_t binder_buffer, size_t size, + binder_size_t *consumed, int non_block) +{ + void __user *buffer = (void __user *)(uintptr_t)binder_buffer; + void __user *ptr = buffer + *consumed; + void __user *end = buffer + size; + + int ret = 0; + int wait_for_proc_work; + + if (*consumed == 0) { + if (put_user(BR_NOOP, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + } + +retry: + wait_for_proc_work = thread->transaction_stack == NULL && + list_empty(&thread->todo); + + if (thread->return_error != BR_OK && ptr < end) { + if (thread->return_error2 != BR_OK) { + if (put_user(thread->return_error2, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + binder_stat_br(proc, thread, thread->return_error2); + if (ptr == end) + goto done; + thread->return_error2 = BR_OK; + } + if (put_user(thread->return_error, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + binder_stat_br(proc, thread, thread->return_error); + thread->return_error = BR_OK; + goto done; + } + + + thread->looper |= BINDER_LOOPER_STATE_WAITING; + if (wait_for_proc_work) + proc->ready_threads++; + + binder_unlock(__func__); + + trace_binder_wait_for_work(wait_for_proc_work, + !!thread->transaction_stack, + !list_empty(&thread->todo)); + if (wait_for_proc_work) { + if (!(thread->looper & (BINDER_LOOPER_STATE_REGISTERED | + BINDER_LOOPER_STATE_ENTERED))) { + binder_user_error("%d:%d ERROR: Thread waiting for process work before calling BC_REGISTER_LOOPER or BC_ENTER_LOOPER (state %x)\n", + proc->pid, thread->pid, thread->looper); + wait_event_interruptible(binder_user_error_wait, + binder_stop_on_user_error < 2); + } + binder_set_nice(proc->default_priority); + if (non_block) { + if (!binder_has_proc_work(proc, thread)) + ret = -EAGAIN; + } else + ret = wait_event_freezable_exclusive(proc->wait, binder_has_proc_work(proc, thread)); + } else { + if (non_block) { + if (!binder_has_thread_work(thread)) + ret = -EAGAIN; + } else + ret = wait_event_freezable(thread->wait, binder_has_thread_work(thread)); + } + + binder_lock(__func__); + + if (wait_for_proc_work) + proc->ready_threads--; + thread->looper &= ~BINDER_LOOPER_STATE_WAITING; + + if (ret) + return ret; + + while (1) { + uint32_t cmd; + struct binder_transaction_data tr; + struct binder_work *w; + struct binder_transaction *t = NULL; + + if (!list_empty(&thread->todo)) { + w = list_first_entry(&thread->todo, struct binder_work, + entry); + } else if (!list_empty(&proc->todo) && wait_for_proc_work) { + w = list_first_entry(&proc->todo, struct binder_work, + entry); + } else { + /* no data added */ + if (ptr - buffer == 4 && + !(thread->looper & BINDER_LOOPER_STATE_NEED_RETURN)) + goto retry; + break; + } + + if (end - ptr < sizeof(tr) + 4) + break; + + switch (w->type) { + case BINDER_WORK_TRANSACTION: { + t = container_of(w, struct binder_transaction, work); + } break; + case BINDER_WORK_TRANSACTION_COMPLETE: { + cmd = BR_TRANSACTION_COMPLETE; + if (put_user(cmd, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + + binder_stat_br(proc, thread, cmd); + binder_debug(BINDER_DEBUG_TRANSACTION_COMPLETE, + "%d:%d BR_TRANSACTION_COMPLETE\n", + proc->pid, thread->pid); + + list_del(&w->entry); + kfree(w); + binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE); + } break; + case BINDER_WORK_NODE: { + struct binder_node *node = container_of(w, struct binder_node, work); + uint32_t cmd = BR_NOOP; + const char *cmd_name; + int strong = node->internal_strong_refs || node->local_strong_refs; + int weak = !hlist_empty(&node->refs) || node->local_weak_refs || strong; + + if (weak && !node->has_weak_ref) { + cmd = BR_INCREFS; + cmd_name = "BR_INCREFS"; + node->has_weak_ref = 1; + node->pending_weak_ref = 1; + node->local_weak_refs++; + } else if (strong && !node->has_strong_ref) { + cmd = BR_ACQUIRE; + cmd_name = "BR_ACQUIRE"; + node->has_strong_ref = 1; + node->pending_strong_ref = 1; + node->local_strong_refs++; + } else if (!strong && node->has_strong_ref) { + cmd = BR_RELEASE; + cmd_name = "BR_RELEASE"; + node->has_strong_ref = 0; + } else if (!weak && node->has_weak_ref) { + cmd = BR_DECREFS; + cmd_name = "BR_DECREFS"; + node->has_weak_ref = 0; + } + if (cmd != BR_NOOP) { + if (put_user(cmd, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + if (put_user(node->ptr, + (binder_uintptr_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(binder_uintptr_t); + if (put_user(node->cookie, + (binder_uintptr_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(binder_uintptr_t); + + binder_stat_br(proc, thread, cmd); + binder_debug(BINDER_DEBUG_USER_REFS, + "%d:%d %s %d u%016llx c%016llx\n", + proc->pid, thread->pid, cmd_name, + node->debug_id, + (u64)node->ptr, (u64)node->cookie); + } else { + list_del_init(&w->entry); + if (!weak && !strong) { + binder_debug(BINDER_DEBUG_INTERNAL_REFS, + "%d:%d node %d u%016llx c%016llx deleted\n", + proc->pid, thread->pid, + node->debug_id, + (u64)node->ptr, + (u64)node->cookie); + rb_erase(&node->rb_node, &proc->nodes); + kfree(node); + binder_stats_deleted(BINDER_STAT_NODE); + } else { + binder_debug(BINDER_DEBUG_INTERNAL_REFS, + "%d:%d node %d u%016llx c%016llx state unchanged\n", + proc->pid, thread->pid, + node->debug_id, + (u64)node->ptr, + (u64)node->cookie); + } + } + } break; + case BINDER_WORK_DEAD_BINDER: + case BINDER_WORK_DEAD_BINDER_AND_CLEAR: + case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: { + struct binder_ref_death *death; + uint32_t cmd; + + death = container_of(w, struct binder_ref_death, work); + if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION) + cmd = BR_CLEAR_DEATH_NOTIFICATION_DONE; + else + cmd = BR_DEAD_BINDER; + if (put_user(cmd, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + if (put_user(death->cookie, + (binder_uintptr_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(binder_uintptr_t); + binder_stat_br(proc, thread, cmd); + binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, + "%d:%d %s %016llx\n", + proc->pid, thread->pid, + cmd == BR_DEAD_BINDER ? + "BR_DEAD_BINDER" : + "BR_CLEAR_DEATH_NOTIFICATION_DONE", + (u64)death->cookie); + + if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION) { + list_del(&w->entry); + kfree(death); + binder_stats_deleted(BINDER_STAT_DEATH); + } else + list_move(&w->entry, &proc->delivered_death); + if (cmd == BR_DEAD_BINDER) + goto done; /* DEAD_BINDER notifications can cause transactions */ + } break; + } + + if (!t) + continue; + + BUG_ON(t->buffer == NULL); + if (t->buffer->target_node) { + struct binder_node *target_node = t->buffer->target_node; + + tr.target.ptr = target_node->ptr; + tr.cookie = target_node->cookie; + t->saved_priority = task_nice(current); + if (t->priority < target_node->min_priority && + !(t->flags & TF_ONE_WAY)) + binder_set_nice(t->priority); + else if (!(t->flags & TF_ONE_WAY) || + t->saved_priority > target_node->min_priority) + binder_set_nice(target_node->min_priority); + cmd = BR_TRANSACTION; + } else { + tr.target.ptr = 0; + tr.cookie = 0; + cmd = BR_REPLY; + } + tr.code = t->code; + tr.flags = t->flags; + tr.sender_euid = from_kuid(current_user_ns(), t->sender_euid); + + if (t->from) { + struct task_struct *sender = t->from->proc->tsk; + + tr.sender_pid = task_tgid_nr_ns(sender, + task_active_pid_ns(current)); + } else { + tr.sender_pid = 0; + } + + tr.data_size = t->buffer->data_size; + tr.offsets_size = t->buffer->offsets_size; + tr.data.ptr.buffer = (binder_uintptr_t)( + (uintptr_t)t->buffer->data + + proc->user_buffer_offset); + tr.data.ptr.offsets = tr.data.ptr.buffer + + ALIGN(t->buffer->data_size, + sizeof(void *)); + + if (put_user(cmd, (uint32_t __user *)ptr)) + return -EFAULT; + ptr += sizeof(uint32_t); + if (copy_to_user(ptr, &tr, sizeof(tr))) + return -EFAULT; + ptr += sizeof(tr); + + trace_binder_transaction_received(t); + binder_stat_br(proc, thread, cmd); + binder_debug(BINDER_DEBUG_TRANSACTION, + "%d:%d %s %d %d:%d, cmd %d size %zd-%zd ptr %016llx-%016llx\n", + proc->pid, thread->pid, + (cmd == BR_TRANSACTION) ? "BR_TRANSACTION" : + "BR_REPLY", + t->debug_id, t->from ? t->from->proc->pid : 0, + t->from ? t->from->pid : 0, cmd, + t->buffer->data_size, t->buffer->offsets_size, + (u64)tr.data.ptr.buffer, (u64)tr.data.ptr.offsets); + + list_del(&t->work.entry); + t->buffer->allow_user_free = 1; + if (cmd == BR_TRANSACTION && !(t->flags & TF_ONE_WAY)) { + t->to_parent = thread->transaction_stack; + t->to_thread = thread; + thread->transaction_stack = t; + } else { + t->buffer->transaction = NULL; + kfree(t); + binder_stats_deleted(BINDER_STAT_TRANSACTION); + } + break; + } + +done: + + *consumed = ptr - buffer; + if (proc->requested_threads + proc->ready_threads == 0 && + proc->requested_threads_started < proc->max_threads && + (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | + BINDER_LOOPER_STATE_ENTERED)) /* the user-space code fails to */ + /*spawn a new thread if we leave this out */) { + proc->requested_threads++; + binder_debug(BINDER_DEBUG_THREADS, + "%d:%d BR_SPAWN_LOOPER\n", + proc->pid, thread->pid); + if (put_user(BR_SPAWN_LOOPER, (uint32_t __user *)buffer)) + return -EFAULT; + binder_stat_br(proc, thread, BR_SPAWN_LOOPER); + } + return 0; +} + +static void binder_release_work(struct list_head *list) +{ + struct binder_work *w; + + while (!list_empty(list)) { + w = list_first_entry(list, struct binder_work, entry); + list_del_init(&w->entry); + switch (w->type) { + case BINDER_WORK_TRANSACTION: { + struct binder_transaction *t; + + t = container_of(w, struct binder_transaction, work); + if (t->buffer->target_node && + !(t->flags & TF_ONE_WAY)) { + binder_send_failed_reply(t, BR_DEAD_REPLY); + } else { + binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, + "undelivered transaction %d\n", + t->debug_id); + t->buffer->transaction = NULL; + kfree(t); + binder_stats_deleted(BINDER_STAT_TRANSACTION); + } + } break; + case BINDER_WORK_TRANSACTION_COMPLETE: { + binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, + "undelivered TRANSACTION_COMPLETE\n"); + kfree(w); + binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE); + } break; + case BINDER_WORK_DEAD_BINDER_AND_CLEAR: + case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: { + struct binder_ref_death *death; + + death = container_of(w, struct binder_ref_death, work); + binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, + "undelivered death notification, %016llx\n", + (u64)death->cookie); + kfree(death); + binder_stats_deleted(BINDER_STAT_DEATH); + } break; + default: + pr_err("unexpected work type, %d, not freed\n", + w->type); + break; + } + } + +} + +static struct binder_thread *binder_get_thread(struct binder_proc *proc) +{ + struct binder_thread *thread = NULL; + struct rb_node *parent = NULL; + struct rb_node **p = &proc->threads.rb_node; + + while (*p) { + parent = *p; + thread = rb_entry(parent, struct binder_thread, rb_node); + + if (current->pid < thread->pid) + p = &(*p)->rb_left; + else if (current->pid > thread->pid) + p = &(*p)->rb_right; + else + break; + } + if (*p == NULL) { + thread = kzalloc(sizeof(*thread), GFP_KERNEL); + if (thread == NULL) + return NULL; + binder_stats_created(BINDER_STAT_THREAD); + thread->proc = proc; + thread->pid = current->pid; + init_waitqueue_head(&thread->wait); + INIT_LIST_HEAD(&thread->todo); + rb_link_node(&thread->rb_node, parent, p); + rb_insert_color(&thread->rb_node, &proc->threads); + thread->looper |= BINDER_LOOPER_STATE_NEED_RETURN; + thread->return_error = BR_OK; + thread->return_error2 = BR_OK; + } + return thread; +} + +static int binder_free_thread(struct binder_proc *proc, + struct binder_thread *thread) +{ + struct binder_transaction *t; + struct binder_transaction *send_reply = NULL; + int active_transactions = 0; + + rb_erase(&thread->rb_node, &proc->threads); + t = thread->transaction_stack; + if (t && t->to_thread == thread) + send_reply = t; + while (t) { + active_transactions++; + binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, + "release %d:%d transaction %d %s, still active\n", + proc->pid, thread->pid, + t->debug_id, + (t->to_thread == thread) ? "in" : "out"); + + if (t->to_thread == thread) { + t->to_proc = NULL; + t->to_thread = NULL; + if (t->buffer) { + t->buffer->transaction = NULL; + t->buffer = NULL; + } + t = t->to_parent; + } else if (t->from == thread) { + t->from = NULL; + t = t->from_parent; + } else + BUG(); + } + if (send_reply) + binder_send_failed_reply(send_reply, BR_DEAD_REPLY); + binder_release_work(&thread->todo); + kfree(thread); + binder_stats_deleted(BINDER_STAT_THREAD); + return active_transactions; +} + +static unsigned int binder_poll(struct file *filp, + struct poll_table_struct *wait) +{ + struct binder_proc *proc = filp->private_data; + struct binder_thread *thread = NULL; + int wait_for_proc_work; + + binder_lock(__func__); + + thread = binder_get_thread(proc); + + wait_for_proc_work = thread->transaction_stack == NULL && + list_empty(&thread->todo) && thread->return_error == BR_OK; + + binder_unlock(__func__); + + if (wait_for_proc_work) { + if (binder_has_proc_work(proc, thread)) + return POLLIN; + poll_wait(filp, &proc->wait, wait); + if (binder_has_proc_work(proc, thread)) + return POLLIN; + } else { + if (binder_has_thread_work(thread)) + return POLLIN; + poll_wait(filp, &thread->wait, wait); + if (binder_has_thread_work(thread)) + return POLLIN; + } + return 0; +} + +static int binder_ioctl_write_read(struct file *filp, + unsigned int cmd, unsigned long arg, + struct binder_thread *thread) +{ + int ret = 0; + struct binder_proc *proc = filp->private_data; + unsigned int size = _IOC_SIZE(cmd); + void __user *ubuf = (void __user *)arg; + struct binder_write_read bwr; + + if (size != sizeof(struct binder_write_read)) { + ret = -EINVAL; + goto out; + } + if (copy_from_user(&bwr, ubuf, sizeof(bwr))) { + ret = -EFAULT; + goto out; + } + binder_debug(BINDER_DEBUG_READ_WRITE, + "%d:%d write %lld at %016llx, read %lld at %016llx\n", + proc->pid, thread->pid, + (u64)bwr.write_size, (u64)bwr.write_buffer, + (u64)bwr.read_size, (u64)bwr.read_buffer); + + if (bwr.write_size > 0) { + ret = binder_thread_write(proc, thread, + bwr.write_buffer, + bwr.write_size, + &bwr.write_consumed); + trace_binder_write_done(ret); + if (ret < 0) { + bwr.read_consumed = 0; + if (copy_to_user(ubuf, &bwr, sizeof(bwr))) + ret = -EFAULT; + goto out; + } + } + if (bwr.read_size > 0) { + ret = binder_thread_read(proc, thread, bwr.read_buffer, + bwr.read_size, + &bwr.read_consumed, + filp->f_flags & O_NONBLOCK); + trace_binder_read_done(ret); + if (!list_empty(&proc->todo)) + wake_up_interruptible(&proc->wait); + if (ret < 0) { + if (copy_to_user(ubuf, &bwr, sizeof(bwr))) + ret = -EFAULT; + goto out; + } + } + binder_debug(BINDER_DEBUG_READ_WRITE, + "%d:%d wrote %lld of %lld, read return %lld of %lld\n", + proc->pid, thread->pid, + (u64)bwr.write_consumed, (u64)bwr.write_size, + (u64)bwr.read_consumed, (u64)bwr.read_size); + if (copy_to_user(ubuf, &bwr, sizeof(bwr))) { + ret = -EFAULT; + goto out; + } +out: + return ret; +} + +static int binder_ioctl_set_ctx_mgr(struct file *filp) +{ + int ret = 0; + struct binder_proc *proc = filp->private_data; + kuid_t curr_euid = current_euid(); + + if (binder_context_mgr_node != NULL) { + pr_err("BINDER_SET_CONTEXT_MGR already set\n"); + ret = -EBUSY; + goto out; + } + if (uid_valid(binder_context_mgr_uid)) { + if (!uid_eq(binder_context_mgr_uid, curr_euid)) { + pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", + from_kuid(&init_user_ns, curr_euid), + from_kuid(&init_user_ns, + binder_context_mgr_uid)); + ret = -EPERM; + goto out; + } + } else { + binder_context_mgr_uid = curr_euid; + } + binder_context_mgr_node = binder_new_node(proc, 0, 0); + if (binder_context_mgr_node == NULL) { + ret = -ENOMEM; + goto out; + } + binder_context_mgr_node->local_weak_refs++; + binder_context_mgr_node->local_strong_refs++; + binder_context_mgr_node->has_strong_ref = 1; + binder_context_mgr_node->has_weak_ref = 1; +out: + return ret; +} + +static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + int ret; + struct binder_proc *proc = filp->private_data; + struct binder_thread *thread; + unsigned int size = _IOC_SIZE(cmd); + void __user *ubuf = (void __user *)arg; + + /*pr_info("binder_ioctl: %d:%d %x %lx\n", + proc->pid, current->pid, cmd, arg);*/ + + trace_binder_ioctl(cmd, arg); + + ret = wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2); + if (ret) + goto err_unlocked; + + binder_lock(__func__); + thread = binder_get_thread(proc); + if (thread == NULL) { + ret = -ENOMEM; + goto err; + } + + switch (cmd) { + case BINDER_WRITE_READ: + ret = binder_ioctl_write_read(filp, cmd, arg, thread); + if (ret) + goto err; + break; + case BINDER_SET_MAX_THREADS: + if (copy_from_user(&proc->max_threads, ubuf, sizeof(proc->max_threads))) { + ret = -EINVAL; + goto err; + } + break; + case BINDER_SET_CONTEXT_MGR: + ret = binder_ioctl_set_ctx_mgr(filp); + if (ret) + goto err; + break; + case BINDER_THREAD_EXIT: + binder_debug(BINDER_DEBUG_THREADS, "%d:%d exit\n", + proc->pid, thread->pid); + binder_free_thread(proc, thread); + thread = NULL; + break; + case BINDER_VERSION: { + struct binder_version __user *ver = ubuf; + + if (size != sizeof(struct binder_version)) { + ret = -EINVAL; + goto err; + } + if (put_user(BINDER_CURRENT_PROTOCOL_VERSION, + &ver->protocol_version)) { + ret = -EINVAL; + goto err; + } + break; + } + default: + ret = -EINVAL; + goto err; + } + ret = 0; +err: + if (thread) + thread->looper &= ~BINDER_LOOPER_STATE_NEED_RETURN; + binder_unlock(__func__); + wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2); + if (ret && ret != -ERESTARTSYS) + pr_info("%d:%d ioctl %x %lx returned %d\n", proc->pid, current->pid, cmd, arg, ret); +err_unlocked: + trace_binder_ioctl_done(ret); + return ret; +} + +static void binder_vma_open(struct vm_area_struct *vma) +{ + struct binder_proc *proc = vma->vm_private_data; + + binder_debug(BINDER_DEBUG_OPEN_CLOSE, + "%d open vm area %lx-%lx (%ld K) vma %lx pagep %lx\n", + proc->pid, vma->vm_start, vma->vm_end, + (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags, + (unsigned long)pgprot_val(vma->vm_page_prot)); +} + +static void binder_vma_close(struct vm_area_struct *vma) +{ + struct binder_proc *proc = vma->vm_private_data; + + binder_debug(BINDER_DEBUG_OPEN_CLOSE, + "%d close vm area %lx-%lx (%ld K) vma %lx pagep %lx\n", + proc->pid, vma->vm_start, vma->vm_end, + (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags, + (unsigned long)pgprot_val(vma->vm_page_prot)); + proc->vma = NULL; + proc->vma_vm_mm = NULL; + binder_defer_work(proc, BINDER_DEFERRED_PUT_FILES); +} + +static int binder_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + return VM_FAULT_SIGBUS; +} + +static struct vm_operations_struct binder_vm_ops = { + .open = binder_vma_open, + .close = binder_vma_close, + .fault = binder_vm_fault, +}; + +static int binder_mmap(struct file *filp, struct vm_area_struct *vma) +{ + int ret; + struct vm_struct *area; + struct binder_proc *proc = filp->private_data; + const char *failure_string; + struct binder_buffer *buffer; + + if (proc->tsk != current) + return -EINVAL; + + if ((vma->vm_end - vma->vm_start) > SZ_4M) + vma->vm_end = vma->vm_start + SZ_4M; + + binder_debug(BINDER_DEBUG_OPEN_CLOSE, + "binder_mmap: %d %lx-%lx (%ld K) vma %lx pagep %lx\n", + proc->pid, vma->vm_start, vma->vm_end, + (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags, + (unsigned long)pgprot_val(vma->vm_page_prot)); + + if (vma->vm_flags & FORBIDDEN_MMAP_FLAGS) { + ret = -EPERM; + failure_string = "bad vm_flags"; + goto err_bad_arg; + } + vma->vm_flags = (vma->vm_flags | VM_DONTCOPY) & ~VM_MAYWRITE; + + mutex_lock(&binder_mmap_lock); + if (proc->buffer) { + ret = -EBUSY; + failure_string = "already mapped"; + goto err_already_mapped; + } + + area = get_vm_area(vma->vm_end - vma->vm_start, VM_IOREMAP); + if (area == NULL) { + ret = -ENOMEM; + failure_string = "get_vm_area"; + goto err_get_vm_area_failed; + } + proc->buffer = area->addr; + proc->user_buffer_offset = vma->vm_start - (uintptr_t)proc->buffer; + mutex_unlock(&binder_mmap_lock); + +#ifdef CONFIG_CPU_CACHE_VIPT + if (cache_is_vipt_aliasing()) { + while (CACHE_COLOUR((vma->vm_start ^ (uint32_t)proc->buffer))) { + pr_info("binder_mmap: %d %lx-%lx maps %p bad alignment\n", proc->pid, vma->vm_start, vma->vm_end, proc->buffer); + vma->vm_start += PAGE_SIZE; + } + } +#endif + proc->pages = kzalloc(sizeof(proc->pages[0]) * ((vma->vm_end - vma->vm_start) / PAGE_SIZE), GFP_KERNEL); + if (proc->pages == NULL) { + ret = -ENOMEM; + failure_string = "alloc page array"; + goto err_alloc_pages_failed; + } + proc->buffer_size = vma->vm_end - vma->vm_start; + + vma->vm_ops = &binder_vm_ops; + vma->vm_private_data = proc; + + if (binder_update_page_range(proc, 1, proc->buffer, proc->buffer + PAGE_SIZE, vma)) { + ret = -ENOMEM; + failure_string = "alloc small buf"; + goto err_alloc_small_buf_failed; + } + buffer = proc->buffer; + INIT_LIST_HEAD(&proc->buffers); + list_add(&buffer->entry, &proc->buffers); + buffer->free = 1; + binder_insert_free_buffer(proc, buffer); + proc->free_async_space = proc->buffer_size / 2; + barrier(); + proc->files = get_files_struct(current); + proc->vma = vma; + proc->vma_vm_mm = vma->vm_mm; + + /*pr_info("binder_mmap: %d %lx-%lx maps %p\n", + proc->pid, vma->vm_start, vma->vm_end, proc->buffer);*/ + return 0; + +err_alloc_small_buf_failed: + kfree(proc->pages); + proc->pages = NULL; +err_alloc_pages_failed: + mutex_lock(&binder_mmap_lock); + vfree(proc->buffer); + proc->buffer = NULL; +err_get_vm_area_failed: +err_already_mapped: + mutex_unlock(&binder_mmap_lock); +err_bad_arg: + pr_err("binder_mmap: %d %lx-%lx %s failed %d\n", + proc->pid, vma->vm_start, vma->vm_end, failure_string, ret); + return ret; +} + +static int binder_open(struct inode *nodp, struct file *filp) +{ + struct binder_proc *proc; + + binder_debug(BINDER_DEBUG_OPEN_CLOSE, "binder_open: %d:%d\n", + current->group_leader->pid, current->pid); + + proc = kzalloc(sizeof(*proc), GFP_KERNEL); + if (proc == NULL) + return -ENOMEM; + get_task_struct(current); + proc->tsk = current; + INIT_LIST_HEAD(&proc->todo); + init_waitqueue_head(&proc->wait); + proc->default_priority = task_nice(current); + + binder_lock(__func__); + + binder_stats_created(BINDER_STAT_PROC); + hlist_add_head(&proc->proc_node, &binder_procs); + proc->pid = current->group_leader->pid; + INIT_LIST_HEAD(&proc->delivered_death); + filp->private_data = proc; + + binder_unlock(__func__); + + if (binder_debugfs_dir_entry_proc) { + char strbuf[11]; + + snprintf(strbuf, sizeof(strbuf), "%u", proc->pid); + proc->debugfs_entry = debugfs_create_file(strbuf, S_IRUGO, + binder_debugfs_dir_entry_proc, proc, &binder_proc_fops); + } + + return 0; +} + +static int binder_flush(struct file *filp, fl_owner_t id) +{ + struct binder_proc *proc = filp->private_data; + + binder_defer_work(proc, BINDER_DEFERRED_FLUSH); + + return 0; +} + +static void binder_deferred_flush(struct binder_proc *proc) +{ + struct rb_node *n; + int wake_count = 0; + + for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) { + struct binder_thread *thread = rb_entry(n, struct binder_thread, rb_node); + + thread->looper |= BINDER_LOOPER_STATE_NEED_RETURN; + if (thread->looper & BINDER_LOOPER_STATE_WAITING) { + wake_up_interruptible(&thread->wait); + wake_count++; + } + } + wake_up_interruptible_all(&proc->wait); + + binder_debug(BINDER_DEBUG_OPEN_CLOSE, + "binder_flush: %d woke %d threads\n", proc->pid, + wake_count); +} + +static int binder_release(struct inode *nodp, struct file *filp) +{ + struct binder_proc *proc = filp->private_data; + + debugfs_remove(proc->debugfs_entry); + binder_defer_work(proc, BINDER_DEFERRED_RELEASE); + + return 0; +} + +static int binder_node_release(struct binder_node *node, int refs) +{ + struct binder_ref *ref; + int death = 0; + + list_del_init(&node->work.entry); + binder_release_work(&node->async_todo); + + if (hlist_empty(&node->refs)) { + kfree(node); + binder_stats_deleted(BINDER_STAT_NODE); + + return refs; + } + + node->proc = NULL; + node->local_strong_refs = 0; + node->local_weak_refs = 0; + hlist_add_head(&node->dead_node, &binder_dead_nodes); + + hlist_for_each_entry(ref, &node->refs, node_entry) { + refs++; + + if (!ref->death) + continue; + + death++; + + if (list_empty(&ref->death->work.entry)) { + ref->death->work.type = BINDER_WORK_DEAD_BINDER; + list_add_tail(&ref->death->work.entry, + &ref->proc->todo); + wake_up_interruptible(&ref->proc->wait); + } else + BUG(); + } + + binder_debug(BINDER_DEBUG_DEAD_BINDER, + "node %d now dead, refs %d, death %d\n", + node->debug_id, refs, death); + + return refs; +} + +static void binder_deferred_release(struct binder_proc *proc) +{ + struct binder_transaction *t; + struct rb_node *n; + int threads, nodes, incoming_refs, outgoing_refs, buffers, + active_transactions, page_count; + + BUG_ON(proc->vma); + BUG_ON(proc->files); + + hlist_del(&proc->proc_node); + + if (binder_context_mgr_node && binder_context_mgr_node->proc == proc) { + binder_debug(BINDER_DEBUG_DEAD_BINDER, + "%s: %d context_mgr_node gone\n", + __func__, proc->pid); + binder_context_mgr_node = NULL; + } + + threads = 0; + active_transactions = 0; + while ((n = rb_first(&proc->threads))) { + struct binder_thread *thread; + + thread = rb_entry(n, struct binder_thread, rb_node); + threads++; + active_transactions += binder_free_thread(proc, thread); + } + + nodes = 0; + incoming_refs = 0; + while ((n = rb_first(&proc->nodes))) { + struct binder_node *node; + + node = rb_entry(n, struct binder_node, rb_node); + nodes++; + rb_erase(&node->rb_node, &proc->nodes); + incoming_refs = binder_node_release(node, incoming_refs); + } + + outgoing_refs = 0; + while ((n = rb_first(&proc->refs_by_desc))) { + struct binder_ref *ref; + + ref = rb_entry(n, struct binder_ref, rb_node_desc); + outgoing_refs++; + binder_delete_ref(ref); + } + + binder_release_work(&proc->todo); + binder_release_work(&proc->delivered_death); + + buffers = 0; + while ((n = rb_first(&proc->allocated_buffers))) { + struct binder_buffer *buffer; + + buffer = rb_entry(n, struct binder_buffer, rb_node); + + t = buffer->transaction; + if (t) { + t->buffer = NULL; + buffer->transaction = NULL; + pr_err("release proc %d, transaction %d, not freed\n", + proc->pid, t->debug_id); + /*BUG();*/ + } + + binder_free_buf(proc, buffer); + buffers++; + } + + binder_stats_deleted(BINDER_STAT_PROC); + + page_count = 0; + if (proc->pages) { + int i; + + for (i = 0; i < proc->buffer_size / PAGE_SIZE; i++) { + void *page_addr; + + if (!proc->pages[i]) + continue; + + page_addr = proc->buffer + i * PAGE_SIZE; + binder_debug(BINDER_DEBUG_BUFFER_ALLOC, + "%s: %d: page %d at %p not freed\n", + __func__, proc->pid, i, page_addr); + unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE); + __free_page(proc->pages[i]); + page_count++; + } + kfree(proc->pages); + vfree(proc->buffer); + } + + put_task_struct(proc->tsk); + + binder_debug(BINDER_DEBUG_OPEN_CLOSE, + "%s: %d threads %d, nodes %d (ref %d), refs %d, active transactions %d, buffers %d, pages %d\n", + __func__, proc->pid, threads, nodes, incoming_refs, + outgoing_refs, active_transactions, buffers, page_count); + + kfree(proc); +} + +static void binder_deferred_func(struct work_struct *work) +{ + struct binder_proc *proc; + struct files_struct *files; + + int defer; + + do { + binder_lock(__func__); + mutex_lock(&binder_deferred_lock); + if (!hlist_empty(&binder_deferred_list)) { + proc = hlist_entry(binder_deferred_list.first, + struct binder_proc, deferred_work_node); + hlist_del_init(&proc->deferred_work_node); + defer = proc->deferred_work; + proc->deferred_work = 0; + } else { + proc = NULL; + defer = 0; + } + mutex_unlock(&binder_deferred_lock); + + files = NULL; + if (defer & BINDER_DEFERRED_PUT_FILES) { + files = proc->files; + if (files) + proc->files = NULL; + } + + if (defer & BINDER_DEFERRED_FLUSH) + binder_deferred_flush(proc); + + if (defer & BINDER_DEFERRED_RELEASE) + binder_deferred_release(proc); /* frees proc */ + + binder_unlock(__func__); + if (files) + put_files_struct(files); + } while (proc); +} +static DECLARE_WORK(binder_deferred_work, binder_deferred_func); + +static void +binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer) +{ + mutex_lock(&binder_deferred_lock); + proc->deferred_work |= defer; + if (hlist_unhashed(&proc->deferred_work_node)) { + hlist_add_head(&proc->deferred_work_node, + &binder_deferred_list); + queue_work(binder_deferred_workqueue, &binder_deferred_work); + } + mutex_unlock(&binder_deferred_lock); +} + +static void print_binder_transaction(struct seq_file *m, const char *prefix, + struct binder_transaction *t) +{ + seq_printf(m, + "%s %d: %p from %d:%d to %d:%d code %x flags %x pri %ld r%d", + prefix, t->debug_id, t, + t->from ? t->from->proc->pid : 0, + t->from ? t->from->pid : 0, + t->to_proc ? t->to_proc->pid : 0, + t->to_thread ? t->to_thread->pid : 0, + t->code, t->flags, t->priority, t->need_reply); + if (t->buffer == NULL) { + seq_puts(m, " buffer free\n"); + return; + } + if (t->buffer->target_node) + seq_printf(m, " node %d", + t->buffer->target_node->debug_id); + seq_printf(m, " size %zd:%zd data %p\n", + t->buffer->data_size, t->buffer->offsets_size, + t->buffer->data); +} + +static void print_binder_buffer(struct seq_file *m, const char *prefix, + struct binder_buffer *buffer) +{ + seq_printf(m, "%s %d: %p size %zd:%zd %s\n", + prefix, buffer->debug_id, buffer->data, + buffer->data_size, buffer->offsets_size, + buffer->transaction ? "active" : "delivered"); +} + +static void print_binder_work(struct seq_file *m, const char *prefix, + const char *transaction_prefix, + struct binder_work *w) +{ + struct binder_node *node; + struct binder_transaction *t; + + switch (w->type) { + case BINDER_WORK_TRANSACTION: + t = container_of(w, struct binder_transaction, work); + print_binder_transaction(m, transaction_prefix, t); + break; + case BINDER_WORK_TRANSACTION_COMPLETE: + seq_printf(m, "%stransaction complete\n", prefix); + break; + case BINDER_WORK_NODE: + node = container_of(w, struct binder_node, work); + seq_printf(m, "%snode work %d: u%016llx c%016llx\n", + prefix, node->debug_id, + (u64)node->ptr, (u64)node->cookie); + break; + case BINDER_WORK_DEAD_BINDER: + seq_printf(m, "%shas dead binder\n", prefix); + break; + case BINDER_WORK_DEAD_BINDER_AND_CLEAR: + seq_printf(m, "%shas cleared dead binder\n", prefix); + break; + case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: + seq_printf(m, "%shas cleared death notification\n", prefix); + break; + default: + seq_printf(m, "%sunknown work: type %d\n", prefix, w->type); + break; + } +} + +static void print_binder_thread(struct seq_file *m, + struct binder_thread *thread, + int print_always) +{ + struct binder_transaction *t; + struct binder_work *w; + size_t start_pos = m->count; + size_t header_pos; + + seq_printf(m, " thread %d: l %02x\n", thread->pid, thread->looper); + header_pos = m->count; + t = thread->transaction_stack; + while (t) { + if (t->from == thread) { + print_binder_transaction(m, + " outgoing transaction", t); + t = t->from_parent; + } else if (t->to_thread == thread) { + print_binder_transaction(m, + " incoming transaction", t); + t = t->to_parent; + } else { + print_binder_transaction(m, " bad transaction", t); + t = NULL; + } + } + list_for_each_entry(w, &thread->todo, entry) { + print_binder_work(m, " ", " pending transaction", w); + } + if (!print_always && m->count == header_pos) + m->count = start_pos; +} + +static void print_binder_node(struct seq_file *m, struct binder_node *node) +{ + struct binder_ref *ref; + struct binder_work *w; + int count; + + count = 0; + hlist_for_each_entry(ref, &node->refs, node_entry) + count++; + + seq_printf(m, " node %d: u%016llx c%016llx hs %d hw %d ls %d lw %d is %d iw %d", + node->debug_id, (u64)node->ptr, (u64)node->cookie, + node->has_strong_ref, node->has_weak_ref, + node->local_strong_refs, node->local_weak_refs, + node->internal_strong_refs, count); + if (count) { + seq_puts(m, " proc"); + hlist_for_each_entry(ref, &node->refs, node_entry) + seq_printf(m, " %d", ref->proc->pid); + } + seq_puts(m, "\n"); + list_for_each_entry(w, &node->async_todo, entry) + print_binder_work(m, " ", + " pending async transaction", w); +} + +static void print_binder_ref(struct seq_file *m, struct binder_ref *ref) +{ + seq_printf(m, " ref %d: desc %d %snode %d s %d w %d d %p\n", + ref->debug_id, ref->desc, ref->node->proc ? "" : "dead ", + ref->node->debug_id, ref->strong, ref->weak, ref->death); +} + +static void print_binder_proc(struct seq_file *m, + struct binder_proc *proc, int print_all) +{ + struct binder_work *w; + struct rb_node *n; + size_t start_pos = m->count; + size_t header_pos; + + seq_printf(m, "proc %d\n", proc->pid); + header_pos = m->count; + + for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) + print_binder_thread(m, rb_entry(n, struct binder_thread, + rb_node), print_all); + for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) { + struct binder_node *node = rb_entry(n, struct binder_node, + rb_node); + if (print_all || node->has_async_transaction) + print_binder_node(m, node); + } + if (print_all) { + for (n = rb_first(&proc->refs_by_desc); + n != NULL; + n = rb_next(n)) + print_binder_ref(m, rb_entry(n, struct binder_ref, + rb_node_desc)); + } + for (n = rb_first(&proc->allocated_buffers); n != NULL; n = rb_next(n)) + print_binder_buffer(m, " buffer", + rb_entry(n, struct binder_buffer, rb_node)); + list_for_each_entry(w, &proc->todo, entry) + print_binder_work(m, " ", " pending transaction", w); + list_for_each_entry(w, &proc->delivered_death, entry) { + seq_puts(m, " has delivered dead binder\n"); + break; + } + if (!print_all && m->count == header_pos) + m->count = start_pos; +} + +static const char * const binder_return_strings[] = { + "BR_ERROR", + "BR_OK", + "BR_TRANSACTION", + "BR_REPLY", + "BR_ACQUIRE_RESULT", + "BR_DEAD_REPLY", + "BR_TRANSACTION_COMPLETE", + "BR_INCREFS", + "BR_ACQUIRE", + "BR_RELEASE", + "BR_DECREFS", + "BR_ATTEMPT_ACQUIRE", + "BR_NOOP", + "BR_SPAWN_LOOPER", + "BR_FINISHED", + "BR_DEAD_BINDER", + "BR_CLEAR_DEATH_NOTIFICATION_DONE", + "BR_FAILED_REPLY" +}; + +static const char * const binder_command_strings[] = { + "BC_TRANSACTION", + "BC_REPLY", + "BC_ACQUIRE_RESULT", + "BC_FREE_BUFFER", + "BC_INCREFS", + "BC_ACQUIRE", + "BC_RELEASE", + "BC_DECREFS", + "BC_INCREFS_DONE", + "BC_ACQUIRE_DONE", + "BC_ATTEMPT_ACQUIRE", + "BC_REGISTER_LOOPER", + "BC_ENTER_LOOPER", + "BC_EXIT_LOOPER", + "BC_REQUEST_DEATH_NOTIFICATION", + "BC_CLEAR_DEATH_NOTIFICATION", + "BC_DEAD_BINDER_DONE" +}; + +static const char * const binder_objstat_strings[] = { + "proc", + "thread", + "node", + "ref", + "death", + "transaction", + "transaction_complete" +}; + +static void print_binder_stats(struct seq_file *m, const char *prefix, + struct binder_stats *stats) +{ + int i; + + BUILD_BUG_ON(ARRAY_SIZE(stats->bc) != + ARRAY_SIZE(binder_command_strings)); + for (i = 0; i < ARRAY_SIZE(stats->bc); i++) { + if (stats->bc[i]) + seq_printf(m, "%s%s: %d\n", prefix, + binder_command_strings[i], stats->bc[i]); + } + + BUILD_BUG_ON(ARRAY_SIZE(stats->br) != + ARRAY_SIZE(binder_return_strings)); + for (i = 0; i < ARRAY_SIZE(stats->br); i++) { + if (stats->br[i]) + seq_printf(m, "%s%s: %d\n", prefix, + binder_return_strings[i], stats->br[i]); + } + + BUILD_BUG_ON(ARRAY_SIZE(stats->obj_created) != + ARRAY_SIZE(binder_objstat_strings)); + BUILD_BUG_ON(ARRAY_SIZE(stats->obj_created) != + ARRAY_SIZE(stats->obj_deleted)); + for (i = 0; i < ARRAY_SIZE(stats->obj_created); i++) { + if (stats->obj_created[i] || stats->obj_deleted[i]) + seq_printf(m, "%s%s: active %d total %d\n", prefix, + binder_objstat_strings[i], + stats->obj_created[i] - stats->obj_deleted[i], + stats->obj_created[i]); + } +} + +static void print_binder_proc_stats(struct seq_file *m, + struct binder_proc *proc) +{ + struct binder_work *w; + struct rb_node *n; + int count, strong, weak; + + seq_printf(m, "proc %d\n", proc->pid); + count = 0; + for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) + count++; + seq_printf(m, " threads: %d\n", count); + seq_printf(m, " requested threads: %d+%d/%d\n" + " ready threads %d\n" + " free async space %zd\n", proc->requested_threads, + proc->requested_threads_started, proc->max_threads, + proc->ready_threads, proc->free_async_space); + count = 0; + for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) + count++; + seq_printf(m, " nodes: %d\n", count); + count = 0; + strong = 0; + weak = 0; + for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) { + struct binder_ref *ref = rb_entry(n, struct binder_ref, + rb_node_desc); + count++; + strong += ref->strong; + weak += ref->weak; + } + seq_printf(m, " refs: %d s %d w %d\n", count, strong, weak); + + count = 0; + for (n = rb_first(&proc->allocated_buffers); n != NULL; n = rb_next(n)) + count++; + seq_printf(m, " buffers: %d\n", count); + + count = 0; + list_for_each_entry(w, &proc->todo, entry) { + switch (w->type) { + case BINDER_WORK_TRANSACTION: + count++; + break; + default: + break; + } + } + seq_printf(m, " pending transactions: %d\n", count); + + print_binder_stats(m, " ", &proc->stats); +} + + +static int binder_state_show(struct seq_file *m, void *unused) +{ + struct binder_proc *proc; + struct binder_node *node; + int do_lock = !binder_debug_no_lock; + + if (do_lock) + binder_lock(__func__); + + seq_puts(m, "binder state:\n"); + + if (!hlist_empty(&binder_dead_nodes)) + seq_puts(m, "dead nodes:\n"); + hlist_for_each_entry(node, &binder_dead_nodes, dead_node) + print_binder_node(m, node); + + hlist_for_each_entry(proc, &binder_procs, proc_node) + print_binder_proc(m, proc, 1); + if (do_lock) + binder_unlock(__func__); + return 0; +} + +static int binder_stats_show(struct seq_file *m, void *unused) +{ + struct binder_proc *proc; + int do_lock = !binder_debug_no_lock; + + if (do_lock) + binder_lock(__func__); + + seq_puts(m, "binder stats:\n"); + + print_binder_stats(m, "", &binder_stats); + + hlist_for_each_entry(proc, &binder_procs, proc_node) + print_binder_proc_stats(m, proc); + if (do_lock) + binder_unlock(__func__); + return 0; +} + +static int binder_transactions_show(struct seq_file *m, void *unused) +{ + struct binder_proc *proc; + int do_lock = !binder_debug_no_lock; + + if (do_lock) + binder_lock(__func__); + + seq_puts(m, "binder transactions:\n"); + hlist_for_each_entry(proc, &binder_procs, proc_node) + print_binder_proc(m, proc, 0); + if (do_lock) + binder_unlock(__func__); + return 0; +} + +static int binder_proc_show(struct seq_file *m, void *unused) +{ + struct binder_proc *proc = m->private; + int do_lock = !binder_debug_no_lock; + + if (do_lock) + binder_lock(__func__); + seq_puts(m, "binder proc state:\n"); + print_binder_proc(m, proc, 1); + if (do_lock) + binder_unlock(__func__); + return 0; +} + +static void print_binder_transaction_log_entry(struct seq_file *m, + struct binder_transaction_log_entry *e) +{ + seq_printf(m, + "%d: %s from %d:%d to %d:%d node %d handle %d size %d:%d\n", + e->debug_id, (e->call_type == 2) ? "reply" : + ((e->call_type == 1) ? "async" : "call "), e->from_proc, + e->from_thread, e->to_proc, e->to_thread, e->to_node, + e->target_handle, e->data_size, e->offsets_size); +} + +static int binder_transaction_log_show(struct seq_file *m, void *unused) +{ + struct binder_transaction_log *log = m->private; + int i; + + if (log->full) { + for (i = log->next; i < ARRAY_SIZE(log->entry); i++) + print_binder_transaction_log_entry(m, &log->entry[i]); + } + for (i = 0; i < log->next; i++) + print_binder_transaction_log_entry(m, &log->entry[i]); + return 0; +} + +static const struct file_operations binder_fops = { + .owner = THIS_MODULE, + .poll = binder_poll, + .unlocked_ioctl = binder_ioctl, + .compat_ioctl = binder_ioctl, + .mmap = binder_mmap, + .open = binder_open, + .flush = binder_flush, + .release = binder_release, +}; + +static struct miscdevice binder_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "binder", + .fops = &binder_fops +}; + +BINDER_DEBUG_ENTRY(state); +BINDER_DEBUG_ENTRY(stats); +BINDER_DEBUG_ENTRY(transactions); +BINDER_DEBUG_ENTRY(transaction_log); + +static int __init binder_init(void) +{ + int ret; + + binder_deferred_workqueue = create_singlethread_workqueue("binder"); + if (!binder_deferred_workqueue) + return -ENOMEM; + + binder_debugfs_dir_entry_root = debugfs_create_dir("binder", NULL); + if (binder_debugfs_dir_entry_root) + binder_debugfs_dir_entry_proc = debugfs_create_dir("proc", + binder_debugfs_dir_entry_root); + ret = misc_register(&binder_miscdev); + if (binder_debugfs_dir_entry_root) { + debugfs_create_file("state", + S_IRUGO, + binder_debugfs_dir_entry_root, + NULL, + &binder_state_fops); + debugfs_create_file("stats", + S_IRUGO, + binder_debugfs_dir_entry_root, + NULL, + &binder_stats_fops); + debugfs_create_file("transactions", + S_IRUGO, + binder_debugfs_dir_entry_root, + NULL, + &binder_transactions_fops); + debugfs_create_file("transaction_log", + S_IRUGO, + binder_debugfs_dir_entry_root, + &binder_transaction_log, + &binder_transaction_log_fops); + debugfs_create_file("failed_transaction_log", + S_IRUGO, + binder_debugfs_dir_entry_root, + &binder_transaction_log_failed, + &binder_transaction_log_fops); + } + return ret; +} + +device_initcall(binder_init); + +#define CREATE_TRACE_POINTS +#include "binder_trace.h" + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/android/binder.h b/drivers/android/binder.h new file mode 100644 index 000000000000..5dc6a66b0665 --- /dev/null +++ b/drivers/android/binder.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2008 Google, Inc. + * + * Based on, but no longer compatible with, the original + * OpenBinder.org binder driver interface, which is: + * + * Copyright (c) 2005 Palmsource, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_BINDER_H +#define _LINUX_BINDER_H + +#ifdef CONFIG_ANDROID_BINDER_IPC_32BIT +#define BINDER_IPC_32BIT 1 +#endif + +#include + +#endif /* _LINUX_BINDER_H */ + diff --git a/drivers/android/binder_trace.h b/drivers/android/binder_trace.h new file mode 100644 index 000000000000..7f20f3dc8369 --- /dev/null +++ b/drivers/android/binder_trace.h @@ -0,0 +1,329 @@ +/* + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM binder + +#if !defined(_BINDER_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _BINDER_TRACE_H + +#include + +struct binder_buffer; +struct binder_node; +struct binder_proc; +struct binder_ref; +struct binder_thread; +struct binder_transaction; + +TRACE_EVENT(binder_ioctl, + TP_PROTO(unsigned int cmd, unsigned long arg), + TP_ARGS(cmd, arg), + + TP_STRUCT__entry( + __field(unsigned int, cmd) + __field(unsigned long, arg) + ), + TP_fast_assign( + __entry->cmd = cmd; + __entry->arg = arg; + ), + TP_printk("cmd=0x%x arg=0x%lx", __entry->cmd, __entry->arg) +); + +DECLARE_EVENT_CLASS(binder_lock_class, + TP_PROTO(const char *tag), + TP_ARGS(tag), + TP_STRUCT__entry( + __field(const char *, tag) + ), + TP_fast_assign( + __entry->tag = tag; + ), + TP_printk("tag=%s", __entry->tag) +); + +#define DEFINE_BINDER_LOCK_EVENT(name) \ +DEFINE_EVENT(binder_lock_class, name, \ + TP_PROTO(const char *func), \ + TP_ARGS(func)) + +DEFINE_BINDER_LOCK_EVENT(binder_lock); +DEFINE_BINDER_LOCK_EVENT(binder_locked); +DEFINE_BINDER_LOCK_EVENT(binder_unlock); + +DECLARE_EVENT_CLASS(binder_function_return_class, + TP_PROTO(int ret), + TP_ARGS(ret), + TP_STRUCT__entry( + __field(int, ret) + ), + TP_fast_assign( + __entry->ret = ret; + ), + TP_printk("ret=%d", __entry->ret) +); + +#define DEFINE_BINDER_FUNCTION_RETURN_EVENT(name) \ +DEFINE_EVENT(binder_function_return_class, name, \ + TP_PROTO(int ret), \ + TP_ARGS(ret)) + +DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_ioctl_done); +DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_write_done); +DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_read_done); + +TRACE_EVENT(binder_wait_for_work, + TP_PROTO(bool proc_work, bool transaction_stack, bool thread_todo), + TP_ARGS(proc_work, transaction_stack, thread_todo), + + TP_STRUCT__entry( + __field(bool, proc_work) + __field(bool, transaction_stack) + __field(bool, thread_todo) + ), + TP_fast_assign( + __entry->proc_work = proc_work; + __entry->transaction_stack = transaction_stack; + __entry->thread_todo = thread_todo; + ), + TP_printk("proc_work=%d transaction_stack=%d thread_todo=%d", + __entry->proc_work, __entry->transaction_stack, + __entry->thread_todo) +); + +TRACE_EVENT(binder_transaction, + TP_PROTO(bool reply, struct binder_transaction *t, + struct binder_node *target_node), + TP_ARGS(reply, t, target_node), + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, target_node) + __field(int, to_proc) + __field(int, to_thread) + __field(int, reply) + __field(unsigned int, code) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->target_node = target_node ? target_node->debug_id : 0; + __entry->to_proc = t->to_proc->pid; + __entry->to_thread = t->to_thread ? t->to_thread->pid : 0; + __entry->reply = reply; + __entry->code = t->code; + __entry->flags = t->flags; + ), + TP_printk("transaction=%d dest_node=%d dest_proc=%d dest_thread=%d reply=%d flags=0x%x code=0x%x", + __entry->debug_id, __entry->target_node, + __entry->to_proc, __entry->to_thread, + __entry->reply, __entry->flags, __entry->code) +); + +TRACE_EVENT(binder_transaction_received, + TP_PROTO(struct binder_transaction *t), + TP_ARGS(t), + + TP_STRUCT__entry( + __field(int, debug_id) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + ), + TP_printk("transaction=%d", __entry->debug_id) +); + +TRACE_EVENT(binder_transaction_node_to_ref, + TP_PROTO(struct binder_transaction *t, struct binder_node *node, + struct binder_ref *ref), + TP_ARGS(t, node, ref), + + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, node_debug_id) + __field(binder_uintptr_t, node_ptr) + __field(int, ref_debug_id) + __field(uint32_t, ref_desc) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->node_debug_id = node->debug_id; + __entry->node_ptr = node->ptr; + __entry->ref_debug_id = ref->debug_id; + __entry->ref_desc = ref->desc; + ), + TP_printk("transaction=%d node=%d src_ptr=0x%016llx ==> dest_ref=%d dest_desc=%d", + __entry->debug_id, __entry->node_debug_id, + (u64)__entry->node_ptr, + __entry->ref_debug_id, __entry->ref_desc) +); + +TRACE_EVENT(binder_transaction_ref_to_node, + TP_PROTO(struct binder_transaction *t, struct binder_ref *ref), + TP_ARGS(t, ref), + + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, ref_debug_id) + __field(uint32_t, ref_desc) + __field(int, node_debug_id) + __field(binder_uintptr_t, node_ptr) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->ref_debug_id = ref->debug_id; + __entry->ref_desc = ref->desc; + __entry->node_debug_id = ref->node->debug_id; + __entry->node_ptr = ref->node->ptr; + ), + TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%016llx", + __entry->debug_id, __entry->node_debug_id, + __entry->ref_debug_id, __entry->ref_desc, + (u64)__entry->node_ptr) +); + +TRACE_EVENT(binder_transaction_ref_to_ref, + TP_PROTO(struct binder_transaction *t, struct binder_ref *src_ref, + struct binder_ref *dest_ref), + TP_ARGS(t, src_ref, dest_ref), + + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, node_debug_id) + __field(int, src_ref_debug_id) + __field(uint32_t, src_ref_desc) + __field(int, dest_ref_debug_id) + __field(uint32_t, dest_ref_desc) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->node_debug_id = src_ref->node->debug_id; + __entry->src_ref_debug_id = src_ref->debug_id; + __entry->src_ref_desc = src_ref->desc; + __entry->dest_ref_debug_id = dest_ref->debug_id; + __entry->dest_ref_desc = dest_ref->desc; + ), + TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ref=%d dest_desc=%d", + __entry->debug_id, __entry->node_debug_id, + __entry->src_ref_debug_id, __entry->src_ref_desc, + __entry->dest_ref_debug_id, __entry->dest_ref_desc) +); + +TRACE_EVENT(binder_transaction_fd, + TP_PROTO(struct binder_transaction *t, int src_fd, int dest_fd), + TP_ARGS(t, src_fd, dest_fd), + + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, src_fd) + __field(int, dest_fd) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->src_fd = src_fd; + __entry->dest_fd = dest_fd; + ), + TP_printk("transaction=%d src_fd=%d ==> dest_fd=%d", + __entry->debug_id, __entry->src_fd, __entry->dest_fd) +); + +DECLARE_EVENT_CLASS(binder_buffer_class, + TP_PROTO(struct binder_buffer *buf), + TP_ARGS(buf), + TP_STRUCT__entry( + __field(int, debug_id) + __field(size_t, data_size) + __field(size_t, offsets_size) + ), + TP_fast_assign( + __entry->debug_id = buf->debug_id; + __entry->data_size = buf->data_size; + __entry->offsets_size = buf->offsets_size; + ), + TP_printk("transaction=%d data_size=%zd offsets_size=%zd", + __entry->debug_id, __entry->data_size, __entry->offsets_size) +); + +DEFINE_EVENT(binder_buffer_class, binder_transaction_alloc_buf, + TP_PROTO(struct binder_buffer *buffer), + TP_ARGS(buffer)); + +DEFINE_EVENT(binder_buffer_class, binder_transaction_buffer_release, + TP_PROTO(struct binder_buffer *buffer), + TP_ARGS(buffer)); + +DEFINE_EVENT(binder_buffer_class, binder_transaction_failed_buffer_release, + TP_PROTO(struct binder_buffer *buffer), + TP_ARGS(buffer)); + +TRACE_EVENT(binder_update_page_range, + TP_PROTO(struct binder_proc *proc, bool allocate, + void *start, void *end), + TP_ARGS(proc, allocate, start, end), + TP_STRUCT__entry( + __field(int, proc) + __field(bool, allocate) + __field(size_t, offset) + __field(size_t, size) + ), + TP_fast_assign( + __entry->proc = proc->pid; + __entry->allocate = allocate; + __entry->offset = start - proc->buffer; + __entry->size = end - start; + ), + TP_printk("proc=%d allocate=%d offset=%zu size=%zu", + __entry->proc, __entry->allocate, + __entry->offset, __entry->size) +); + +TRACE_EVENT(binder_command, + TP_PROTO(uint32_t cmd), + TP_ARGS(cmd), + TP_STRUCT__entry( + __field(uint32_t, cmd) + ), + TP_fast_assign( + __entry->cmd = cmd; + ), + TP_printk("cmd=0x%x %s", + __entry->cmd, + _IOC_NR(__entry->cmd) < ARRAY_SIZE(binder_command_strings) ? + binder_command_strings[_IOC_NR(__entry->cmd)] : + "unknown") +); + +TRACE_EVENT(binder_return, + TP_PROTO(uint32_t cmd), + TP_ARGS(cmd), + TP_STRUCT__entry( + __field(uint32_t, cmd) + ), + TP_fast_assign( + __entry->cmd = cmd; + ), + TP_printk("cmd=0x%x %s", + __entry->cmd, + _IOC_NR(__entry->cmd) < ARRAY_SIZE(binder_return_strings) ? + binder_return_strings[_IOC_NR(__entry->cmd)] : + "unknown") +); + +#endif /* _BINDER_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE binder_trace +#include diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig index 45e3380accbc..a817875bbf85 100644 --- a/drivers/staging/android/Kconfig +++ b/drivers/staging/android/Kconfig @@ -8,29 +8,6 @@ config ANDROID if ANDROID -config ANDROID_BINDER_IPC - bool "Android Binder IPC Driver" - default n - ---help--- - Binder is used in Android for both communication between processes, - and remote method invocation. - - This means one Android process can call a method/routine in another - Android process, using Binder to identify, invoke and pass arguments - between said processes. - -config ANDROID_BINDER_IPC_32BIT - bool "Use old 32-bit binder api" - depends on !64BIT - ---help--- - The Binder API has been changed to support both 32 and 64bit - applications in a mixed environment. - - Enable this to support an old 32-bit Android user-space (v4.4 and - earlier). - - Note that enabling this will break newer Android user-space. - config ASHMEM bool "Enable the Anonymous Shared Memory Subsystem" default n diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile index 907b62f56203..c223ef72bb38 100644 --- a/drivers/staging/android/Makefile +++ b/drivers/staging/android/Makefile @@ -3,7 +3,6 @@ ccflags-y += -I$(src) # needed for trace events obj-y += ion/ obj-$(CONFIG_FIQ_DEBUGGER) += fiq_debugger/ -obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o obj-$(CONFIG_ASHMEM) += ashmem.o obj-$(CONFIG_ANDROID_LOGGER) += logger.o obj-$(CONFIG_ANDROID_TIMED_OUTPUT) += timed_output.o diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c deleted file mode 100644 index b44b9398810c..000000000000 --- a/drivers/staging/android/binder.c +++ /dev/null @@ -1,3675 +0,0 @@ -/* binder.c - * - * Android IPC Subsystem - * - * Copyright (C) 2007-2008 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "binder.h" -#include "binder_trace.h" - -static DEFINE_MUTEX(binder_main_lock); -static DEFINE_MUTEX(binder_deferred_lock); -static DEFINE_MUTEX(binder_mmap_lock); - -static HLIST_HEAD(binder_procs); -static HLIST_HEAD(binder_deferred_list); -static HLIST_HEAD(binder_dead_nodes); - -static struct dentry *binder_debugfs_dir_entry_root; -static struct dentry *binder_debugfs_dir_entry_proc; -static struct binder_node *binder_context_mgr_node; -static kuid_t binder_context_mgr_uid = INVALID_UID; -static int binder_last_id; -static struct workqueue_struct *binder_deferred_workqueue; - -#define BINDER_DEBUG_ENTRY(name) \ -static int binder_##name##_open(struct inode *inode, struct file *file) \ -{ \ - return single_open(file, binder_##name##_show, inode->i_private); \ -} \ -\ -static const struct file_operations binder_##name##_fops = { \ - .owner = THIS_MODULE, \ - .open = binder_##name##_open, \ - .read = seq_read, \ - .llseek = seq_lseek, \ - .release = single_release, \ -} - -static int binder_proc_show(struct seq_file *m, void *unused); -BINDER_DEBUG_ENTRY(proc); - -/* This is only defined in include/asm-arm/sizes.h */ -#ifndef SZ_1K -#define SZ_1K 0x400 -#endif - -#ifndef SZ_4M -#define SZ_4M 0x400000 -#endif - -#define FORBIDDEN_MMAP_FLAGS (VM_WRITE) - -#define BINDER_SMALL_BUF_SIZE (PAGE_SIZE * 64) - -enum { - BINDER_DEBUG_USER_ERROR = 1U << 0, - BINDER_DEBUG_FAILED_TRANSACTION = 1U << 1, - BINDER_DEBUG_DEAD_TRANSACTION = 1U << 2, - BINDER_DEBUG_OPEN_CLOSE = 1U << 3, - BINDER_DEBUG_DEAD_BINDER = 1U << 4, - BINDER_DEBUG_DEATH_NOTIFICATION = 1U << 5, - BINDER_DEBUG_READ_WRITE = 1U << 6, - BINDER_DEBUG_USER_REFS = 1U << 7, - BINDER_DEBUG_THREADS = 1U << 8, - BINDER_DEBUG_TRANSACTION = 1U << 9, - BINDER_DEBUG_TRANSACTION_COMPLETE = 1U << 10, - BINDER_DEBUG_FREE_BUFFER = 1U << 11, - BINDER_DEBUG_INTERNAL_REFS = 1U << 12, - BINDER_DEBUG_BUFFER_ALLOC = 1U << 13, - BINDER_DEBUG_PRIORITY_CAP = 1U << 14, - BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 15, -}; -static uint32_t binder_debug_mask = BINDER_DEBUG_USER_ERROR | - BINDER_DEBUG_FAILED_TRANSACTION | BINDER_DEBUG_DEAD_TRANSACTION; -module_param_named(debug_mask, binder_debug_mask, uint, S_IWUSR | S_IRUGO); - -static bool binder_debug_no_lock; -module_param_named(proc_no_lock, binder_debug_no_lock, bool, S_IWUSR | S_IRUGO); - -static DECLARE_WAIT_QUEUE_HEAD(binder_user_error_wait); -static int binder_stop_on_user_error; - -static int binder_set_stop_on_user_error(const char *val, - struct kernel_param *kp) -{ - int ret; - - ret = param_set_int(val, kp); - if (binder_stop_on_user_error < 2) - wake_up(&binder_user_error_wait); - return ret; -} -module_param_call(stop_on_user_error, binder_set_stop_on_user_error, - param_get_int, &binder_stop_on_user_error, S_IWUSR | S_IRUGO); - -#define binder_debug(mask, x...) \ - do { \ - if (binder_debug_mask & mask) \ - pr_info(x); \ - } while (0) - -#define binder_user_error(x...) \ - do { \ - if (binder_debug_mask & BINDER_DEBUG_USER_ERROR) \ - pr_info(x); \ - if (binder_stop_on_user_error) \ - binder_stop_on_user_error = 2; \ - } while (0) - -enum binder_stat_types { - BINDER_STAT_PROC, - BINDER_STAT_THREAD, - BINDER_STAT_NODE, - BINDER_STAT_REF, - BINDER_STAT_DEATH, - BINDER_STAT_TRANSACTION, - BINDER_STAT_TRANSACTION_COMPLETE, - BINDER_STAT_COUNT -}; - -struct binder_stats { - int br[_IOC_NR(BR_FAILED_REPLY) + 1]; - int bc[_IOC_NR(BC_DEAD_BINDER_DONE) + 1]; - int obj_created[BINDER_STAT_COUNT]; - int obj_deleted[BINDER_STAT_COUNT]; -}; - -static struct binder_stats binder_stats; - -static inline void binder_stats_deleted(enum binder_stat_types type) -{ - binder_stats.obj_deleted[type]++; -} - -static inline void binder_stats_created(enum binder_stat_types type) -{ - binder_stats.obj_created[type]++; -} - -struct binder_transaction_log_entry { - int debug_id; - int call_type; - int from_proc; - int from_thread; - int target_handle; - int to_proc; - int to_thread; - int to_node; - int data_size; - int offsets_size; -}; -struct binder_transaction_log { - int next; - int full; - struct binder_transaction_log_entry entry[32]; -}; -static struct binder_transaction_log binder_transaction_log; -static struct binder_transaction_log binder_transaction_log_failed; - -static struct binder_transaction_log_entry *binder_transaction_log_add( - struct binder_transaction_log *log) -{ - struct binder_transaction_log_entry *e; - - e = &log->entry[log->next]; - memset(e, 0, sizeof(*e)); - log->next++; - if (log->next == ARRAY_SIZE(log->entry)) { - log->next = 0; - log->full = 1; - } - return e; -} - -struct binder_work { - struct list_head entry; - enum { - BINDER_WORK_TRANSACTION = 1, - BINDER_WORK_TRANSACTION_COMPLETE, - BINDER_WORK_NODE, - BINDER_WORK_DEAD_BINDER, - BINDER_WORK_DEAD_BINDER_AND_CLEAR, - BINDER_WORK_CLEAR_DEATH_NOTIFICATION, - } type; -}; - -struct binder_node { - int debug_id; - struct binder_work work; - union { - struct rb_node rb_node; - struct hlist_node dead_node; - }; - struct binder_proc *proc; - struct hlist_head refs; - int internal_strong_refs; - int local_weak_refs; - int local_strong_refs; - binder_uintptr_t ptr; - binder_uintptr_t cookie; - unsigned has_strong_ref:1; - unsigned pending_strong_ref:1; - unsigned has_weak_ref:1; - unsigned pending_weak_ref:1; - unsigned has_async_transaction:1; - unsigned accept_fds:1; - unsigned min_priority:8; - struct list_head async_todo; -}; - -struct binder_ref_death { - struct binder_work work; - binder_uintptr_t cookie; -}; - -struct binder_ref { - /* Lookups needed: */ - /* node + proc => ref (transaction) */ - /* desc + proc => ref (transaction, inc/dec ref) */ - /* node => refs + procs (proc exit) */ - int debug_id; - struct rb_node rb_node_desc; - struct rb_node rb_node_node; - struct hlist_node node_entry; - struct binder_proc *proc; - struct binder_node *node; - uint32_t desc; - int strong; - int weak; - struct binder_ref_death *death; -}; - -struct binder_buffer { - struct list_head entry; /* free and allocated entries by address */ - struct rb_node rb_node; /* free entry by size or allocated entry */ - /* by address */ - unsigned free:1; - unsigned allow_user_free:1; - unsigned async_transaction:1; - unsigned debug_id:29; - - struct binder_transaction *transaction; - - struct binder_node *target_node; - size_t data_size; - size_t offsets_size; - uint8_t data[0]; -}; - -enum binder_deferred_state { - BINDER_DEFERRED_PUT_FILES = 0x01, - BINDER_DEFERRED_FLUSH = 0x02, - BINDER_DEFERRED_RELEASE = 0x04, -}; - -struct binder_proc { - struct hlist_node proc_node; - struct rb_root threads; - struct rb_root nodes; - struct rb_root refs_by_desc; - struct rb_root refs_by_node; - int pid; - struct vm_area_struct *vma; - struct mm_struct *vma_vm_mm; - struct task_struct *tsk; - struct files_struct *files; - struct hlist_node deferred_work_node; - int deferred_work; - void *buffer; - ptrdiff_t user_buffer_offset; - - struct list_head buffers; - struct rb_root free_buffers; - struct rb_root allocated_buffers; - size_t free_async_space; - - struct page **pages; - size_t buffer_size; - uint32_t buffer_free; - struct list_head todo; - wait_queue_head_t wait; - struct binder_stats stats; - struct list_head delivered_death; - int max_threads; - int requested_threads; - int requested_threads_started; - int ready_threads; - long default_priority; - struct dentry *debugfs_entry; -}; - -enum { - BINDER_LOOPER_STATE_REGISTERED = 0x01, - BINDER_LOOPER_STATE_ENTERED = 0x02, - BINDER_LOOPER_STATE_EXITED = 0x04, - BINDER_LOOPER_STATE_INVALID = 0x08, - BINDER_LOOPER_STATE_WAITING = 0x10, - BINDER_LOOPER_STATE_NEED_RETURN = 0x20 -}; - -struct binder_thread { - struct binder_proc *proc; - struct rb_node rb_node; - int pid; - int looper; - struct binder_transaction *transaction_stack; - struct list_head todo; - uint32_t return_error; /* Write failed, return error code in read buf */ - uint32_t return_error2; /* Write failed, return error code in read */ - /* buffer. Used when sending a reply to a dead process that */ - /* we are also waiting on */ - wait_queue_head_t wait; - struct binder_stats stats; -}; - -struct binder_transaction { - int debug_id; - struct binder_work work; - struct binder_thread *from; - struct binder_transaction *from_parent; - struct binder_proc *to_proc; - struct binder_thread *to_thread; - struct binder_transaction *to_parent; - unsigned need_reply:1; - /* unsigned is_dead:1; */ /* not used at the moment */ - - struct binder_buffer *buffer; - unsigned int code; - unsigned int flags; - long priority; - long saved_priority; - kuid_t sender_euid; -}; - -static void -binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer); - -static int task_get_unused_fd_flags(struct binder_proc *proc, int flags) -{ - struct files_struct *files = proc->files; - unsigned long rlim_cur; - unsigned long irqs; - - if (files == NULL) - return -ESRCH; - - if (!lock_task_sighand(proc->tsk, &irqs)) - return -EMFILE; - - rlim_cur = task_rlimit(proc->tsk, RLIMIT_NOFILE); - unlock_task_sighand(proc->tsk, &irqs); - - return __alloc_fd(files, 0, rlim_cur, flags); -} - -/* - * copied from fd_install - */ -static void task_fd_install( - struct binder_proc *proc, unsigned int fd, struct file *file) -{ - if (proc->files) - __fd_install(proc->files, fd, file); -} - -/* - * copied from sys_close - */ -static long task_close_fd(struct binder_proc *proc, unsigned int fd) -{ - int retval; - - if (proc->files == NULL) - return -ESRCH; - - retval = __close_fd(proc->files, fd); - /* can't restart close syscall because file table entry was cleared */ - if (unlikely(retval == -ERESTARTSYS || - retval == -ERESTARTNOINTR || - retval == -ERESTARTNOHAND || - retval == -ERESTART_RESTARTBLOCK)) - retval = -EINTR; - - return retval; -} - -static inline void binder_lock(const char *tag) -{ - trace_binder_lock(tag); - mutex_lock(&binder_main_lock); - trace_binder_locked(tag); -} - -static inline void binder_unlock(const char *tag) -{ - trace_binder_unlock(tag); - mutex_unlock(&binder_main_lock); -} - -static void binder_set_nice(long nice) -{ - long min_nice; - - if (can_nice(current, nice)) { - set_user_nice(current, nice); - return; - } - min_nice = 20 - current->signal->rlim[RLIMIT_NICE].rlim_cur; - binder_debug(BINDER_DEBUG_PRIORITY_CAP, - "%d: nice value %ld not allowed use %ld instead\n", - current->pid, nice, min_nice); - set_user_nice(current, min_nice); - if (min_nice < 20) - return; - binder_user_error("%d RLIMIT_NICE not set\n", current->pid); -} - -static size_t binder_buffer_size(struct binder_proc *proc, - struct binder_buffer *buffer) -{ - if (list_is_last(&buffer->entry, &proc->buffers)) - return proc->buffer + proc->buffer_size - (void *)buffer->data; - return (size_t)list_entry(buffer->entry.next, - struct binder_buffer, entry) - (size_t)buffer->data; -} - -static void binder_insert_free_buffer(struct binder_proc *proc, - struct binder_buffer *new_buffer) -{ - struct rb_node **p = &proc->free_buffers.rb_node; - struct rb_node *parent = NULL; - struct binder_buffer *buffer; - size_t buffer_size; - size_t new_buffer_size; - - BUG_ON(!new_buffer->free); - - new_buffer_size = binder_buffer_size(proc, new_buffer); - - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: add free buffer, size %zd, at %p\n", - proc->pid, new_buffer_size, new_buffer); - - while (*p) { - parent = *p; - buffer = rb_entry(parent, struct binder_buffer, rb_node); - BUG_ON(!buffer->free); - - buffer_size = binder_buffer_size(proc, buffer); - - if (new_buffer_size < buffer_size) - p = &parent->rb_left; - else - p = &parent->rb_right; - } - rb_link_node(&new_buffer->rb_node, parent, p); - rb_insert_color(&new_buffer->rb_node, &proc->free_buffers); -} - -static void binder_insert_allocated_buffer(struct binder_proc *proc, - struct binder_buffer *new_buffer) -{ - struct rb_node **p = &proc->allocated_buffers.rb_node; - struct rb_node *parent = NULL; - struct binder_buffer *buffer; - - BUG_ON(new_buffer->free); - - while (*p) { - parent = *p; - buffer = rb_entry(parent, struct binder_buffer, rb_node); - BUG_ON(buffer->free); - - if (new_buffer < buffer) - p = &parent->rb_left; - else if (new_buffer > buffer) - p = &parent->rb_right; - else - BUG(); - } - rb_link_node(&new_buffer->rb_node, parent, p); - rb_insert_color(&new_buffer->rb_node, &proc->allocated_buffers); -} - -static struct binder_buffer *binder_buffer_lookup(struct binder_proc *proc, - uintptr_t user_ptr) -{ - struct rb_node *n = proc->allocated_buffers.rb_node; - struct binder_buffer *buffer; - struct binder_buffer *kern_ptr; - - kern_ptr = (struct binder_buffer *)(user_ptr - proc->user_buffer_offset - - offsetof(struct binder_buffer, data)); - - while (n) { - buffer = rb_entry(n, struct binder_buffer, rb_node); - BUG_ON(buffer->free); - - if (kern_ptr < buffer) - n = n->rb_left; - else if (kern_ptr > buffer) - n = n->rb_right; - else - return buffer; - } - return NULL; -} - -static int binder_update_page_range(struct binder_proc *proc, int allocate, - void *start, void *end, - struct vm_area_struct *vma) -{ - void *page_addr; - unsigned long user_page_addr; - struct vm_struct tmp_area; - struct page **page; - struct mm_struct *mm; - - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: %s pages %p-%p\n", proc->pid, - allocate ? "allocate" : "free", start, end); - - if (end <= start) - return 0; - - trace_binder_update_page_range(proc, allocate, start, end); - - if (vma) - mm = NULL; - else - mm = get_task_mm(proc->tsk); - - if (mm) { - down_write(&mm->mmap_sem); - vma = proc->vma; - if (vma && mm != proc->vma_vm_mm) { - pr_err("%d: vma mm and task mm mismatch\n", - proc->pid); - vma = NULL; - } - } - - if (allocate == 0) - goto free_range; - - if (vma == NULL) { - pr_err("%d: binder_alloc_buf failed to map pages in userspace, no vma\n", - proc->pid); - goto err_no_vma; - } - - for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { - int ret; - struct page **page_array_ptr; - - page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE]; - - BUG_ON(*page); - *page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); - if (*page == NULL) { - pr_err("%d: binder_alloc_buf failed for page at %p\n", - proc->pid, page_addr); - goto err_alloc_page_failed; - } - tmp_area.addr = page_addr; - tmp_area.size = PAGE_SIZE + PAGE_SIZE /* guard page? */; - page_array_ptr = page; - ret = map_vm_area(&tmp_area, PAGE_KERNEL, &page_array_ptr); - if (ret) { - pr_err("%d: binder_alloc_buf failed to map page at %p in kernel\n", - proc->pid, page_addr); - goto err_map_kernel_failed; - } - user_page_addr = - (uintptr_t)page_addr + proc->user_buffer_offset; - ret = vm_insert_page(vma, user_page_addr, page[0]); - if (ret) { - pr_err("%d: binder_alloc_buf failed to map page at %lx in userspace\n", - proc->pid, user_page_addr); - goto err_vm_insert_page_failed; - } - /* vm_insert_page does not seem to increment the refcount */ - } - if (mm) { - up_write(&mm->mmap_sem); - mmput(mm); - } - return 0; - -free_range: - for (page_addr = end - PAGE_SIZE; page_addr >= start; - page_addr -= PAGE_SIZE) { - page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE]; - if (vma) - zap_page_range(vma, (uintptr_t)page_addr + - proc->user_buffer_offset, PAGE_SIZE, NULL); -err_vm_insert_page_failed: - unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE); -err_map_kernel_failed: - __free_page(*page); - *page = NULL; -err_alloc_page_failed: - ; - } -err_no_vma: - if (mm) { - up_write(&mm->mmap_sem); - mmput(mm); - } - return -ENOMEM; -} - -static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, - size_t data_size, - size_t offsets_size, int is_async) -{ - struct rb_node *n = proc->free_buffers.rb_node; - struct binder_buffer *buffer; - size_t buffer_size; - struct rb_node *best_fit = NULL; - void *has_page_addr; - void *end_page_addr; - size_t size; - - if (proc->vma == NULL) { - pr_err("%d: binder_alloc_buf, no vma\n", - proc->pid); - return NULL; - } - - size = ALIGN(data_size, sizeof(void *)) + - ALIGN(offsets_size, sizeof(void *)); - - if (size < data_size || size < offsets_size) { - binder_user_error("%d: got transaction with invalid size %zd-%zd\n", - proc->pid, data_size, offsets_size); - return NULL; - } - - if (is_async && - proc->free_async_space < size + sizeof(struct binder_buffer)) { - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_alloc_buf size %zd failed, no async space left\n", - proc->pid, size); - return NULL; - } - - while (n) { - buffer = rb_entry(n, struct binder_buffer, rb_node); - BUG_ON(!buffer->free); - buffer_size = binder_buffer_size(proc, buffer); - - if (size < buffer_size) { - best_fit = n; - n = n->rb_left; - } else if (size > buffer_size) - n = n->rb_right; - else { - best_fit = n; - break; - } - } - if (best_fit == NULL) { - pr_err("%d: binder_alloc_buf size %zd failed, no address space\n", - proc->pid, size); - return NULL; - } - if (n == NULL) { - buffer = rb_entry(best_fit, struct binder_buffer, rb_node); - buffer_size = binder_buffer_size(proc, buffer); - } - - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_alloc_buf size %zd got buffer %p size %zd\n", - proc->pid, size, buffer, buffer_size); - - has_page_addr = - (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK); - if (n == NULL) { - if (size + sizeof(struct binder_buffer) + 4 >= buffer_size) - buffer_size = size; /* no room for other buffers */ - else - buffer_size = size + sizeof(struct binder_buffer); - } - end_page_addr = - (void *)PAGE_ALIGN((uintptr_t)buffer->data + buffer_size); - if (end_page_addr > has_page_addr) - end_page_addr = has_page_addr; - if (binder_update_page_range(proc, 1, - (void *)PAGE_ALIGN((uintptr_t)buffer->data), end_page_addr, NULL)) - return NULL; - - rb_erase(best_fit, &proc->free_buffers); - buffer->free = 0; - binder_insert_allocated_buffer(proc, buffer); - if (buffer_size != size) { - struct binder_buffer *new_buffer = (void *)buffer->data + size; - - list_add(&new_buffer->entry, &buffer->entry); - new_buffer->free = 1; - binder_insert_free_buffer(proc, new_buffer); - } - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_alloc_buf size %zd got %p\n", - proc->pid, size, buffer); - buffer->data_size = data_size; - buffer->offsets_size = offsets_size; - buffer->async_transaction = is_async; - if (is_async) { - proc->free_async_space -= size + sizeof(struct binder_buffer); - binder_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, - "%d: binder_alloc_buf size %zd async free %zd\n", - proc->pid, size, proc->free_async_space); - } - - return buffer; -} - -static void *buffer_start_page(struct binder_buffer *buffer) -{ - return (void *)((uintptr_t)buffer & PAGE_MASK); -} - -static void *buffer_end_page(struct binder_buffer *buffer) -{ - return (void *)(((uintptr_t)(buffer + 1) - 1) & PAGE_MASK); -} - -static void binder_delete_free_buffer(struct binder_proc *proc, - struct binder_buffer *buffer) -{ - struct binder_buffer *prev, *next = NULL; - int free_page_end = 1; - int free_page_start = 1; - - BUG_ON(proc->buffers.next == &buffer->entry); - prev = list_entry(buffer->entry.prev, struct binder_buffer, entry); - BUG_ON(!prev->free); - if (buffer_end_page(prev) == buffer_start_page(buffer)) { - free_page_start = 0; - if (buffer_end_page(prev) == buffer_end_page(buffer)) - free_page_end = 0; - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: merge free, buffer %p share page with %p\n", - proc->pid, buffer, prev); - } - - if (!list_is_last(&buffer->entry, &proc->buffers)) { - next = list_entry(buffer->entry.next, - struct binder_buffer, entry); - if (buffer_start_page(next) == buffer_end_page(buffer)) { - free_page_end = 0; - if (buffer_start_page(next) == - buffer_start_page(buffer)) - free_page_start = 0; - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: merge free, buffer %p share page with %p\n", - proc->pid, buffer, prev); - } - } - list_del(&buffer->entry); - if (free_page_start || free_page_end) { - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: merge free, buffer %p do not share page%s%s with %p or %p\n", - proc->pid, buffer, free_page_start ? "" : " end", - free_page_end ? "" : " start", prev, next); - binder_update_page_range(proc, 0, free_page_start ? - buffer_start_page(buffer) : buffer_end_page(buffer), - (free_page_end ? buffer_end_page(buffer) : - buffer_start_page(buffer)) + PAGE_SIZE, NULL); - } -} - -static void binder_free_buf(struct binder_proc *proc, - struct binder_buffer *buffer) -{ - size_t size, buffer_size; - - buffer_size = binder_buffer_size(proc, buffer); - - size = ALIGN(buffer->data_size, sizeof(void *)) + - ALIGN(buffer->offsets_size, sizeof(void *)); - - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_free_buf %p size %zd buffer_size %zd\n", - proc->pid, buffer, size, buffer_size); - - BUG_ON(buffer->free); - BUG_ON(size > buffer_size); - BUG_ON(buffer->transaction != NULL); - BUG_ON((void *)buffer < proc->buffer); - BUG_ON((void *)buffer > proc->buffer + proc->buffer_size); - - if (buffer->async_transaction) { - proc->free_async_space += size + sizeof(struct binder_buffer); - - binder_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, - "%d: binder_free_buf size %zd async free %zd\n", - proc->pid, size, proc->free_async_space); - } - - binder_update_page_range(proc, 0, - (void *)PAGE_ALIGN((uintptr_t)buffer->data), - (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK), - NULL); - rb_erase(&buffer->rb_node, &proc->allocated_buffers); - buffer->free = 1; - if (!list_is_last(&buffer->entry, &proc->buffers)) { - struct binder_buffer *next = list_entry(buffer->entry.next, - struct binder_buffer, entry); - - if (next->free) { - rb_erase(&next->rb_node, &proc->free_buffers); - binder_delete_free_buffer(proc, next); - } - } - if (proc->buffers.next != &buffer->entry) { - struct binder_buffer *prev = list_entry(buffer->entry.prev, - struct binder_buffer, entry); - - if (prev->free) { - binder_delete_free_buffer(proc, buffer); - rb_erase(&prev->rb_node, &proc->free_buffers); - buffer = prev; - } - } - binder_insert_free_buffer(proc, buffer); -} - -static struct binder_node *binder_get_node(struct binder_proc *proc, - binder_uintptr_t ptr) -{ - struct rb_node *n = proc->nodes.rb_node; - struct binder_node *node; - - while (n) { - node = rb_entry(n, struct binder_node, rb_node); - - if (ptr < node->ptr) - n = n->rb_left; - else if (ptr > node->ptr) - n = n->rb_right; - else - return node; - } - return NULL; -} - -static struct binder_node *binder_new_node(struct binder_proc *proc, - binder_uintptr_t ptr, - binder_uintptr_t cookie) -{ - struct rb_node **p = &proc->nodes.rb_node; - struct rb_node *parent = NULL; - struct binder_node *node; - - while (*p) { - parent = *p; - node = rb_entry(parent, struct binder_node, rb_node); - - if (ptr < node->ptr) - p = &(*p)->rb_left; - else if (ptr > node->ptr) - p = &(*p)->rb_right; - else - return NULL; - } - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (node == NULL) - return NULL; - binder_stats_created(BINDER_STAT_NODE); - rb_link_node(&node->rb_node, parent, p); - rb_insert_color(&node->rb_node, &proc->nodes); - node->debug_id = ++binder_last_id; - node->proc = proc; - node->ptr = ptr; - node->cookie = cookie; - node->work.type = BINDER_WORK_NODE; - INIT_LIST_HEAD(&node->work.entry); - INIT_LIST_HEAD(&node->async_todo); - binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%016llx c%016llx created\n", - proc->pid, current->pid, node->debug_id, - (u64)node->ptr, (u64)node->cookie); - return node; -} - -static int binder_inc_node(struct binder_node *node, int strong, int internal, - struct list_head *target_list) -{ - if (strong) { - if (internal) { - if (target_list == NULL && - node->internal_strong_refs == 0 && - !(node == binder_context_mgr_node && - node->has_strong_ref)) { - pr_err("invalid inc strong node for %d\n", - node->debug_id); - return -EINVAL; - } - node->internal_strong_refs++; - } else - node->local_strong_refs++; - if (!node->has_strong_ref && target_list) { - list_del_init(&node->work.entry); - list_add_tail(&node->work.entry, target_list); - } - } else { - if (!internal) - node->local_weak_refs++; - if (!node->has_weak_ref && list_empty(&node->work.entry)) { - if (target_list == NULL) { - pr_err("invalid inc weak node for %d\n", - node->debug_id); - return -EINVAL; - } - list_add_tail(&node->work.entry, target_list); - } - } - return 0; -} - -static int binder_dec_node(struct binder_node *node, int strong, int internal) -{ - if (strong) { - if (internal) - node->internal_strong_refs--; - else - node->local_strong_refs--; - if (node->local_strong_refs || node->internal_strong_refs) - return 0; - } else { - if (!internal) - node->local_weak_refs--; - if (node->local_weak_refs || !hlist_empty(&node->refs)) - return 0; - } - if (node->proc && (node->has_strong_ref || node->has_weak_ref)) { - if (list_empty(&node->work.entry)) { - list_add_tail(&node->work.entry, &node->proc->todo); - wake_up_interruptible(&node->proc->wait); - } - } else { - if (hlist_empty(&node->refs) && !node->local_strong_refs && - !node->local_weak_refs) { - list_del_init(&node->work.entry); - if (node->proc) { - rb_erase(&node->rb_node, &node->proc->nodes); - binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "refless node %d deleted\n", - node->debug_id); - } else { - hlist_del(&node->dead_node); - binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "dead node %d deleted\n", - node->debug_id); - } - kfree(node); - binder_stats_deleted(BINDER_STAT_NODE); - } - } - - return 0; -} - - -static struct binder_ref *binder_get_ref(struct binder_proc *proc, - uint32_t desc) -{ - struct rb_node *n = proc->refs_by_desc.rb_node; - struct binder_ref *ref; - - while (n) { - ref = rb_entry(n, struct binder_ref, rb_node_desc); - - if (desc < ref->desc) - n = n->rb_left; - else if (desc > ref->desc) - n = n->rb_right; - else - return ref; - } - return NULL; -} - -static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc, - struct binder_node *node) -{ - struct rb_node *n; - struct rb_node **p = &proc->refs_by_node.rb_node; - struct rb_node *parent = NULL; - struct binder_ref *ref, *new_ref; - - while (*p) { - parent = *p; - ref = rb_entry(parent, struct binder_ref, rb_node_node); - - if (node < ref->node) - p = &(*p)->rb_left; - else if (node > ref->node) - p = &(*p)->rb_right; - else - return ref; - } - new_ref = kzalloc(sizeof(*ref), GFP_KERNEL); - if (new_ref == NULL) - return NULL; - binder_stats_created(BINDER_STAT_REF); - new_ref->debug_id = ++binder_last_id; - new_ref->proc = proc; - new_ref->node = node; - rb_link_node(&new_ref->rb_node_node, parent, p); - rb_insert_color(&new_ref->rb_node_node, &proc->refs_by_node); - - new_ref->desc = (node == binder_context_mgr_node) ? 0 : 1; - for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) { - ref = rb_entry(n, struct binder_ref, rb_node_desc); - if (ref->desc > new_ref->desc) - break; - new_ref->desc = ref->desc + 1; - } - - p = &proc->refs_by_desc.rb_node; - while (*p) { - parent = *p; - ref = rb_entry(parent, struct binder_ref, rb_node_desc); - - if (new_ref->desc < ref->desc) - p = &(*p)->rb_left; - else if (new_ref->desc > ref->desc) - p = &(*p)->rb_right; - else - BUG(); - } - rb_link_node(&new_ref->rb_node_desc, parent, p); - rb_insert_color(&new_ref->rb_node_desc, &proc->refs_by_desc); - if (node) { - hlist_add_head(&new_ref->node_entry, &node->refs); - - binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d new ref %d desc %d for node %d\n", - proc->pid, new_ref->debug_id, new_ref->desc, - node->debug_id); - } else { - binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d new ref %d desc %d for dead node\n", - proc->pid, new_ref->debug_id, new_ref->desc); - } - return new_ref; -} - -static void binder_delete_ref(struct binder_ref *ref) -{ - binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d delete ref %d desc %d for node %d\n", - ref->proc->pid, ref->debug_id, ref->desc, - ref->node->debug_id); - - rb_erase(&ref->rb_node_desc, &ref->proc->refs_by_desc); - rb_erase(&ref->rb_node_node, &ref->proc->refs_by_node); - if (ref->strong) - binder_dec_node(ref->node, 1, 1); - hlist_del(&ref->node_entry); - binder_dec_node(ref->node, 0, 1); - if (ref->death) { - binder_debug(BINDER_DEBUG_DEAD_BINDER, - "%d delete ref %d desc %d has death notification\n", - ref->proc->pid, ref->debug_id, ref->desc); - list_del(&ref->death->work.entry); - kfree(ref->death); - binder_stats_deleted(BINDER_STAT_DEATH); - } - kfree(ref); - binder_stats_deleted(BINDER_STAT_REF); -} - -static int binder_inc_ref(struct binder_ref *ref, int strong, - struct list_head *target_list) -{ - int ret; - - if (strong) { - if (ref->strong == 0) { - ret = binder_inc_node(ref->node, 1, 1, target_list); - if (ret) - return ret; - } - ref->strong++; - } else { - if (ref->weak == 0) { - ret = binder_inc_node(ref->node, 0, 1, target_list); - if (ret) - return ret; - } - ref->weak++; - } - return 0; -} - - -static int binder_dec_ref(struct binder_ref *ref, int strong) -{ - if (strong) { - if (ref->strong == 0) { - binder_user_error("%d invalid dec strong, ref %d desc %d s %d w %d\n", - ref->proc->pid, ref->debug_id, - ref->desc, ref->strong, ref->weak); - return -EINVAL; - } - ref->strong--; - if (ref->strong == 0) { - int ret; - - ret = binder_dec_node(ref->node, strong, 1); - if (ret) - return ret; - } - } else { - if (ref->weak == 0) { - binder_user_error("%d invalid dec weak, ref %d desc %d s %d w %d\n", - ref->proc->pid, ref->debug_id, - ref->desc, ref->strong, ref->weak); - return -EINVAL; - } - ref->weak--; - } - if (ref->strong == 0 && ref->weak == 0) - binder_delete_ref(ref); - return 0; -} - -static void binder_pop_transaction(struct binder_thread *target_thread, - struct binder_transaction *t) -{ - if (target_thread) { - BUG_ON(target_thread->transaction_stack != t); - BUG_ON(target_thread->transaction_stack->from != target_thread); - target_thread->transaction_stack = - target_thread->transaction_stack->from_parent; - t->from = NULL; - } - t->need_reply = 0; - if (t->buffer) - t->buffer->transaction = NULL; - kfree(t); - binder_stats_deleted(BINDER_STAT_TRANSACTION); -} - -static void binder_send_failed_reply(struct binder_transaction *t, - uint32_t error_code) -{ - struct binder_thread *target_thread; - struct binder_transaction *next; - - BUG_ON(t->flags & TF_ONE_WAY); - while (1) { - target_thread = t->from; - if (target_thread) { - if (target_thread->return_error != BR_OK && - target_thread->return_error2 == BR_OK) { - target_thread->return_error2 = - target_thread->return_error; - target_thread->return_error = BR_OK; - } - if (target_thread->return_error == BR_OK) { - binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, - "send failed reply for transaction %d to %d:%d\n", - t->debug_id, - target_thread->proc->pid, - target_thread->pid); - - binder_pop_transaction(target_thread, t); - target_thread->return_error = error_code; - wake_up_interruptible(&target_thread->wait); - } else { - pr_err("reply failed, target thread, %d:%d, has error code %d already\n", - target_thread->proc->pid, - target_thread->pid, - target_thread->return_error); - } - return; - } - next = t->from_parent; - - binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, - "send failed reply for transaction %d, target dead\n", - t->debug_id); - - binder_pop_transaction(target_thread, t); - if (next == NULL) { - binder_debug(BINDER_DEBUG_DEAD_BINDER, - "reply failed, no target thread at root\n"); - return; - } - t = next; - binder_debug(BINDER_DEBUG_DEAD_BINDER, - "reply failed, no target thread -- retry %d\n", - t->debug_id); - } -} - -static void binder_transaction_buffer_release(struct binder_proc *proc, - struct binder_buffer *buffer, - binder_size_t *failed_at) -{ - binder_size_t *offp, *off_end; - int debug_id = buffer->debug_id; - - binder_debug(BINDER_DEBUG_TRANSACTION, - "%d buffer release %d, size %zd-%zd, failed at %p\n", - proc->pid, buffer->debug_id, - buffer->data_size, buffer->offsets_size, failed_at); - - if (buffer->target_node) - binder_dec_node(buffer->target_node, 1, 0); - - offp = (binder_size_t *)(buffer->data + - ALIGN(buffer->data_size, sizeof(void *))); - if (failed_at) - off_end = failed_at; - else - off_end = (void *)offp + buffer->offsets_size; - for (; offp < off_end; offp++) { - struct flat_binder_object *fp; - - if (*offp > buffer->data_size - sizeof(*fp) || - buffer->data_size < sizeof(*fp) || - !IS_ALIGNED(*offp, sizeof(u32))) { - pr_err("transaction release %d bad offset %lld, size %zd\n", - debug_id, (u64)*offp, buffer->data_size); - continue; - } - fp = (struct flat_binder_object *)(buffer->data + *offp); - switch (fp->type) { - case BINDER_TYPE_BINDER: - case BINDER_TYPE_WEAK_BINDER: { - struct binder_node *node = binder_get_node(proc, fp->binder); - - if (node == NULL) { - pr_err("transaction release %d bad node %016llx\n", - debug_id, (u64)fp->binder); - break; - } - binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%016llx\n", - node->debug_id, (u64)node->ptr); - binder_dec_node(node, fp->type == BINDER_TYPE_BINDER, 0); - } break; - case BINDER_TYPE_HANDLE: - case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref = binder_get_ref(proc, fp->handle); - - if (ref == NULL) { - pr_err("transaction release %d bad handle %d\n", - debug_id, fp->handle); - break; - } - binder_debug(BINDER_DEBUG_TRANSACTION, - " ref %d desc %d (node %d)\n", - ref->debug_id, ref->desc, ref->node->debug_id); - binder_dec_ref(ref, fp->type == BINDER_TYPE_HANDLE); - } break; - - case BINDER_TYPE_FD: - binder_debug(BINDER_DEBUG_TRANSACTION, - " fd %d\n", fp->handle); - if (failed_at) - task_close_fd(proc, fp->handle); - break; - - default: - pr_err("transaction release %d bad object type %x\n", - debug_id, fp->type); - break; - } - } -} - -static void binder_transaction(struct binder_proc *proc, - struct binder_thread *thread, - struct binder_transaction_data *tr, int reply) -{ - struct binder_transaction *t; - struct binder_work *tcomplete; - binder_size_t *offp, *off_end; - struct binder_proc *target_proc; - struct binder_thread *target_thread = NULL; - struct binder_node *target_node = NULL; - struct list_head *target_list; - wait_queue_head_t *target_wait; - struct binder_transaction *in_reply_to = NULL; - struct binder_transaction_log_entry *e; - uint32_t return_error; - - e = binder_transaction_log_add(&binder_transaction_log); - e->call_type = reply ? 2 : !!(tr->flags & TF_ONE_WAY); - e->from_proc = proc->pid; - e->from_thread = thread->pid; - e->target_handle = tr->target.handle; - e->data_size = tr->data_size; - e->offsets_size = tr->offsets_size; - - if (reply) { - in_reply_to = thread->transaction_stack; - if (in_reply_to == NULL) { - binder_user_error("%d:%d got reply transaction with no transaction stack\n", - proc->pid, thread->pid); - return_error = BR_FAILED_REPLY; - goto err_empty_call_stack; - } - binder_set_nice(in_reply_to->saved_priority); - if (in_reply_to->to_thread != thread) { - binder_user_error("%d:%d got reply transaction with bad transaction stack, transaction %d has target %d:%d\n", - proc->pid, thread->pid, in_reply_to->debug_id, - in_reply_to->to_proc ? - in_reply_to->to_proc->pid : 0, - in_reply_to->to_thread ? - in_reply_to->to_thread->pid : 0); - return_error = BR_FAILED_REPLY; - in_reply_to = NULL; - goto err_bad_call_stack; - } - thread->transaction_stack = in_reply_to->to_parent; - target_thread = in_reply_to->from; - if (target_thread == NULL) { - return_error = BR_DEAD_REPLY; - goto err_dead_binder; - } - if (target_thread->transaction_stack != in_reply_to) { - binder_user_error("%d:%d got reply transaction with bad target transaction stack %d, expected %d\n", - proc->pid, thread->pid, - target_thread->transaction_stack ? - target_thread->transaction_stack->debug_id : 0, - in_reply_to->debug_id); - return_error = BR_FAILED_REPLY; - in_reply_to = NULL; - target_thread = NULL; - goto err_dead_binder; - } - target_proc = target_thread->proc; - } else { - if (tr->target.handle) { - struct binder_ref *ref; - - ref = binder_get_ref(proc, tr->target.handle); - if (ref == NULL) { - binder_user_error("%d:%d got transaction to invalid handle\n", - proc->pid, thread->pid); - return_error = BR_FAILED_REPLY; - goto err_invalid_target_handle; - } - target_node = ref->node; - } else { - target_node = binder_context_mgr_node; - if (target_node == NULL) { - return_error = BR_DEAD_REPLY; - goto err_no_context_mgr_node; - } - } - e->to_node = target_node->debug_id; - target_proc = target_node->proc; - if (target_proc == NULL) { - return_error = BR_DEAD_REPLY; - goto err_dead_binder; - } - if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) { - struct binder_transaction *tmp; - - tmp = thread->transaction_stack; - if (tmp->to_thread != thread) { - binder_user_error("%d:%d got new transaction with bad transaction stack, transaction %d has target %d:%d\n", - proc->pid, thread->pid, tmp->debug_id, - tmp->to_proc ? tmp->to_proc->pid : 0, - tmp->to_thread ? - tmp->to_thread->pid : 0); - return_error = BR_FAILED_REPLY; - goto err_bad_call_stack; - } - while (tmp) { - if (tmp->from && tmp->from->proc == target_proc) - target_thread = tmp->from; - tmp = tmp->from_parent; - } - } - } - if (target_thread) { - e->to_thread = target_thread->pid; - target_list = &target_thread->todo; - target_wait = &target_thread->wait; - } else { - target_list = &target_proc->todo; - target_wait = &target_proc->wait; - } - e->to_proc = target_proc->pid; - - /* TODO: reuse incoming transaction for reply */ - t = kzalloc(sizeof(*t), GFP_KERNEL); - if (t == NULL) { - return_error = BR_FAILED_REPLY; - goto err_alloc_t_failed; - } - binder_stats_created(BINDER_STAT_TRANSACTION); - - tcomplete = kzalloc(sizeof(*tcomplete), GFP_KERNEL); - if (tcomplete == NULL) { - return_error = BR_FAILED_REPLY; - goto err_alloc_tcomplete_failed; - } - binder_stats_created(BINDER_STAT_TRANSACTION_COMPLETE); - - t->debug_id = ++binder_last_id; - e->debug_id = t->debug_id; - - if (reply) - binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d BC_REPLY %d -> %d:%d, data %016llx-%016llx size %lld-%lld\n", - proc->pid, thread->pid, t->debug_id, - target_proc->pid, target_thread->pid, - (u64)tr->data.ptr.buffer, - (u64)tr->data.ptr.offsets, - (u64)tr->data_size, (u64)tr->offsets_size); - else - binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d BC_TRANSACTION %d -> %d - node %d, data %016llx-%016llx size %lld-%lld\n", - proc->pid, thread->pid, t->debug_id, - target_proc->pid, target_node->debug_id, - (u64)tr->data.ptr.buffer, - (u64)tr->data.ptr.offsets, - (u64)tr->data_size, (u64)tr->offsets_size); - - if (!reply && !(tr->flags & TF_ONE_WAY)) - t->from = thread; - else - t->from = NULL; - t->sender_euid = task_euid(proc->tsk); - t->to_proc = target_proc; - t->to_thread = target_thread; - t->code = tr->code; - t->flags = tr->flags; - t->priority = task_nice(current); - - trace_binder_transaction(reply, t, target_node); - - t->buffer = binder_alloc_buf(target_proc, tr->data_size, - tr->offsets_size, !reply && (t->flags & TF_ONE_WAY)); - if (t->buffer == NULL) { - return_error = BR_FAILED_REPLY; - goto err_binder_alloc_buf_failed; - } - t->buffer->allow_user_free = 0; - t->buffer->debug_id = t->debug_id; - t->buffer->transaction = t; - t->buffer->target_node = target_node; - trace_binder_transaction_alloc_buf(t->buffer); - if (target_node) - binder_inc_node(target_node, 1, 0, NULL); - - offp = (binder_size_t *)(t->buffer->data + - ALIGN(tr->data_size, sizeof(void *))); - - if (copy_from_user(t->buffer->data, (const void __user *)(uintptr_t) - tr->data.ptr.buffer, tr->data_size)) { - binder_user_error("%d:%d got transaction with invalid data ptr\n", - proc->pid, thread->pid); - return_error = BR_FAILED_REPLY; - goto err_copy_data_failed; - } - if (copy_from_user(offp, (const void __user *)(uintptr_t) - tr->data.ptr.offsets, tr->offsets_size)) { - binder_user_error("%d:%d got transaction with invalid offsets ptr\n", - proc->pid, thread->pid); - return_error = BR_FAILED_REPLY; - goto err_copy_data_failed; - } - if (!IS_ALIGNED(tr->offsets_size, sizeof(binder_size_t))) { - binder_user_error("%d:%d got transaction with invalid offsets size, %lld\n", - proc->pid, thread->pid, (u64)tr->offsets_size); - return_error = BR_FAILED_REPLY; - goto err_bad_offset; - } - off_end = (void *)offp + tr->offsets_size; - for (; offp < off_end; offp++) { - struct flat_binder_object *fp; - - if (*offp > t->buffer->data_size - sizeof(*fp) || - t->buffer->data_size < sizeof(*fp) || - !IS_ALIGNED(*offp, sizeof(u32))) { - binder_user_error("%d:%d got transaction with invalid offset, %lld\n", - proc->pid, thread->pid, (u64)*offp); - return_error = BR_FAILED_REPLY; - goto err_bad_offset; - } - fp = (struct flat_binder_object *)(t->buffer->data + *offp); - switch (fp->type) { - case BINDER_TYPE_BINDER: - case BINDER_TYPE_WEAK_BINDER: { - struct binder_ref *ref; - struct binder_node *node = binder_get_node(proc, fp->binder); - - if (node == NULL) { - node = binder_new_node(proc, fp->binder, fp->cookie); - if (node == NULL) { - return_error = BR_FAILED_REPLY; - goto err_binder_new_node_failed; - } - node->min_priority = fp->flags & FLAT_BINDER_FLAG_PRIORITY_MASK; - node->accept_fds = !!(fp->flags & FLAT_BINDER_FLAG_ACCEPTS_FDS); - } - if (fp->cookie != node->cookie) { - binder_user_error("%d:%d sending u%016llx node %d, cookie mismatch %016llx != %016llx\n", - proc->pid, thread->pid, - (u64)fp->binder, node->debug_id, - (u64)fp->cookie, (u64)node->cookie); - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_for_node_failed; - } - ref = binder_get_ref_for_node(target_proc, node); - if (ref == NULL) { - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_for_node_failed; - } - if (fp->type == BINDER_TYPE_BINDER) - fp->type = BINDER_TYPE_HANDLE; - else - fp->type = BINDER_TYPE_WEAK_HANDLE; - fp->handle = ref->desc; - binder_inc_ref(ref, fp->type == BINDER_TYPE_HANDLE, - &thread->todo); - - trace_binder_transaction_node_to_ref(t, node, ref); - binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%016llx -> ref %d desc %d\n", - node->debug_id, (u64)node->ptr, - ref->debug_id, ref->desc); - } break; - case BINDER_TYPE_HANDLE: - case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref = binder_get_ref(proc, fp->handle); - - if (ref == NULL) { - binder_user_error("%d:%d got transaction with invalid handle, %d\n", - proc->pid, - thread->pid, fp->handle); - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_failed; - } - if (ref->node->proc == target_proc) { - if (fp->type == BINDER_TYPE_HANDLE) - fp->type = BINDER_TYPE_BINDER; - else - fp->type = BINDER_TYPE_WEAK_BINDER; - fp->binder = ref->node->ptr; - fp->cookie = ref->node->cookie; - binder_inc_node(ref->node, fp->type == BINDER_TYPE_BINDER, 0, NULL); - trace_binder_transaction_ref_to_node(t, ref); - binder_debug(BINDER_DEBUG_TRANSACTION, - " ref %d desc %d -> node %d u%016llx\n", - ref->debug_id, ref->desc, ref->node->debug_id, - (u64)ref->node->ptr); - } else { - struct binder_ref *new_ref; - - new_ref = binder_get_ref_for_node(target_proc, ref->node); - if (new_ref == NULL) { - return_error = BR_FAILED_REPLY; - goto err_binder_get_ref_for_node_failed; - } - fp->handle = new_ref->desc; - binder_inc_ref(new_ref, fp->type == BINDER_TYPE_HANDLE, NULL); - trace_binder_transaction_ref_to_ref(t, ref, - new_ref); - binder_debug(BINDER_DEBUG_TRANSACTION, - " ref %d desc %d -> ref %d desc %d (node %d)\n", - ref->debug_id, ref->desc, new_ref->debug_id, - new_ref->desc, ref->node->debug_id); - } - } break; - - case BINDER_TYPE_FD: { - int target_fd; - struct file *file; - - if (reply) { - if (!(in_reply_to->flags & TF_ACCEPT_FDS)) { - binder_user_error("%d:%d got reply with fd, %d, but target does not allow fds\n", - proc->pid, thread->pid, fp->handle); - return_error = BR_FAILED_REPLY; - goto err_fd_not_allowed; - } - } else if (!target_node->accept_fds) { - binder_user_error("%d:%d got transaction with fd, %d, but target does not allow fds\n", - proc->pid, thread->pid, fp->handle); - return_error = BR_FAILED_REPLY; - goto err_fd_not_allowed; - } - - file = fget(fp->handle); - if (file == NULL) { - binder_user_error("%d:%d got transaction with invalid fd, %d\n", - proc->pid, thread->pid, fp->handle); - return_error = BR_FAILED_REPLY; - goto err_fget_failed; - } - target_fd = task_get_unused_fd_flags(target_proc, O_CLOEXEC); - if (target_fd < 0) { - fput(file); - return_error = BR_FAILED_REPLY; - goto err_get_unused_fd_failed; - } - task_fd_install(target_proc, target_fd, file); - trace_binder_transaction_fd(t, fp->handle, target_fd); - binder_debug(BINDER_DEBUG_TRANSACTION, - " fd %d -> %d\n", fp->handle, target_fd); - /* TODO: fput? */ - fp->handle = target_fd; - } break; - - default: - binder_user_error("%d:%d got transaction with invalid object type, %x\n", - proc->pid, thread->pid, fp->type); - return_error = BR_FAILED_REPLY; - goto err_bad_object_type; - } - } - if (reply) { - BUG_ON(t->buffer->async_transaction != 0); - binder_pop_transaction(target_thread, in_reply_to); - } else if (!(t->flags & TF_ONE_WAY)) { - BUG_ON(t->buffer->async_transaction != 0); - t->need_reply = 1; - t->from_parent = thread->transaction_stack; - thread->transaction_stack = t; - } else { - BUG_ON(target_node == NULL); - BUG_ON(t->buffer->async_transaction != 1); - if (target_node->has_async_transaction) { - target_list = &target_node->async_todo; - target_wait = NULL; - } else - target_node->has_async_transaction = 1; - } - t->work.type = BINDER_WORK_TRANSACTION; - list_add_tail(&t->work.entry, target_list); - tcomplete->type = BINDER_WORK_TRANSACTION_COMPLETE; - list_add_tail(&tcomplete->entry, &thread->todo); - if (target_wait) - wake_up_interruptible(target_wait); - return; - -err_get_unused_fd_failed: -err_fget_failed: -err_fd_not_allowed: -err_binder_get_ref_for_node_failed: -err_binder_get_ref_failed: -err_binder_new_node_failed: -err_bad_object_type: -err_bad_offset: -err_copy_data_failed: - trace_binder_transaction_failed_buffer_release(t->buffer); - binder_transaction_buffer_release(target_proc, t->buffer, offp); - t->buffer->transaction = NULL; - binder_free_buf(target_proc, t->buffer); -err_binder_alloc_buf_failed: - kfree(tcomplete); - binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE); -err_alloc_tcomplete_failed: - kfree(t); - binder_stats_deleted(BINDER_STAT_TRANSACTION); -err_alloc_t_failed: -err_bad_call_stack: -err_empty_call_stack: -err_dead_binder: -err_invalid_target_handle: -err_no_context_mgr_node: - binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, - "%d:%d transaction failed %d, size %lld-%lld\n", - proc->pid, thread->pid, return_error, - (u64)tr->data_size, (u64)tr->offsets_size); - - { - struct binder_transaction_log_entry *fe; - - fe = binder_transaction_log_add(&binder_transaction_log_failed); - *fe = *e; - } - - BUG_ON(thread->return_error != BR_OK); - if (in_reply_to) { - thread->return_error = BR_TRANSACTION_COMPLETE; - binder_send_failed_reply(in_reply_to, return_error); - } else - thread->return_error = return_error; -} - -static int binder_thread_write(struct binder_proc *proc, - struct binder_thread *thread, - binder_uintptr_t binder_buffer, size_t size, - binder_size_t *consumed) -{ - uint32_t cmd; - void __user *buffer = (void __user *)(uintptr_t)binder_buffer; - void __user *ptr = buffer + *consumed; - void __user *end = buffer + size; - - while (ptr < end && thread->return_error == BR_OK) { - if (get_user(cmd, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - trace_binder_command(cmd); - if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.bc)) { - binder_stats.bc[_IOC_NR(cmd)]++; - proc->stats.bc[_IOC_NR(cmd)]++; - thread->stats.bc[_IOC_NR(cmd)]++; - } - switch (cmd) { - case BC_INCREFS: - case BC_ACQUIRE: - case BC_RELEASE: - case BC_DECREFS: { - uint32_t target; - struct binder_ref *ref; - const char *debug_string; - - if (get_user(target, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - if (target == 0 && binder_context_mgr_node && - (cmd == BC_INCREFS || cmd == BC_ACQUIRE)) { - ref = binder_get_ref_for_node(proc, - binder_context_mgr_node); - if (ref->desc != target) { - binder_user_error("%d:%d tried to acquire reference to desc 0, got %d instead\n", - proc->pid, thread->pid, - ref->desc); - } - } else - ref = binder_get_ref(proc, target); - if (ref == NULL) { - binder_user_error("%d:%d refcount change on invalid ref %d\n", - proc->pid, thread->pid, target); - break; - } - switch (cmd) { - case BC_INCREFS: - debug_string = "IncRefs"; - binder_inc_ref(ref, 0, NULL); - break; - case BC_ACQUIRE: - debug_string = "Acquire"; - binder_inc_ref(ref, 1, NULL); - break; - case BC_RELEASE: - debug_string = "Release"; - binder_dec_ref(ref, 1); - break; - case BC_DECREFS: - default: - debug_string = "DecRefs"; - binder_dec_ref(ref, 0); - break; - } - binder_debug(BINDER_DEBUG_USER_REFS, - "%d:%d %s ref %d desc %d s %d w %d for node %d\n", - proc->pid, thread->pid, debug_string, ref->debug_id, - ref->desc, ref->strong, ref->weak, ref->node->debug_id); - break; - } - case BC_INCREFS_DONE: - case BC_ACQUIRE_DONE: { - binder_uintptr_t node_ptr; - binder_uintptr_t cookie; - struct binder_node *node; - - if (get_user(node_ptr, (binder_uintptr_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(binder_uintptr_t); - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(binder_uintptr_t); - node = binder_get_node(proc, node_ptr); - if (node == NULL) { - binder_user_error("%d:%d %s u%016llx no match\n", - proc->pid, thread->pid, - cmd == BC_INCREFS_DONE ? - "BC_INCREFS_DONE" : - "BC_ACQUIRE_DONE", - (u64)node_ptr); - break; - } - if (cookie != node->cookie) { - binder_user_error("%d:%d %s u%016llx node %d cookie mismatch %016llx != %016llx\n", - proc->pid, thread->pid, - cmd == BC_INCREFS_DONE ? - "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", - (u64)node_ptr, node->debug_id, - (u64)cookie, (u64)node->cookie); - break; - } - if (cmd == BC_ACQUIRE_DONE) { - if (node->pending_strong_ref == 0) { - binder_user_error("%d:%d BC_ACQUIRE_DONE node %d has no pending acquire request\n", - proc->pid, thread->pid, - node->debug_id); - break; - } - node->pending_strong_ref = 0; - } else { - if (node->pending_weak_ref == 0) { - binder_user_error("%d:%d BC_INCREFS_DONE node %d has no pending increfs request\n", - proc->pid, thread->pid, - node->debug_id); - break; - } - node->pending_weak_ref = 0; - } - binder_dec_node(node, cmd == BC_ACQUIRE_DONE, 0); - binder_debug(BINDER_DEBUG_USER_REFS, - "%d:%d %s node %d ls %d lw %d\n", - proc->pid, thread->pid, - cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", - node->debug_id, node->local_strong_refs, node->local_weak_refs); - break; - } - case BC_ATTEMPT_ACQUIRE: - pr_err("BC_ATTEMPT_ACQUIRE not supported\n"); - return -EINVAL; - case BC_ACQUIRE_RESULT: - pr_err("BC_ACQUIRE_RESULT not supported\n"); - return -EINVAL; - - case BC_FREE_BUFFER: { - binder_uintptr_t data_ptr; - struct binder_buffer *buffer; - - if (get_user(data_ptr, (binder_uintptr_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(binder_uintptr_t); - - buffer = binder_buffer_lookup(proc, data_ptr); - if (buffer == NULL) { - binder_user_error("%d:%d BC_FREE_BUFFER u%016llx no match\n", - proc->pid, thread->pid, (u64)data_ptr); - break; - } - if (!buffer->allow_user_free) { - binder_user_error("%d:%d BC_FREE_BUFFER u%016llx matched unreturned buffer\n", - proc->pid, thread->pid, (u64)data_ptr); - break; - } - binder_debug(BINDER_DEBUG_FREE_BUFFER, - "%d:%d BC_FREE_BUFFER u%016llx found buffer %d for %s transaction\n", - proc->pid, thread->pid, (u64)data_ptr, - buffer->debug_id, - buffer->transaction ? "active" : "finished"); - - if (buffer->transaction) { - buffer->transaction->buffer = NULL; - buffer->transaction = NULL; - } - if (buffer->async_transaction && buffer->target_node) { - BUG_ON(!buffer->target_node->has_async_transaction); - if (list_empty(&buffer->target_node->async_todo)) - buffer->target_node->has_async_transaction = 0; - else - list_move_tail(buffer->target_node->async_todo.next, &thread->todo); - } - trace_binder_transaction_buffer_release(buffer); - binder_transaction_buffer_release(proc, buffer, NULL); - binder_free_buf(proc, buffer); - break; - } - - case BC_TRANSACTION: - case BC_REPLY: { - struct binder_transaction_data tr; - - if (copy_from_user(&tr, ptr, sizeof(tr))) - return -EFAULT; - ptr += sizeof(tr); - binder_transaction(proc, thread, &tr, cmd == BC_REPLY); - break; - } - - case BC_REGISTER_LOOPER: - binder_debug(BINDER_DEBUG_THREADS, - "%d:%d BC_REGISTER_LOOPER\n", - proc->pid, thread->pid); - if (thread->looper & BINDER_LOOPER_STATE_ENTERED) { - thread->looper |= BINDER_LOOPER_STATE_INVALID; - binder_user_error("%d:%d ERROR: BC_REGISTER_LOOPER called after BC_ENTER_LOOPER\n", - proc->pid, thread->pid); - } else if (proc->requested_threads == 0) { - thread->looper |= BINDER_LOOPER_STATE_INVALID; - binder_user_error("%d:%d ERROR: BC_REGISTER_LOOPER called without request\n", - proc->pid, thread->pid); - } else { - proc->requested_threads--; - proc->requested_threads_started++; - } - thread->looper |= BINDER_LOOPER_STATE_REGISTERED; - break; - case BC_ENTER_LOOPER: - binder_debug(BINDER_DEBUG_THREADS, - "%d:%d BC_ENTER_LOOPER\n", - proc->pid, thread->pid); - if (thread->looper & BINDER_LOOPER_STATE_REGISTERED) { - thread->looper |= BINDER_LOOPER_STATE_INVALID; - binder_user_error("%d:%d ERROR: BC_ENTER_LOOPER called after BC_REGISTER_LOOPER\n", - proc->pid, thread->pid); - } - thread->looper |= BINDER_LOOPER_STATE_ENTERED; - break; - case BC_EXIT_LOOPER: - binder_debug(BINDER_DEBUG_THREADS, - "%d:%d BC_EXIT_LOOPER\n", - proc->pid, thread->pid); - thread->looper |= BINDER_LOOPER_STATE_EXITED; - break; - - case BC_REQUEST_DEATH_NOTIFICATION: - case BC_CLEAR_DEATH_NOTIFICATION: { - uint32_t target; - binder_uintptr_t cookie; - struct binder_ref *ref; - struct binder_ref_death *death; - - if (get_user(target, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(binder_uintptr_t); - ref = binder_get_ref(proc, target); - if (ref == NULL) { - binder_user_error("%d:%d %s invalid ref %d\n", - proc->pid, thread->pid, - cmd == BC_REQUEST_DEATH_NOTIFICATION ? - "BC_REQUEST_DEATH_NOTIFICATION" : - "BC_CLEAR_DEATH_NOTIFICATION", - target); - break; - } - - binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, - "%d:%d %s %016llx ref %d desc %d s %d w %d for node %d\n", - proc->pid, thread->pid, - cmd == BC_REQUEST_DEATH_NOTIFICATION ? - "BC_REQUEST_DEATH_NOTIFICATION" : - "BC_CLEAR_DEATH_NOTIFICATION", - (u64)cookie, ref->debug_id, ref->desc, - ref->strong, ref->weak, ref->node->debug_id); - - if (cmd == BC_REQUEST_DEATH_NOTIFICATION) { - if (ref->death) { - binder_user_error("%d:%d BC_REQUEST_DEATH_NOTIFICATION death notification already set\n", - proc->pid, thread->pid); - break; - } - death = kzalloc(sizeof(*death), GFP_KERNEL); - if (death == NULL) { - thread->return_error = BR_ERROR; - binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, - "%d:%d BC_REQUEST_DEATH_NOTIFICATION failed\n", - proc->pid, thread->pid); - break; - } - binder_stats_created(BINDER_STAT_DEATH); - INIT_LIST_HEAD(&death->work.entry); - death->cookie = cookie; - ref->death = death; - if (ref->node->proc == NULL) { - ref->death->work.type = BINDER_WORK_DEAD_BINDER; - if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) { - list_add_tail(&ref->death->work.entry, &thread->todo); - } else { - list_add_tail(&ref->death->work.entry, &proc->todo); - wake_up_interruptible(&proc->wait); - } - } - } else { - if (ref->death == NULL) { - binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification not active\n", - proc->pid, thread->pid); - break; - } - death = ref->death; - if (death->cookie != cookie) { - binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch %016llx != %016llx\n", - proc->pid, thread->pid, - (u64)death->cookie, - (u64)cookie); - break; - } - ref->death = NULL; - if (list_empty(&death->work.entry)) { - death->work.type = BINDER_WORK_CLEAR_DEATH_NOTIFICATION; - if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) { - list_add_tail(&death->work.entry, &thread->todo); - } else { - list_add_tail(&death->work.entry, &proc->todo); - wake_up_interruptible(&proc->wait); - } - } else { - BUG_ON(death->work.type != BINDER_WORK_DEAD_BINDER); - death->work.type = BINDER_WORK_DEAD_BINDER_AND_CLEAR; - } - } - } break; - case BC_DEAD_BINDER_DONE: { - struct binder_work *w; - binder_uintptr_t cookie; - struct binder_ref_death *death = NULL; - - if (get_user(cookie, (binder_uintptr_t __user *)ptr)) - return -EFAULT; - - ptr += sizeof(void *); - list_for_each_entry(w, &proc->delivered_death, entry) { - struct binder_ref_death *tmp_death = container_of(w, struct binder_ref_death, work); - - if (tmp_death->cookie == cookie) { - death = tmp_death; - break; - } - } - binder_debug(BINDER_DEBUG_DEAD_BINDER, - "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n", - proc->pid, thread->pid, (u64)cookie, - death); - if (death == NULL) { - binder_user_error("%d:%d BC_DEAD_BINDER_DONE %016llx not found\n", - proc->pid, thread->pid, (u64)cookie); - break; - } - - list_del_init(&death->work.entry); - if (death->work.type == BINDER_WORK_DEAD_BINDER_AND_CLEAR) { - death->work.type = BINDER_WORK_CLEAR_DEATH_NOTIFICATION; - if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) { - list_add_tail(&death->work.entry, &thread->todo); - } else { - list_add_tail(&death->work.entry, &proc->todo); - wake_up_interruptible(&proc->wait); - } - } - } break; - - default: - pr_err("%d:%d unknown command %d\n", - proc->pid, thread->pid, cmd); - return -EINVAL; - } - *consumed = ptr - buffer; - } - return 0; -} - -static void binder_stat_br(struct binder_proc *proc, - struct binder_thread *thread, uint32_t cmd) -{ - trace_binder_return(cmd); - if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.br)) { - binder_stats.br[_IOC_NR(cmd)]++; - proc->stats.br[_IOC_NR(cmd)]++; - thread->stats.br[_IOC_NR(cmd)]++; - } -} - -static int binder_has_proc_work(struct binder_proc *proc, - struct binder_thread *thread) -{ - return !list_empty(&proc->todo) || - (thread->looper & BINDER_LOOPER_STATE_NEED_RETURN); -} - -static int binder_has_thread_work(struct binder_thread *thread) -{ - return !list_empty(&thread->todo) || thread->return_error != BR_OK || - (thread->looper & BINDER_LOOPER_STATE_NEED_RETURN); -} - -static int binder_thread_read(struct binder_proc *proc, - struct binder_thread *thread, - binder_uintptr_t binder_buffer, size_t size, - binder_size_t *consumed, int non_block) -{ - void __user *buffer = (void __user *)(uintptr_t)binder_buffer; - void __user *ptr = buffer + *consumed; - void __user *end = buffer + size; - - int ret = 0; - int wait_for_proc_work; - - if (*consumed == 0) { - if (put_user(BR_NOOP, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - } - -retry: - wait_for_proc_work = thread->transaction_stack == NULL && - list_empty(&thread->todo); - - if (thread->return_error != BR_OK && ptr < end) { - if (thread->return_error2 != BR_OK) { - if (put_user(thread->return_error2, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - binder_stat_br(proc, thread, thread->return_error2); - if (ptr == end) - goto done; - thread->return_error2 = BR_OK; - } - if (put_user(thread->return_error, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - binder_stat_br(proc, thread, thread->return_error); - thread->return_error = BR_OK; - goto done; - } - - - thread->looper |= BINDER_LOOPER_STATE_WAITING; - if (wait_for_proc_work) - proc->ready_threads++; - - binder_unlock(__func__); - - trace_binder_wait_for_work(wait_for_proc_work, - !!thread->transaction_stack, - !list_empty(&thread->todo)); - if (wait_for_proc_work) { - if (!(thread->looper & (BINDER_LOOPER_STATE_REGISTERED | - BINDER_LOOPER_STATE_ENTERED))) { - binder_user_error("%d:%d ERROR: Thread waiting for process work before calling BC_REGISTER_LOOPER or BC_ENTER_LOOPER (state %x)\n", - proc->pid, thread->pid, thread->looper); - wait_event_interruptible(binder_user_error_wait, - binder_stop_on_user_error < 2); - } - binder_set_nice(proc->default_priority); - if (non_block) { - if (!binder_has_proc_work(proc, thread)) - ret = -EAGAIN; - } else - ret = wait_event_freezable_exclusive(proc->wait, binder_has_proc_work(proc, thread)); - } else { - if (non_block) { - if (!binder_has_thread_work(thread)) - ret = -EAGAIN; - } else - ret = wait_event_freezable(thread->wait, binder_has_thread_work(thread)); - } - - binder_lock(__func__); - - if (wait_for_proc_work) - proc->ready_threads--; - thread->looper &= ~BINDER_LOOPER_STATE_WAITING; - - if (ret) - return ret; - - while (1) { - uint32_t cmd; - struct binder_transaction_data tr; - struct binder_work *w; - struct binder_transaction *t = NULL; - - if (!list_empty(&thread->todo)) { - w = list_first_entry(&thread->todo, struct binder_work, - entry); - } else if (!list_empty(&proc->todo) && wait_for_proc_work) { - w = list_first_entry(&proc->todo, struct binder_work, - entry); - } else { - /* no data added */ - if (ptr - buffer == 4 && - !(thread->looper & BINDER_LOOPER_STATE_NEED_RETURN)) - goto retry; - break; - } - - if (end - ptr < sizeof(tr) + 4) - break; - - switch (w->type) { - case BINDER_WORK_TRANSACTION: { - t = container_of(w, struct binder_transaction, work); - } break; - case BINDER_WORK_TRANSACTION_COMPLETE: { - cmd = BR_TRANSACTION_COMPLETE; - if (put_user(cmd, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - - binder_stat_br(proc, thread, cmd); - binder_debug(BINDER_DEBUG_TRANSACTION_COMPLETE, - "%d:%d BR_TRANSACTION_COMPLETE\n", - proc->pid, thread->pid); - - list_del(&w->entry); - kfree(w); - binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE); - } break; - case BINDER_WORK_NODE: { - struct binder_node *node = container_of(w, struct binder_node, work); - uint32_t cmd = BR_NOOP; - const char *cmd_name; - int strong = node->internal_strong_refs || node->local_strong_refs; - int weak = !hlist_empty(&node->refs) || node->local_weak_refs || strong; - - if (weak && !node->has_weak_ref) { - cmd = BR_INCREFS; - cmd_name = "BR_INCREFS"; - node->has_weak_ref = 1; - node->pending_weak_ref = 1; - node->local_weak_refs++; - } else if (strong && !node->has_strong_ref) { - cmd = BR_ACQUIRE; - cmd_name = "BR_ACQUIRE"; - node->has_strong_ref = 1; - node->pending_strong_ref = 1; - node->local_strong_refs++; - } else if (!strong && node->has_strong_ref) { - cmd = BR_RELEASE; - cmd_name = "BR_RELEASE"; - node->has_strong_ref = 0; - } else if (!weak && node->has_weak_ref) { - cmd = BR_DECREFS; - cmd_name = "BR_DECREFS"; - node->has_weak_ref = 0; - } - if (cmd != BR_NOOP) { - if (put_user(cmd, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - if (put_user(node->ptr, - (binder_uintptr_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(binder_uintptr_t); - if (put_user(node->cookie, - (binder_uintptr_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(binder_uintptr_t); - - binder_stat_br(proc, thread, cmd); - binder_debug(BINDER_DEBUG_USER_REFS, - "%d:%d %s %d u%016llx c%016llx\n", - proc->pid, thread->pid, cmd_name, - node->debug_id, - (u64)node->ptr, (u64)node->cookie); - } else { - list_del_init(&w->entry); - if (!weak && !strong) { - binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%016llx c%016llx deleted\n", - proc->pid, thread->pid, - node->debug_id, - (u64)node->ptr, - (u64)node->cookie); - rb_erase(&node->rb_node, &proc->nodes); - kfree(node); - binder_stats_deleted(BINDER_STAT_NODE); - } else { - binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "%d:%d node %d u%016llx c%016llx state unchanged\n", - proc->pid, thread->pid, - node->debug_id, - (u64)node->ptr, - (u64)node->cookie); - } - } - } break; - case BINDER_WORK_DEAD_BINDER: - case BINDER_WORK_DEAD_BINDER_AND_CLEAR: - case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: { - struct binder_ref_death *death; - uint32_t cmd; - - death = container_of(w, struct binder_ref_death, work); - if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION) - cmd = BR_CLEAR_DEATH_NOTIFICATION_DONE; - else - cmd = BR_DEAD_BINDER; - if (put_user(cmd, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - if (put_user(death->cookie, - (binder_uintptr_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(binder_uintptr_t); - binder_stat_br(proc, thread, cmd); - binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, - "%d:%d %s %016llx\n", - proc->pid, thread->pid, - cmd == BR_DEAD_BINDER ? - "BR_DEAD_BINDER" : - "BR_CLEAR_DEATH_NOTIFICATION_DONE", - (u64)death->cookie); - - if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION) { - list_del(&w->entry); - kfree(death); - binder_stats_deleted(BINDER_STAT_DEATH); - } else - list_move(&w->entry, &proc->delivered_death); - if (cmd == BR_DEAD_BINDER) - goto done; /* DEAD_BINDER notifications can cause transactions */ - } break; - } - - if (!t) - continue; - - BUG_ON(t->buffer == NULL); - if (t->buffer->target_node) { - struct binder_node *target_node = t->buffer->target_node; - - tr.target.ptr = target_node->ptr; - tr.cookie = target_node->cookie; - t->saved_priority = task_nice(current); - if (t->priority < target_node->min_priority && - !(t->flags & TF_ONE_WAY)) - binder_set_nice(t->priority); - else if (!(t->flags & TF_ONE_WAY) || - t->saved_priority > target_node->min_priority) - binder_set_nice(target_node->min_priority); - cmd = BR_TRANSACTION; - } else { - tr.target.ptr = 0; - tr.cookie = 0; - cmd = BR_REPLY; - } - tr.code = t->code; - tr.flags = t->flags; - tr.sender_euid = from_kuid(current_user_ns(), t->sender_euid); - - if (t->from) { - struct task_struct *sender = t->from->proc->tsk; - - tr.sender_pid = task_tgid_nr_ns(sender, - task_active_pid_ns(current)); - } else { - tr.sender_pid = 0; - } - - tr.data_size = t->buffer->data_size; - tr.offsets_size = t->buffer->offsets_size; - tr.data.ptr.buffer = (binder_uintptr_t)( - (uintptr_t)t->buffer->data + - proc->user_buffer_offset); - tr.data.ptr.offsets = tr.data.ptr.buffer + - ALIGN(t->buffer->data_size, - sizeof(void *)); - - if (put_user(cmd, (uint32_t __user *)ptr)) - return -EFAULT; - ptr += sizeof(uint32_t); - if (copy_to_user(ptr, &tr, sizeof(tr))) - return -EFAULT; - ptr += sizeof(tr); - - trace_binder_transaction_received(t); - binder_stat_br(proc, thread, cmd); - binder_debug(BINDER_DEBUG_TRANSACTION, - "%d:%d %s %d %d:%d, cmd %d size %zd-%zd ptr %016llx-%016llx\n", - proc->pid, thread->pid, - (cmd == BR_TRANSACTION) ? "BR_TRANSACTION" : - "BR_REPLY", - t->debug_id, t->from ? t->from->proc->pid : 0, - t->from ? t->from->pid : 0, cmd, - t->buffer->data_size, t->buffer->offsets_size, - (u64)tr.data.ptr.buffer, (u64)tr.data.ptr.offsets); - - list_del(&t->work.entry); - t->buffer->allow_user_free = 1; - if (cmd == BR_TRANSACTION && !(t->flags & TF_ONE_WAY)) { - t->to_parent = thread->transaction_stack; - t->to_thread = thread; - thread->transaction_stack = t; - } else { - t->buffer->transaction = NULL; - kfree(t); - binder_stats_deleted(BINDER_STAT_TRANSACTION); - } - break; - } - -done: - - *consumed = ptr - buffer; - if (proc->requested_threads + proc->ready_threads == 0 && - proc->requested_threads_started < proc->max_threads && - (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | - BINDER_LOOPER_STATE_ENTERED)) /* the user-space code fails to */ - /*spawn a new thread if we leave this out */) { - proc->requested_threads++; - binder_debug(BINDER_DEBUG_THREADS, - "%d:%d BR_SPAWN_LOOPER\n", - proc->pid, thread->pid); - if (put_user(BR_SPAWN_LOOPER, (uint32_t __user *)buffer)) - return -EFAULT; - binder_stat_br(proc, thread, BR_SPAWN_LOOPER); - } - return 0; -} - -static void binder_release_work(struct list_head *list) -{ - struct binder_work *w; - - while (!list_empty(list)) { - w = list_first_entry(list, struct binder_work, entry); - list_del_init(&w->entry); - switch (w->type) { - case BINDER_WORK_TRANSACTION: { - struct binder_transaction *t; - - t = container_of(w, struct binder_transaction, work); - if (t->buffer->target_node && - !(t->flags & TF_ONE_WAY)) { - binder_send_failed_reply(t, BR_DEAD_REPLY); - } else { - binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, - "undelivered transaction %d\n", - t->debug_id); - t->buffer->transaction = NULL; - kfree(t); - binder_stats_deleted(BINDER_STAT_TRANSACTION); - } - } break; - case BINDER_WORK_TRANSACTION_COMPLETE: { - binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, - "undelivered TRANSACTION_COMPLETE\n"); - kfree(w); - binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE); - } break; - case BINDER_WORK_DEAD_BINDER_AND_CLEAR: - case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: { - struct binder_ref_death *death; - - death = container_of(w, struct binder_ref_death, work); - binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, - "undelivered death notification, %016llx\n", - (u64)death->cookie); - kfree(death); - binder_stats_deleted(BINDER_STAT_DEATH); - } break; - default: - pr_err("unexpected work type, %d, not freed\n", - w->type); - break; - } - } - -} - -static struct binder_thread *binder_get_thread(struct binder_proc *proc) -{ - struct binder_thread *thread = NULL; - struct rb_node *parent = NULL; - struct rb_node **p = &proc->threads.rb_node; - - while (*p) { - parent = *p; - thread = rb_entry(parent, struct binder_thread, rb_node); - - if (current->pid < thread->pid) - p = &(*p)->rb_left; - else if (current->pid > thread->pid) - p = &(*p)->rb_right; - else - break; - } - if (*p == NULL) { - thread = kzalloc(sizeof(*thread), GFP_KERNEL); - if (thread == NULL) - return NULL; - binder_stats_created(BINDER_STAT_THREAD); - thread->proc = proc; - thread->pid = current->pid; - init_waitqueue_head(&thread->wait); - INIT_LIST_HEAD(&thread->todo); - rb_link_node(&thread->rb_node, parent, p); - rb_insert_color(&thread->rb_node, &proc->threads); - thread->looper |= BINDER_LOOPER_STATE_NEED_RETURN; - thread->return_error = BR_OK; - thread->return_error2 = BR_OK; - } - return thread; -} - -static int binder_free_thread(struct binder_proc *proc, - struct binder_thread *thread) -{ - struct binder_transaction *t; - struct binder_transaction *send_reply = NULL; - int active_transactions = 0; - - rb_erase(&thread->rb_node, &proc->threads); - t = thread->transaction_stack; - if (t && t->to_thread == thread) - send_reply = t; - while (t) { - active_transactions++; - binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, - "release %d:%d transaction %d %s, still active\n", - proc->pid, thread->pid, - t->debug_id, - (t->to_thread == thread) ? "in" : "out"); - - if (t->to_thread == thread) { - t->to_proc = NULL; - t->to_thread = NULL; - if (t->buffer) { - t->buffer->transaction = NULL; - t->buffer = NULL; - } - t = t->to_parent; - } else if (t->from == thread) { - t->from = NULL; - t = t->from_parent; - } else - BUG(); - } - if (send_reply) - binder_send_failed_reply(send_reply, BR_DEAD_REPLY); - binder_release_work(&thread->todo); - kfree(thread); - binder_stats_deleted(BINDER_STAT_THREAD); - return active_transactions; -} - -static unsigned int binder_poll(struct file *filp, - struct poll_table_struct *wait) -{ - struct binder_proc *proc = filp->private_data; - struct binder_thread *thread = NULL; - int wait_for_proc_work; - - binder_lock(__func__); - - thread = binder_get_thread(proc); - - wait_for_proc_work = thread->transaction_stack == NULL && - list_empty(&thread->todo) && thread->return_error == BR_OK; - - binder_unlock(__func__); - - if (wait_for_proc_work) { - if (binder_has_proc_work(proc, thread)) - return POLLIN; - poll_wait(filp, &proc->wait, wait); - if (binder_has_proc_work(proc, thread)) - return POLLIN; - } else { - if (binder_has_thread_work(thread)) - return POLLIN; - poll_wait(filp, &thread->wait, wait); - if (binder_has_thread_work(thread)) - return POLLIN; - } - return 0; -} - -static int binder_ioctl_write_read(struct file *filp, - unsigned int cmd, unsigned long arg, - struct binder_thread *thread) -{ - int ret = 0; - struct binder_proc *proc = filp->private_data; - unsigned int size = _IOC_SIZE(cmd); - void __user *ubuf = (void __user *)arg; - struct binder_write_read bwr; - - if (size != sizeof(struct binder_write_read)) { - ret = -EINVAL; - goto out; - } - if (copy_from_user(&bwr, ubuf, sizeof(bwr))) { - ret = -EFAULT; - goto out; - } - binder_debug(BINDER_DEBUG_READ_WRITE, - "%d:%d write %lld at %016llx, read %lld at %016llx\n", - proc->pid, thread->pid, - (u64)bwr.write_size, (u64)bwr.write_buffer, - (u64)bwr.read_size, (u64)bwr.read_buffer); - - if (bwr.write_size > 0) { - ret = binder_thread_write(proc, thread, - bwr.write_buffer, - bwr.write_size, - &bwr.write_consumed); - trace_binder_write_done(ret); - if (ret < 0) { - bwr.read_consumed = 0; - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) - ret = -EFAULT; - goto out; - } - } - if (bwr.read_size > 0) { - ret = binder_thread_read(proc, thread, bwr.read_buffer, - bwr.read_size, - &bwr.read_consumed, - filp->f_flags & O_NONBLOCK); - trace_binder_read_done(ret); - if (!list_empty(&proc->todo)) - wake_up_interruptible(&proc->wait); - if (ret < 0) { - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) - ret = -EFAULT; - goto out; - } - } - binder_debug(BINDER_DEBUG_READ_WRITE, - "%d:%d wrote %lld of %lld, read return %lld of %lld\n", - proc->pid, thread->pid, - (u64)bwr.write_consumed, (u64)bwr.write_size, - (u64)bwr.read_consumed, (u64)bwr.read_size); - if (copy_to_user(ubuf, &bwr, sizeof(bwr))) { - ret = -EFAULT; - goto out; - } -out: - return ret; -} - -static int binder_ioctl_set_ctx_mgr(struct file *filp) -{ - int ret = 0; - struct binder_proc *proc = filp->private_data; - kuid_t curr_euid = current_euid(); - - if (binder_context_mgr_node != NULL) { - pr_err("BINDER_SET_CONTEXT_MGR already set\n"); - ret = -EBUSY; - goto out; - } - if (uid_valid(binder_context_mgr_uid)) { - if (!uid_eq(binder_context_mgr_uid, curr_euid)) { - pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", - from_kuid(&init_user_ns, curr_euid), - from_kuid(&init_user_ns, - binder_context_mgr_uid)); - ret = -EPERM; - goto out; - } - } else { - binder_context_mgr_uid = curr_euid; - } - binder_context_mgr_node = binder_new_node(proc, 0, 0); - if (binder_context_mgr_node == NULL) { - ret = -ENOMEM; - goto out; - } - binder_context_mgr_node->local_weak_refs++; - binder_context_mgr_node->local_strong_refs++; - binder_context_mgr_node->has_strong_ref = 1; - binder_context_mgr_node->has_weak_ref = 1; -out: - return ret; -} - -static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) -{ - int ret; - struct binder_proc *proc = filp->private_data; - struct binder_thread *thread; - unsigned int size = _IOC_SIZE(cmd); - void __user *ubuf = (void __user *)arg; - - /*pr_info("binder_ioctl: %d:%d %x %lx\n", - proc->pid, current->pid, cmd, arg);*/ - - trace_binder_ioctl(cmd, arg); - - ret = wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2); - if (ret) - goto err_unlocked; - - binder_lock(__func__); - thread = binder_get_thread(proc); - if (thread == NULL) { - ret = -ENOMEM; - goto err; - } - - switch (cmd) { - case BINDER_WRITE_READ: - ret = binder_ioctl_write_read(filp, cmd, arg, thread); - if (ret) - goto err; - break; - case BINDER_SET_MAX_THREADS: - if (copy_from_user(&proc->max_threads, ubuf, sizeof(proc->max_threads))) { - ret = -EINVAL; - goto err; - } - break; - case BINDER_SET_CONTEXT_MGR: - ret = binder_ioctl_set_ctx_mgr(filp); - if (ret) - goto err; - break; - case BINDER_THREAD_EXIT: - binder_debug(BINDER_DEBUG_THREADS, "%d:%d exit\n", - proc->pid, thread->pid); - binder_free_thread(proc, thread); - thread = NULL; - break; - case BINDER_VERSION: { - struct binder_version __user *ver = ubuf; - - if (size != sizeof(struct binder_version)) { - ret = -EINVAL; - goto err; - } - if (put_user(BINDER_CURRENT_PROTOCOL_VERSION, - &ver->protocol_version)) { - ret = -EINVAL; - goto err; - } - break; - } - default: - ret = -EINVAL; - goto err; - } - ret = 0; -err: - if (thread) - thread->looper &= ~BINDER_LOOPER_STATE_NEED_RETURN; - binder_unlock(__func__); - wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2); - if (ret && ret != -ERESTARTSYS) - pr_info("%d:%d ioctl %x %lx returned %d\n", proc->pid, current->pid, cmd, arg, ret); -err_unlocked: - trace_binder_ioctl_done(ret); - return ret; -} - -static void binder_vma_open(struct vm_area_struct *vma) -{ - struct binder_proc *proc = vma->vm_private_data; - - binder_debug(BINDER_DEBUG_OPEN_CLOSE, - "%d open vm area %lx-%lx (%ld K) vma %lx pagep %lx\n", - proc->pid, vma->vm_start, vma->vm_end, - (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags, - (unsigned long)pgprot_val(vma->vm_page_prot)); -} - -static void binder_vma_close(struct vm_area_struct *vma) -{ - struct binder_proc *proc = vma->vm_private_data; - - binder_debug(BINDER_DEBUG_OPEN_CLOSE, - "%d close vm area %lx-%lx (%ld K) vma %lx pagep %lx\n", - proc->pid, vma->vm_start, vma->vm_end, - (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags, - (unsigned long)pgprot_val(vma->vm_page_prot)); - proc->vma = NULL; - proc->vma_vm_mm = NULL; - binder_defer_work(proc, BINDER_DEFERRED_PUT_FILES); -} - -static int binder_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) -{ - return VM_FAULT_SIGBUS; -} - -static struct vm_operations_struct binder_vm_ops = { - .open = binder_vma_open, - .close = binder_vma_close, - .fault = binder_vm_fault, -}; - -static int binder_mmap(struct file *filp, struct vm_area_struct *vma) -{ - int ret; - struct vm_struct *area; - struct binder_proc *proc = filp->private_data; - const char *failure_string; - struct binder_buffer *buffer; - - if (proc->tsk != current) - return -EINVAL; - - if ((vma->vm_end - vma->vm_start) > SZ_4M) - vma->vm_end = vma->vm_start + SZ_4M; - - binder_debug(BINDER_DEBUG_OPEN_CLOSE, - "binder_mmap: %d %lx-%lx (%ld K) vma %lx pagep %lx\n", - proc->pid, vma->vm_start, vma->vm_end, - (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags, - (unsigned long)pgprot_val(vma->vm_page_prot)); - - if (vma->vm_flags & FORBIDDEN_MMAP_FLAGS) { - ret = -EPERM; - failure_string = "bad vm_flags"; - goto err_bad_arg; - } - vma->vm_flags = (vma->vm_flags | VM_DONTCOPY) & ~VM_MAYWRITE; - - mutex_lock(&binder_mmap_lock); - if (proc->buffer) { - ret = -EBUSY; - failure_string = "already mapped"; - goto err_already_mapped; - } - - area = get_vm_area(vma->vm_end - vma->vm_start, VM_IOREMAP); - if (area == NULL) { - ret = -ENOMEM; - failure_string = "get_vm_area"; - goto err_get_vm_area_failed; - } - proc->buffer = area->addr; - proc->user_buffer_offset = vma->vm_start - (uintptr_t)proc->buffer; - mutex_unlock(&binder_mmap_lock); - -#ifdef CONFIG_CPU_CACHE_VIPT - if (cache_is_vipt_aliasing()) { - while (CACHE_COLOUR((vma->vm_start ^ (uint32_t)proc->buffer))) { - pr_info("binder_mmap: %d %lx-%lx maps %p bad alignment\n", proc->pid, vma->vm_start, vma->vm_end, proc->buffer); - vma->vm_start += PAGE_SIZE; - } - } -#endif - proc->pages = kzalloc(sizeof(proc->pages[0]) * ((vma->vm_end - vma->vm_start) / PAGE_SIZE), GFP_KERNEL); - if (proc->pages == NULL) { - ret = -ENOMEM; - failure_string = "alloc page array"; - goto err_alloc_pages_failed; - } - proc->buffer_size = vma->vm_end - vma->vm_start; - - vma->vm_ops = &binder_vm_ops; - vma->vm_private_data = proc; - - if (binder_update_page_range(proc, 1, proc->buffer, proc->buffer + PAGE_SIZE, vma)) { - ret = -ENOMEM; - failure_string = "alloc small buf"; - goto err_alloc_small_buf_failed; - } - buffer = proc->buffer; - INIT_LIST_HEAD(&proc->buffers); - list_add(&buffer->entry, &proc->buffers); - buffer->free = 1; - binder_insert_free_buffer(proc, buffer); - proc->free_async_space = proc->buffer_size / 2; - barrier(); - proc->files = get_files_struct(current); - proc->vma = vma; - proc->vma_vm_mm = vma->vm_mm; - - /*pr_info("binder_mmap: %d %lx-%lx maps %p\n", - proc->pid, vma->vm_start, vma->vm_end, proc->buffer);*/ - return 0; - -err_alloc_small_buf_failed: - kfree(proc->pages); - proc->pages = NULL; -err_alloc_pages_failed: - mutex_lock(&binder_mmap_lock); - vfree(proc->buffer); - proc->buffer = NULL; -err_get_vm_area_failed: -err_already_mapped: - mutex_unlock(&binder_mmap_lock); -err_bad_arg: - pr_err("binder_mmap: %d %lx-%lx %s failed %d\n", - proc->pid, vma->vm_start, vma->vm_end, failure_string, ret); - return ret; -} - -static int binder_open(struct inode *nodp, struct file *filp) -{ - struct binder_proc *proc; - - binder_debug(BINDER_DEBUG_OPEN_CLOSE, "binder_open: %d:%d\n", - current->group_leader->pid, current->pid); - - proc = kzalloc(sizeof(*proc), GFP_KERNEL); - if (proc == NULL) - return -ENOMEM; - get_task_struct(current); - proc->tsk = current; - INIT_LIST_HEAD(&proc->todo); - init_waitqueue_head(&proc->wait); - proc->default_priority = task_nice(current); - - binder_lock(__func__); - - binder_stats_created(BINDER_STAT_PROC); - hlist_add_head(&proc->proc_node, &binder_procs); - proc->pid = current->group_leader->pid; - INIT_LIST_HEAD(&proc->delivered_death); - filp->private_data = proc; - - binder_unlock(__func__); - - if (binder_debugfs_dir_entry_proc) { - char strbuf[11]; - - snprintf(strbuf, sizeof(strbuf), "%u", proc->pid); - proc->debugfs_entry = debugfs_create_file(strbuf, S_IRUGO, - binder_debugfs_dir_entry_proc, proc, &binder_proc_fops); - } - - return 0; -} - -static int binder_flush(struct file *filp, fl_owner_t id) -{ - struct binder_proc *proc = filp->private_data; - - binder_defer_work(proc, BINDER_DEFERRED_FLUSH); - - return 0; -} - -static void binder_deferred_flush(struct binder_proc *proc) -{ - struct rb_node *n; - int wake_count = 0; - - for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) { - struct binder_thread *thread = rb_entry(n, struct binder_thread, rb_node); - - thread->looper |= BINDER_LOOPER_STATE_NEED_RETURN; - if (thread->looper & BINDER_LOOPER_STATE_WAITING) { - wake_up_interruptible(&thread->wait); - wake_count++; - } - } - wake_up_interruptible_all(&proc->wait); - - binder_debug(BINDER_DEBUG_OPEN_CLOSE, - "binder_flush: %d woke %d threads\n", proc->pid, - wake_count); -} - -static int binder_release(struct inode *nodp, struct file *filp) -{ - struct binder_proc *proc = filp->private_data; - - debugfs_remove(proc->debugfs_entry); - binder_defer_work(proc, BINDER_DEFERRED_RELEASE); - - return 0; -} - -static int binder_node_release(struct binder_node *node, int refs) -{ - struct binder_ref *ref; - int death = 0; - - list_del_init(&node->work.entry); - binder_release_work(&node->async_todo); - - if (hlist_empty(&node->refs)) { - kfree(node); - binder_stats_deleted(BINDER_STAT_NODE); - - return refs; - } - - node->proc = NULL; - node->local_strong_refs = 0; - node->local_weak_refs = 0; - hlist_add_head(&node->dead_node, &binder_dead_nodes); - - hlist_for_each_entry(ref, &node->refs, node_entry) { - refs++; - - if (!ref->death) - continue; - - death++; - - if (list_empty(&ref->death->work.entry)) { - ref->death->work.type = BINDER_WORK_DEAD_BINDER; - list_add_tail(&ref->death->work.entry, - &ref->proc->todo); - wake_up_interruptible(&ref->proc->wait); - } else - BUG(); - } - - binder_debug(BINDER_DEBUG_DEAD_BINDER, - "node %d now dead, refs %d, death %d\n", - node->debug_id, refs, death); - - return refs; -} - -static void binder_deferred_release(struct binder_proc *proc) -{ - struct binder_transaction *t; - struct rb_node *n; - int threads, nodes, incoming_refs, outgoing_refs, buffers, - active_transactions, page_count; - - BUG_ON(proc->vma); - BUG_ON(proc->files); - - hlist_del(&proc->proc_node); - - if (binder_context_mgr_node && binder_context_mgr_node->proc == proc) { - binder_debug(BINDER_DEBUG_DEAD_BINDER, - "%s: %d context_mgr_node gone\n", - __func__, proc->pid); - binder_context_mgr_node = NULL; - } - - threads = 0; - active_transactions = 0; - while ((n = rb_first(&proc->threads))) { - struct binder_thread *thread; - - thread = rb_entry(n, struct binder_thread, rb_node); - threads++; - active_transactions += binder_free_thread(proc, thread); - } - - nodes = 0; - incoming_refs = 0; - while ((n = rb_first(&proc->nodes))) { - struct binder_node *node; - - node = rb_entry(n, struct binder_node, rb_node); - nodes++; - rb_erase(&node->rb_node, &proc->nodes); - incoming_refs = binder_node_release(node, incoming_refs); - } - - outgoing_refs = 0; - while ((n = rb_first(&proc->refs_by_desc))) { - struct binder_ref *ref; - - ref = rb_entry(n, struct binder_ref, rb_node_desc); - outgoing_refs++; - binder_delete_ref(ref); - } - - binder_release_work(&proc->todo); - binder_release_work(&proc->delivered_death); - - buffers = 0; - while ((n = rb_first(&proc->allocated_buffers))) { - struct binder_buffer *buffer; - - buffer = rb_entry(n, struct binder_buffer, rb_node); - - t = buffer->transaction; - if (t) { - t->buffer = NULL; - buffer->transaction = NULL; - pr_err("release proc %d, transaction %d, not freed\n", - proc->pid, t->debug_id); - /*BUG();*/ - } - - binder_free_buf(proc, buffer); - buffers++; - } - - binder_stats_deleted(BINDER_STAT_PROC); - - page_count = 0; - if (proc->pages) { - int i; - - for (i = 0; i < proc->buffer_size / PAGE_SIZE; i++) { - void *page_addr; - - if (!proc->pages[i]) - continue; - - page_addr = proc->buffer + i * PAGE_SIZE; - binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%s: %d: page %d at %p not freed\n", - __func__, proc->pid, i, page_addr); - unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE); - __free_page(proc->pages[i]); - page_count++; - } - kfree(proc->pages); - vfree(proc->buffer); - } - - put_task_struct(proc->tsk); - - binder_debug(BINDER_DEBUG_OPEN_CLOSE, - "%s: %d threads %d, nodes %d (ref %d), refs %d, active transactions %d, buffers %d, pages %d\n", - __func__, proc->pid, threads, nodes, incoming_refs, - outgoing_refs, active_transactions, buffers, page_count); - - kfree(proc); -} - -static void binder_deferred_func(struct work_struct *work) -{ - struct binder_proc *proc; - struct files_struct *files; - - int defer; - - do { - binder_lock(__func__); - mutex_lock(&binder_deferred_lock); - if (!hlist_empty(&binder_deferred_list)) { - proc = hlist_entry(binder_deferred_list.first, - struct binder_proc, deferred_work_node); - hlist_del_init(&proc->deferred_work_node); - defer = proc->deferred_work; - proc->deferred_work = 0; - } else { - proc = NULL; - defer = 0; - } - mutex_unlock(&binder_deferred_lock); - - files = NULL; - if (defer & BINDER_DEFERRED_PUT_FILES) { - files = proc->files; - if (files) - proc->files = NULL; - } - - if (defer & BINDER_DEFERRED_FLUSH) - binder_deferred_flush(proc); - - if (defer & BINDER_DEFERRED_RELEASE) - binder_deferred_release(proc); /* frees proc */ - - binder_unlock(__func__); - if (files) - put_files_struct(files); - } while (proc); -} -static DECLARE_WORK(binder_deferred_work, binder_deferred_func); - -static void -binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer) -{ - mutex_lock(&binder_deferred_lock); - proc->deferred_work |= defer; - if (hlist_unhashed(&proc->deferred_work_node)) { - hlist_add_head(&proc->deferred_work_node, - &binder_deferred_list); - queue_work(binder_deferred_workqueue, &binder_deferred_work); - } - mutex_unlock(&binder_deferred_lock); -} - -static void print_binder_transaction(struct seq_file *m, const char *prefix, - struct binder_transaction *t) -{ - seq_printf(m, - "%s %d: %p from %d:%d to %d:%d code %x flags %x pri %ld r%d", - prefix, t->debug_id, t, - t->from ? t->from->proc->pid : 0, - t->from ? t->from->pid : 0, - t->to_proc ? t->to_proc->pid : 0, - t->to_thread ? t->to_thread->pid : 0, - t->code, t->flags, t->priority, t->need_reply); - if (t->buffer == NULL) { - seq_puts(m, " buffer free\n"); - return; - } - if (t->buffer->target_node) - seq_printf(m, " node %d", - t->buffer->target_node->debug_id); - seq_printf(m, " size %zd:%zd data %p\n", - t->buffer->data_size, t->buffer->offsets_size, - t->buffer->data); -} - -static void print_binder_buffer(struct seq_file *m, const char *prefix, - struct binder_buffer *buffer) -{ - seq_printf(m, "%s %d: %p size %zd:%zd %s\n", - prefix, buffer->debug_id, buffer->data, - buffer->data_size, buffer->offsets_size, - buffer->transaction ? "active" : "delivered"); -} - -static void print_binder_work(struct seq_file *m, const char *prefix, - const char *transaction_prefix, - struct binder_work *w) -{ - struct binder_node *node; - struct binder_transaction *t; - - switch (w->type) { - case BINDER_WORK_TRANSACTION: - t = container_of(w, struct binder_transaction, work); - print_binder_transaction(m, transaction_prefix, t); - break; - case BINDER_WORK_TRANSACTION_COMPLETE: - seq_printf(m, "%stransaction complete\n", prefix); - break; - case BINDER_WORK_NODE: - node = container_of(w, struct binder_node, work); - seq_printf(m, "%snode work %d: u%016llx c%016llx\n", - prefix, node->debug_id, - (u64)node->ptr, (u64)node->cookie); - break; - case BINDER_WORK_DEAD_BINDER: - seq_printf(m, "%shas dead binder\n", prefix); - break; - case BINDER_WORK_DEAD_BINDER_AND_CLEAR: - seq_printf(m, "%shas cleared dead binder\n", prefix); - break; - case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: - seq_printf(m, "%shas cleared death notification\n", prefix); - break; - default: - seq_printf(m, "%sunknown work: type %d\n", prefix, w->type); - break; - } -} - -static void print_binder_thread(struct seq_file *m, - struct binder_thread *thread, - int print_always) -{ - struct binder_transaction *t; - struct binder_work *w; - size_t start_pos = m->count; - size_t header_pos; - - seq_printf(m, " thread %d: l %02x\n", thread->pid, thread->looper); - header_pos = m->count; - t = thread->transaction_stack; - while (t) { - if (t->from == thread) { - print_binder_transaction(m, - " outgoing transaction", t); - t = t->from_parent; - } else if (t->to_thread == thread) { - print_binder_transaction(m, - " incoming transaction", t); - t = t->to_parent; - } else { - print_binder_transaction(m, " bad transaction", t); - t = NULL; - } - } - list_for_each_entry(w, &thread->todo, entry) { - print_binder_work(m, " ", " pending transaction", w); - } - if (!print_always && m->count == header_pos) - m->count = start_pos; -} - -static void print_binder_node(struct seq_file *m, struct binder_node *node) -{ - struct binder_ref *ref; - struct binder_work *w; - int count; - - count = 0; - hlist_for_each_entry(ref, &node->refs, node_entry) - count++; - - seq_printf(m, " node %d: u%016llx c%016llx hs %d hw %d ls %d lw %d is %d iw %d", - node->debug_id, (u64)node->ptr, (u64)node->cookie, - node->has_strong_ref, node->has_weak_ref, - node->local_strong_refs, node->local_weak_refs, - node->internal_strong_refs, count); - if (count) { - seq_puts(m, " proc"); - hlist_for_each_entry(ref, &node->refs, node_entry) - seq_printf(m, " %d", ref->proc->pid); - } - seq_puts(m, "\n"); - list_for_each_entry(w, &node->async_todo, entry) - print_binder_work(m, " ", - " pending async transaction", w); -} - -static void print_binder_ref(struct seq_file *m, struct binder_ref *ref) -{ - seq_printf(m, " ref %d: desc %d %snode %d s %d w %d d %p\n", - ref->debug_id, ref->desc, ref->node->proc ? "" : "dead ", - ref->node->debug_id, ref->strong, ref->weak, ref->death); -} - -static void print_binder_proc(struct seq_file *m, - struct binder_proc *proc, int print_all) -{ - struct binder_work *w; - struct rb_node *n; - size_t start_pos = m->count; - size_t header_pos; - - seq_printf(m, "proc %d\n", proc->pid); - header_pos = m->count; - - for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) - print_binder_thread(m, rb_entry(n, struct binder_thread, - rb_node), print_all); - for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) { - struct binder_node *node = rb_entry(n, struct binder_node, - rb_node); - if (print_all || node->has_async_transaction) - print_binder_node(m, node); - } - if (print_all) { - for (n = rb_first(&proc->refs_by_desc); - n != NULL; - n = rb_next(n)) - print_binder_ref(m, rb_entry(n, struct binder_ref, - rb_node_desc)); - } - for (n = rb_first(&proc->allocated_buffers); n != NULL; n = rb_next(n)) - print_binder_buffer(m, " buffer", - rb_entry(n, struct binder_buffer, rb_node)); - list_for_each_entry(w, &proc->todo, entry) - print_binder_work(m, " ", " pending transaction", w); - list_for_each_entry(w, &proc->delivered_death, entry) { - seq_puts(m, " has delivered dead binder\n"); - break; - } - if (!print_all && m->count == header_pos) - m->count = start_pos; -} - -static const char * const binder_return_strings[] = { - "BR_ERROR", - "BR_OK", - "BR_TRANSACTION", - "BR_REPLY", - "BR_ACQUIRE_RESULT", - "BR_DEAD_REPLY", - "BR_TRANSACTION_COMPLETE", - "BR_INCREFS", - "BR_ACQUIRE", - "BR_RELEASE", - "BR_DECREFS", - "BR_ATTEMPT_ACQUIRE", - "BR_NOOP", - "BR_SPAWN_LOOPER", - "BR_FINISHED", - "BR_DEAD_BINDER", - "BR_CLEAR_DEATH_NOTIFICATION_DONE", - "BR_FAILED_REPLY" -}; - -static const char * const binder_command_strings[] = { - "BC_TRANSACTION", - "BC_REPLY", - "BC_ACQUIRE_RESULT", - "BC_FREE_BUFFER", - "BC_INCREFS", - "BC_ACQUIRE", - "BC_RELEASE", - "BC_DECREFS", - "BC_INCREFS_DONE", - "BC_ACQUIRE_DONE", - "BC_ATTEMPT_ACQUIRE", - "BC_REGISTER_LOOPER", - "BC_ENTER_LOOPER", - "BC_EXIT_LOOPER", - "BC_REQUEST_DEATH_NOTIFICATION", - "BC_CLEAR_DEATH_NOTIFICATION", - "BC_DEAD_BINDER_DONE" -}; - -static const char * const binder_objstat_strings[] = { - "proc", - "thread", - "node", - "ref", - "death", - "transaction", - "transaction_complete" -}; - -static void print_binder_stats(struct seq_file *m, const char *prefix, - struct binder_stats *stats) -{ - int i; - - BUILD_BUG_ON(ARRAY_SIZE(stats->bc) != - ARRAY_SIZE(binder_command_strings)); - for (i = 0; i < ARRAY_SIZE(stats->bc); i++) { - if (stats->bc[i]) - seq_printf(m, "%s%s: %d\n", prefix, - binder_command_strings[i], stats->bc[i]); - } - - BUILD_BUG_ON(ARRAY_SIZE(stats->br) != - ARRAY_SIZE(binder_return_strings)); - for (i = 0; i < ARRAY_SIZE(stats->br); i++) { - if (stats->br[i]) - seq_printf(m, "%s%s: %d\n", prefix, - binder_return_strings[i], stats->br[i]); - } - - BUILD_BUG_ON(ARRAY_SIZE(stats->obj_created) != - ARRAY_SIZE(binder_objstat_strings)); - BUILD_BUG_ON(ARRAY_SIZE(stats->obj_created) != - ARRAY_SIZE(stats->obj_deleted)); - for (i = 0; i < ARRAY_SIZE(stats->obj_created); i++) { - if (stats->obj_created[i] || stats->obj_deleted[i]) - seq_printf(m, "%s%s: active %d total %d\n", prefix, - binder_objstat_strings[i], - stats->obj_created[i] - stats->obj_deleted[i], - stats->obj_created[i]); - } -} - -static void print_binder_proc_stats(struct seq_file *m, - struct binder_proc *proc) -{ - struct binder_work *w; - struct rb_node *n; - int count, strong, weak; - - seq_printf(m, "proc %d\n", proc->pid); - count = 0; - for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) - count++; - seq_printf(m, " threads: %d\n", count); - seq_printf(m, " requested threads: %d+%d/%d\n" - " ready threads %d\n" - " free async space %zd\n", proc->requested_threads, - proc->requested_threads_started, proc->max_threads, - proc->ready_threads, proc->free_async_space); - count = 0; - for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) - count++; - seq_printf(m, " nodes: %d\n", count); - count = 0; - strong = 0; - weak = 0; - for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) { - struct binder_ref *ref = rb_entry(n, struct binder_ref, - rb_node_desc); - count++; - strong += ref->strong; - weak += ref->weak; - } - seq_printf(m, " refs: %d s %d w %d\n", count, strong, weak); - - count = 0; - for (n = rb_first(&proc->allocated_buffers); n != NULL; n = rb_next(n)) - count++; - seq_printf(m, " buffers: %d\n", count); - - count = 0; - list_for_each_entry(w, &proc->todo, entry) { - switch (w->type) { - case BINDER_WORK_TRANSACTION: - count++; - break; - default: - break; - } - } - seq_printf(m, " pending transactions: %d\n", count); - - print_binder_stats(m, " ", &proc->stats); -} - - -static int binder_state_show(struct seq_file *m, void *unused) -{ - struct binder_proc *proc; - struct binder_node *node; - int do_lock = !binder_debug_no_lock; - - if (do_lock) - binder_lock(__func__); - - seq_puts(m, "binder state:\n"); - - if (!hlist_empty(&binder_dead_nodes)) - seq_puts(m, "dead nodes:\n"); - hlist_for_each_entry(node, &binder_dead_nodes, dead_node) - print_binder_node(m, node); - - hlist_for_each_entry(proc, &binder_procs, proc_node) - print_binder_proc(m, proc, 1); - if (do_lock) - binder_unlock(__func__); - return 0; -} - -static int binder_stats_show(struct seq_file *m, void *unused) -{ - struct binder_proc *proc; - int do_lock = !binder_debug_no_lock; - - if (do_lock) - binder_lock(__func__); - - seq_puts(m, "binder stats:\n"); - - print_binder_stats(m, "", &binder_stats); - - hlist_for_each_entry(proc, &binder_procs, proc_node) - print_binder_proc_stats(m, proc); - if (do_lock) - binder_unlock(__func__); - return 0; -} - -static int binder_transactions_show(struct seq_file *m, void *unused) -{ - struct binder_proc *proc; - int do_lock = !binder_debug_no_lock; - - if (do_lock) - binder_lock(__func__); - - seq_puts(m, "binder transactions:\n"); - hlist_for_each_entry(proc, &binder_procs, proc_node) - print_binder_proc(m, proc, 0); - if (do_lock) - binder_unlock(__func__); - return 0; -} - -static int binder_proc_show(struct seq_file *m, void *unused) -{ - struct binder_proc *proc = m->private; - int do_lock = !binder_debug_no_lock; - - if (do_lock) - binder_lock(__func__); - seq_puts(m, "binder proc state:\n"); - print_binder_proc(m, proc, 1); - if (do_lock) - binder_unlock(__func__); - return 0; -} - -static void print_binder_transaction_log_entry(struct seq_file *m, - struct binder_transaction_log_entry *e) -{ - seq_printf(m, - "%d: %s from %d:%d to %d:%d node %d handle %d size %d:%d\n", - e->debug_id, (e->call_type == 2) ? "reply" : - ((e->call_type == 1) ? "async" : "call "), e->from_proc, - e->from_thread, e->to_proc, e->to_thread, e->to_node, - e->target_handle, e->data_size, e->offsets_size); -} - -static int binder_transaction_log_show(struct seq_file *m, void *unused) -{ - struct binder_transaction_log *log = m->private; - int i; - - if (log->full) { - for (i = log->next; i < ARRAY_SIZE(log->entry); i++) - print_binder_transaction_log_entry(m, &log->entry[i]); - } - for (i = 0; i < log->next; i++) - print_binder_transaction_log_entry(m, &log->entry[i]); - return 0; -} - -static const struct file_operations binder_fops = { - .owner = THIS_MODULE, - .poll = binder_poll, - .unlocked_ioctl = binder_ioctl, - .compat_ioctl = binder_ioctl, - .mmap = binder_mmap, - .open = binder_open, - .flush = binder_flush, - .release = binder_release, -}; - -static struct miscdevice binder_miscdev = { - .minor = MISC_DYNAMIC_MINOR, - .name = "binder", - .fops = &binder_fops -}; - -BINDER_DEBUG_ENTRY(state); -BINDER_DEBUG_ENTRY(stats); -BINDER_DEBUG_ENTRY(transactions); -BINDER_DEBUG_ENTRY(transaction_log); - -static int __init binder_init(void) -{ - int ret; - - binder_deferred_workqueue = create_singlethread_workqueue("binder"); - if (!binder_deferred_workqueue) - return -ENOMEM; - - binder_debugfs_dir_entry_root = debugfs_create_dir("binder", NULL); - if (binder_debugfs_dir_entry_root) - binder_debugfs_dir_entry_proc = debugfs_create_dir("proc", - binder_debugfs_dir_entry_root); - ret = misc_register(&binder_miscdev); - if (binder_debugfs_dir_entry_root) { - debugfs_create_file("state", - S_IRUGO, - binder_debugfs_dir_entry_root, - NULL, - &binder_state_fops); - debugfs_create_file("stats", - S_IRUGO, - binder_debugfs_dir_entry_root, - NULL, - &binder_stats_fops); - debugfs_create_file("transactions", - S_IRUGO, - binder_debugfs_dir_entry_root, - NULL, - &binder_transactions_fops); - debugfs_create_file("transaction_log", - S_IRUGO, - binder_debugfs_dir_entry_root, - &binder_transaction_log, - &binder_transaction_log_fops); - debugfs_create_file("failed_transaction_log", - S_IRUGO, - binder_debugfs_dir_entry_root, - &binder_transaction_log_failed, - &binder_transaction_log_fops); - } - return ret; -} - -device_initcall(binder_init); - -#define CREATE_TRACE_POINTS -#include "binder_trace.h" - -MODULE_LICENSE("GPL v2"); diff --git a/drivers/staging/android/binder.h b/drivers/staging/android/binder.h deleted file mode 100644 index eb0834656dfe..000000000000 --- a/drivers/staging/android/binder.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2008 Google, Inc. - * - * Based on, but no longer compatible with, the original - * OpenBinder.org binder driver interface, which is: - * - * Copyright (c) 2005 Palmsource, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#ifndef _LINUX_BINDER_H -#define _LINUX_BINDER_H - -#ifdef CONFIG_ANDROID_BINDER_IPC_32BIT -#define BINDER_IPC_32BIT 1 -#endif - -#include "uapi/binder.h" - -#endif /* _LINUX_BINDER_H */ - diff --git a/drivers/staging/android/binder_trace.h b/drivers/staging/android/binder_trace.h deleted file mode 100644 index 7f20f3dc8369..000000000000 --- a/drivers/staging/android/binder_trace.h +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright (C) 2012 Google, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#undef TRACE_SYSTEM -#define TRACE_SYSTEM binder - -#if !defined(_BINDER_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) -#define _BINDER_TRACE_H - -#include - -struct binder_buffer; -struct binder_node; -struct binder_proc; -struct binder_ref; -struct binder_thread; -struct binder_transaction; - -TRACE_EVENT(binder_ioctl, - TP_PROTO(unsigned int cmd, unsigned long arg), - TP_ARGS(cmd, arg), - - TP_STRUCT__entry( - __field(unsigned int, cmd) - __field(unsigned long, arg) - ), - TP_fast_assign( - __entry->cmd = cmd; - __entry->arg = arg; - ), - TP_printk("cmd=0x%x arg=0x%lx", __entry->cmd, __entry->arg) -); - -DECLARE_EVENT_CLASS(binder_lock_class, - TP_PROTO(const char *tag), - TP_ARGS(tag), - TP_STRUCT__entry( - __field(const char *, tag) - ), - TP_fast_assign( - __entry->tag = tag; - ), - TP_printk("tag=%s", __entry->tag) -); - -#define DEFINE_BINDER_LOCK_EVENT(name) \ -DEFINE_EVENT(binder_lock_class, name, \ - TP_PROTO(const char *func), \ - TP_ARGS(func)) - -DEFINE_BINDER_LOCK_EVENT(binder_lock); -DEFINE_BINDER_LOCK_EVENT(binder_locked); -DEFINE_BINDER_LOCK_EVENT(binder_unlock); - -DECLARE_EVENT_CLASS(binder_function_return_class, - TP_PROTO(int ret), - TP_ARGS(ret), - TP_STRUCT__entry( - __field(int, ret) - ), - TP_fast_assign( - __entry->ret = ret; - ), - TP_printk("ret=%d", __entry->ret) -); - -#define DEFINE_BINDER_FUNCTION_RETURN_EVENT(name) \ -DEFINE_EVENT(binder_function_return_class, name, \ - TP_PROTO(int ret), \ - TP_ARGS(ret)) - -DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_ioctl_done); -DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_write_done); -DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_read_done); - -TRACE_EVENT(binder_wait_for_work, - TP_PROTO(bool proc_work, bool transaction_stack, bool thread_todo), - TP_ARGS(proc_work, transaction_stack, thread_todo), - - TP_STRUCT__entry( - __field(bool, proc_work) - __field(bool, transaction_stack) - __field(bool, thread_todo) - ), - TP_fast_assign( - __entry->proc_work = proc_work; - __entry->transaction_stack = transaction_stack; - __entry->thread_todo = thread_todo; - ), - TP_printk("proc_work=%d transaction_stack=%d thread_todo=%d", - __entry->proc_work, __entry->transaction_stack, - __entry->thread_todo) -); - -TRACE_EVENT(binder_transaction, - TP_PROTO(bool reply, struct binder_transaction *t, - struct binder_node *target_node), - TP_ARGS(reply, t, target_node), - TP_STRUCT__entry( - __field(int, debug_id) - __field(int, target_node) - __field(int, to_proc) - __field(int, to_thread) - __field(int, reply) - __field(unsigned int, code) - __field(unsigned int, flags) - ), - TP_fast_assign( - __entry->debug_id = t->debug_id; - __entry->target_node = target_node ? target_node->debug_id : 0; - __entry->to_proc = t->to_proc->pid; - __entry->to_thread = t->to_thread ? t->to_thread->pid : 0; - __entry->reply = reply; - __entry->code = t->code; - __entry->flags = t->flags; - ), - TP_printk("transaction=%d dest_node=%d dest_proc=%d dest_thread=%d reply=%d flags=0x%x code=0x%x", - __entry->debug_id, __entry->target_node, - __entry->to_proc, __entry->to_thread, - __entry->reply, __entry->flags, __entry->code) -); - -TRACE_EVENT(binder_transaction_received, - TP_PROTO(struct binder_transaction *t), - TP_ARGS(t), - - TP_STRUCT__entry( - __field(int, debug_id) - ), - TP_fast_assign( - __entry->debug_id = t->debug_id; - ), - TP_printk("transaction=%d", __entry->debug_id) -); - -TRACE_EVENT(binder_transaction_node_to_ref, - TP_PROTO(struct binder_transaction *t, struct binder_node *node, - struct binder_ref *ref), - TP_ARGS(t, node, ref), - - TP_STRUCT__entry( - __field(int, debug_id) - __field(int, node_debug_id) - __field(binder_uintptr_t, node_ptr) - __field(int, ref_debug_id) - __field(uint32_t, ref_desc) - ), - TP_fast_assign( - __entry->debug_id = t->debug_id; - __entry->node_debug_id = node->debug_id; - __entry->node_ptr = node->ptr; - __entry->ref_debug_id = ref->debug_id; - __entry->ref_desc = ref->desc; - ), - TP_printk("transaction=%d node=%d src_ptr=0x%016llx ==> dest_ref=%d dest_desc=%d", - __entry->debug_id, __entry->node_debug_id, - (u64)__entry->node_ptr, - __entry->ref_debug_id, __entry->ref_desc) -); - -TRACE_EVENT(binder_transaction_ref_to_node, - TP_PROTO(struct binder_transaction *t, struct binder_ref *ref), - TP_ARGS(t, ref), - - TP_STRUCT__entry( - __field(int, debug_id) - __field(int, ref_debug_id) - __field(uint32_t, ref_desc) - __field(int, node_debug_id) - __field(binder_uintptr_t, node_ptr) - ), - TP_fast_assign( - __entry->debug_id = t->debug_id; - __entry->ref_debug_id = ref->debug_id; - __entry->ref_desc = ref->desc; - __entry->node_debug_id = ref->node->debug_id; - __entry->node_ptr = ref->node->ptr; - ), - TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%016llx", - __entry->debug_id, __entry->node_debug_id, - __entry->ref_debug_id, __entry->ref_desc, - (u64)__entry->node_ptr) -); - -TRACE_EVENT(binder_transaction_ref_to_ref, - TP_PROTO(struct binder_transaction *t, struct binder_ref *src_ref, - struct binder_ref *dest_ref), - TP_ARGS(t, src_ref, dest_ref), - - TP_STRUCT__entry( - __field(int, debug_id) - __field(int, node_debug_id) - __field(int, src_ref_debug_id) - __field(uint32_t, src_ref_desc) - __field(int, dest_ref_debug_id) - __field(uint32_t, dest_ref_desc) - ), - TP_fast_assign( - __entry->debug_id = t->debug_id; - __entry->node_debug_id = src_ref->node->debug_id; - __entry->src_ref_debug_id = src_ref->debug_id; - __entry->src_ref_desc = src_ref->desc; - __entry->dest_ref_debug_id = dest_ref->debug_id; - __entry->dest_ref_desc = dest_ref->desc; - ), - TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ref=%d dest_desc=%d", - __entry->debug_id, __entry->node_debug_id, - __entry->src_ref_debug_id, __entry->src_ref_desc, - __entry->dest_ref_debug_id, __entry->dest_ref_desc) -); - -TRACE_EVENT(binder_transaction_fd, - TP_PROTO(struct binder_transaction *t, int src_fd, int dest_fd), - TP_ARGS(t, src_fd, dest_fd), - - TP_STRUCT__entry( - __field(int, debug_id) - __field(int, src_fd) - __field(int, dest_fd) - ), - TP_fast_assign( - __entry->debug_id = t->debug_id; - __entry->src_fd = src_fd; - __entry->dest_fd = dest_fd; - ), - TP_printk("transaction=%d src_fd=%d ==> dest_fd=%d", - __entry->debug_id, __entry->src_fd, __entry->dest_fd) -); - -DECLARE_EVENT_CLASS(binder_buffer_class, - TP_PROTO(struct binder_buffer *buf), - TP_ARGS(buf), - TP_STRUCT__entry( - __field(int, debug_id) - __field(size_t, data_size) - __field(size_t, offsets_size) - ), - TP_fast_assign( - __entry->debug_id = buf->debug_id; - __entry->data_size = buf->data_size; - __entry->offsets_size = buf->offsets_size; - ), - TP_printk("transaction=%d data_size=%zd offsets_size=%zd", - __entry->debug_id, __entry->data_size, __entry->offsets_size) -); - -DEFINE_EVENT(binder_buffer_class, binder_transaction_alloc_buf, - TP_PROTO(struct binder_buffer *buffer), - TP_ARGS(buffer)); - -DEFINE_EVENT(binder_buffer_class, binder_transaction_buffer_release, - TP_PROTO(struct binder_buffer *buffer), - TP_ARGS(buffer)); - -DEFINE_EVENT(binder_buffer_class, binder_transaction_failed_buffer_release, - TP_PROTO(struct binder_buffer *buffer), - TP_ARGS(buffer)); - -TRACE_EVENT(binder_update_page_range, - TP_PROTO(struct binder_proc *proc, bool allocate, - void *start, void *end), - TP_ARGS(proc, allocate, start, end), - TP_STRUCT__entry( - __field(int, proc) - __field(bool, allocate) - __field(size_t, offset) - __field(size_t, size) - ), - TP_fast_assign( - __entry->proc = proc->pid; - __entry->allocate = allocate; - __entry->offset = start - proc->buffer; - __entry->size = end - start; - ), - TP_printk("proc=%d allocate=%d offset=%zu size=%zu", - __entry->proc, __entry->allocate, - __entry->offset, __entry->size) -); - -TRACE_EVENT(binder_command, - TP_PROTO(uint32_t cmd), - TP_ARGS(cmd), - TP_STRUCT__entry( - __field(uint32_t, cmd) - ), - TP_fast_assign( - __entry->cmd = cmd; - ), - TP_printk("cmd=0x%x %s", - __entry->cmd, - _IOC_NR(__entry->cmd) < ARRAY_SIZE(binder_command_strings) ? - binder_command_strings[_IOC_NR(__entry->cmd)] : - "unknown") -); - -TRACE_EVENT(binder_return, - TP_PROTO(uint32_t cmd), - TP_ARGS(cmd), - TP_STRUCT__entry( - __field(uint32_t, cmd) - ), - TP_fast_assign( - __entry->cmd = cmd; - ), - TP_printk("cmd=0x%x %s", - __entry->cmd, - _IOC_NR(__entry->cmd) < ARRAY_SIZE(binder_return_strings) ? - binder_return_strings[_IOC_NR(__entry->cmd)] : - "unknown") -); - -#endif /* _BINDER_TRACE_H */ - -#undef TRACE_INCLUDE_PATH -#undef TRACE_INCLUDE_FILE -#define TRACE_INCLUDE_PATH . -#define TRACE_INCLUDE_FILE binder_trace -#include diff --git a/drivers/staging/android/uapi/binder.h b/drivers/staging/android/uapi/binder.h deleted file mode 100644 index c653800b3a1a..000000000000 --- a/drivers/staging/android/uapi/binder.h +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Copyright (C) 2008 Google, Inc. - * - * Based on, but no longer compatible with, the original - * OpenBinder.org binder driver interface, which is: - * - * Copyright (c) 2005 Palmsource, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#ifndef _UAPI_LINUX_BINDER_H -#define _UAPI_LINUX_BINDER_H - -#include - -#define B_PACK_CHARS(c1, c2, c3, c4) \ - ((((c1)<<24)) | (((c2)<<16)) | (((c3)<<8)) | (c4)) -#define B_TYPE_LARGE 0x85 - -enum { - BINDER_TYPE_BINDER = B_PACK_CHARS('s', 'b', '*', B_TYPE_LARGE), - BINDER_TYPE_WEAK_BINDER = B_PACK_CHARS('w', 'b', '*', B_TYPE_LARGE), - BINDER_TYPE_HANDLE = B_PACK_CHARS('s', 'h', '*', B_TYPE_LARGE), - BINDER_TYPE_WEAK_HANDLE = B_PACK_CHARS('w', 'h', '*', B_TYPE_LARGE), - BINDER_TYPE_FD = B_PACK_CHARS('f', 'd', '*', B_TYPE_LARGE), -}; - -enum { - FLAT_BINDER_FLAG_PRIORITY_MASK = 0xff, - FLAT_BINDER_FLAG_ACCEPTS_FDS = 0x100, -}; - -#ifdef BINDER_IPC_32BIT -typedef __u32 binder_size_t; -typedef __u32 binder_uintptr_t; -#else -typedef __u64 binder_size_t; -typedef __u64 binder_uintptr_t; -#endif - -/* - * This is the flattened representation of a Binder object for transfer - * between processes. The 'offsets' supplied as part of a binder transaction - * contains offsets into the data where these structures occur. The Binder - * driver takes care of re-writing the structure type and data as it moves - * between processes. - */ -struct flat_binder_object { - /* 8 bytes for large_flat_header. */ - __u32 type; - __u32 flags; - - /* 8 bytes of data. */ - union { - binder_uintptr_t binder; /* local object */ - __u32 handle; /* remote object */ - }; - - /* extra data associated with local object */ - binder_uintptr_t cookie; -}; - -/* - * On 64-bit platforms where user code may run in 32-bits the driver must - * translate the buffer (and local binder) addresses appropriately. - */ - -struct binder_write_read { - binder_size_t write_size; /* bytes to write */ - binder_size_t write_consumed; /* bytes consumed by driver */ - binder_uintptr_t write_buffer; - binder_size_t read_size; /* bytes to read */ - binder_size_t read_consumed; /* bytes consumed by driver */ - binder_uintptr_t read_buffer; -}; - -/* Use with BINDER_VERSION, driver fills in fields. */ -struct binder_version { - /* driver protocol version -- increment with incompatible change */ - __s32 protocol_version; -}; - -/* This is the current protocol version. */ -#ifdef BINDER_IPC_32BIT -#define BINDER_CURRENT_PROTOCOL_VERSION 7 -#else -#define BINDER_CURRENT_PROTOCOL_VERSION 8 -#endif - -#define BINDER_WRITE_READ _IOWR('b', 1, struct binder_write_read) -#define BINDER_SET_IDLE_TIMEOUT _IOW('b', 3, __s64) -#define BINDER_SET_MAX_THREADS _IOW('b', 5, __u32) -#define BINDER_SET_IDLE_PRIORITY _IOW('b', 6, __s32) -#define BINDER_SET_CONTEXT_MGR _IOW('b', 7, __s32) -#define BINDER_THREAD_EXIT _IOW('b', 8, __s32) -#define BINDER_VERSION _IOWR('b', 9, struct binder_version) - -/* - * NOTE: Two special error codes you should check for when calling - * in to the driver are: - * - * EINTR -- The operation has been interupted. This should be - * handled by retrying the ioctl() until a different error code - * is returned. - * - * ECONNREFUSED -- The driver is no longer accepting operations - * from your process. That is, the process is being destroyed. - * You should handle this by exiting from your process. Note - * that once this error code is returned, all further calls to - * the driver from any thread will return this same code. - */ - -enum transaction_flags { - TF_ONE_WAY = 0x01, /* this is a one-way call: async, no return */ - TF_ROOT_OBJECT = 0x04, /* contents are the component's root object */ - TF_STATUS_CODE = 0x08, /* contents are a 32-bit status code */ - TF_ACCEPT_FDS = 0x10, /* allow replies with file descriptors */ -}; - -struct binder_transaction_data { - /* The first two are only used for bcTRANSACTION and brTRANSACTION, - * identifying the target and contents of the transaction. - */ - union { - /* target descriptor of command transaction */ - __u32 handle; - /* target descriptor of return transaction */ - binder_uintptr_t ptr; - } target; - binder_uintptr_t cookie; /* target object cookie */ - __u32 code; /* transaction command */ - - /* General information about the transaction. */ - __u32 flags; - pid_t sender_pid; - uid_t sender_euid; - binder_size_t data_size; /* number of bytes of data */ - binder_size_t offsets_size; /* number of bytes of offsets */ - - /* If this transaction is inline, the data immediately - * follows here; otherwise, it ends with a pointer to - * the data buffer. - */ - union { - struct { - /* transaction data */ - binder_uintptr_t buffer; - /* offsets from buffer to flat_binder_object structs */ - binder_uintptr_t offsets; - } ptr; - __u8 buf[8]; - } data; -}; - -struct binder_ptr_cookie { - binder_uintptr_t ptr; - binder_uintptr_t cookie; -}; - -struct binder_handle_cookie { - __u32 handle; - binder_uintptr_t cookie; -} __packed; - -struct binder_pri_desc { - __s32 priority; - __u32 desc; -}; - -struct binder_pri_ptr_cookie { - __s32 priority; - binder_uintptr_t ptr; - binder_uintptr_t cookie; -}; - -enum binder_driver_return_protocol { - BR_ERROR = _IOR('r', 0, __s32), - /* - * int: error code - */ - - BR_OK = _IO('r', 1), - /* No parameters! */ - - BR_TRANSACTION = _IOR('r', 2, struct binder_transaction_data), - BR_REPLY = _IOR('r', 3, struct binder_transaction_data), - /* - * binder_transaction_data: the received command. - */ - - BR_ACQUIRE_RESULT = _IOR('r', 4, __s32), - /* - * not currently supported - * int: 0 if the last bcATTEMPT_ACQUIRE was not successful. - * Else the remote object has acquired a primary reference. - */ - - BR_DEAD_REPLY = _IO('r', 5), - /* - * The target of the last transaction (either a bcTRANSACTION or - * a bcATTEMPT_ACQUIRE) is no longer with us. No parameters. - */ - - BR_TRANSACTION_COMPLETE = _IO('r', 6), - /* - * No parameters... always refers to the last transaction requested - * (including replies). Note that this will be sent even for - * asynchronous transactions. - */ - - BR_INCREFS = _IOR('r', 7, struct binder_ptr_cookie), - BR_ACQUIRE = _IOR('r', 8, struct binder_ptr_cookie), - BR_RELEASE = _IOR('r', 9, struct binder_ptr_cookie), - BR_DECREFS = _IOR('r', 10, struct binder_ptr_cookie), - /* - * void *: ptr to binder - * void *: cookie for binder - */ - - BR_ATTEMPT_ACQUIRE = _IOR('r', 11, struct binder_pri_ptr_cookie), - /* - * not currently supported - * int: priority - * void *: ptr to binder - * void *: cookie for binder - */ - - BR_NOOP = _IO('r', 12), - /* - * No parameters. Do nothing and examine the next command. It exists - * primarily so that we can replace it with a BR_SPAWN_LOOPER command. - */ - - BR_SPAWN_LOOPER = _IO('r', 13), - /* - * No parameters. The driver has determined that a process has no - * threads waiting to service incoming transactions. When a process - * receives this command, it must spawn a new service thread and - * register it via bcENTER_LOOPER. - */ - - BR_FINISHED = _IO('r', 14), - /* - * not currently supported - * stop threadpool thread - */ - - BR_DEAD_BINDER = _IOR('r', 15, binder_uintptr_t), - /* - * void *: cookie - */ - BR_CLEAR_DEATH_NOTIFICATION_DONE = _IOR('r', 16, binder_uintptr_t), - /* - * void *: cookie - */ - - BR_FAILED_REPLY = _IO('r', 17), - /* - * The the last transaction (either a bcTRANSACTION or - * a bcATTEMPT_ACQUIRE) failed (e.g. out of memory). No parameters. - */ -}; - -enum binder_driver_command_protocol { - BC_TRANSACTION = _IOW('c', 0, struct binder_transaction_data), - BC_REPLY = _IOW('c', 1, struct binder_transaction_data), - /* - * binder_transaction_data: the sent command. - */ - - BC_ACQUIRE_RESULT = _IOW('c', 2, __s32), - /* - * not currently supported - * int: 0 if the last BR_ATTEMPT_ACQUIRE was not successful. - * Else you have acquired a primary reference on the object. - */ - - BC_FREE_BUFFER = _IOW('c', 3, binder_uintptr_t), - /* - * void *: ptr to transaction data received on a read - */ - - BC_INCREFS = _IOW('c', 4, __u32), - BC_ACQUIRE = _IOW('c', 5, __u32), - BC_RELEASE = _IOW('c', 6, __u32), - BC_DECREFS = _IOW('c', 7, __u32), - /* - * int: descriptor - */ - - BC_INCREFS_DONE = _IOW('c', 8, struct binder_ptr_cookie), - BC_ACQUIRE_DONE = _IOW('c', 9, struct binder_ptr_cookie), - /* - * void *: ptr to binder - * void *: cookie for binder - */ - - BC_ATTEMPT_ACQUIRE = _IOW('c', 10, struct binder_pri_desc), - /* - * not currently supported - * int: priority - * int: descriptor - */ - - BC_REGISTER_LOOPER = _IO('c', 11), - /* - * No parameters. - * Register a spawned looper thread with the device. - */ - - BC_ENTER_LOOPER = _IO('c', 12), - BC_EXIT_LOOPER = _IO('c', 13), - /* - * No parameters. - * These two commands are sent as an application-level thread - * enters and exits the binder loop, respectively. They are - * used so the binder can have an accurate count of the number - * of looping threads it has available. - */ - - BC_REQUEST_DEATH_NOTIFICATION = _IOW('c', 14, struct binder_handle_cookie), - /* - * int: handle - * void *: cookie - */ - - BC_CLEAR_DEATH_NOTIFICATION = _IOW('c', 15, struct binder_handle_cookie), - /* - * int: handle - * void *: cookie - */ - - BC_DEAD_BINDER_DONE = _IOW('c', 16, binder_uintptr_t), - /* - * void *: cookie - */ -}; - -#endif /* _UAPI_LINUX_BINDER_H */ - diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild index bdc6e87ff3eb..639cd24a1753 100644 --- a/include/uapi/linux/Kbuild +++ b/include/uapi/linux/Kbuild @@ -1,4 +1,5 @@ # UAPI Header export list +header-y += android/ header-y += byteorder/ header-y += can/ header-y += caif/ diff --git a/include/uapi/linux/android/Kbuild b/include/uapi/linux/android/Kbuild new file mode 100644 index 000000000000..ca011eec252a --- /dev/null +++ b/include/uapi/linux/android/Kbuild @@ -0,0 +1,2 @@ +# UAPI Header export list +header-y += binder.h diff --git a/include/uapi/linux/android/binder.h b/include/uapi/linux/android/binder.h new file mode 100644 index 000000000000..c653800b3a1a --- /dev/null +++ b/include/uapi/linux/android/binder.h @@ -0,0 +1,349 @@ +/* + * Copyright (C) 2008 Google, Inc. + * + * Based on, but no longer compatible with, the original + * OpenBinder.org binder driver interface, which is: + * + * Copyright (c) 2005 Palmsource, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _UAPI_LINUX_BINDER_H +#define _UAPI_LINUX_BINDER_H + +#include + +#define B_PACK_CHARS(c1, c2, c3, c4) \ + ((((c1)<<24)) | (((c2)<<16)) | (((c3)<<8)) | (c4)) +#define B_TYPE_LARGE 0x85 + +enum { + BINDER_TYPE_BINDER = B_PACK_CHARS('s', 'b', '*', B_TYPE_LARGE), + BINDER_TYPE_WEAK_BINDER = B_PACK_CHARS('w', 'b', '*', B_TYPE_LARGE), + BINDER_TYPE_HANDLE = B_PACK_CHARS('s', 'h', '*', B_TYPE_LARGE), + BINDER_TYPE_WEAK_HANDLE = B_PACK_CHARS('w', 'h', '*', B_TYPE_LARGE), + BINDER_TYPE_FD = B_PACK_CHARS('f', 'd', '*', B_TYPE_LARGE), +}; + +enum { + FLAT_BINDER_FLAG_PRIORITY_MASK = 0xff, + FLAT_BINDER_FLAG_ACCEPTS_FDS = 0x100, +}; + +#ifdef BINDER_IPC_32BIT +typedef __u32 binder_size_t; +typedef __u32 binder_uintptr_t; +#else +typedef __u64 binder_size_t; +typedef __u64 binder_uintptr_t; +#endif + +/* + * This is the flattened representation of a Binder object for transfer + * between processes. The 'offsets' supplied as part of a binder transaction + * contains offsets into the data where these structures occur. The Binder + * driver takes care of re-writing the structure type and data as it moves + * between processes. + */ +struct flat_binder_object { + /* 8 bytes for large_flat_header. */ + __u32 type; + __u32 flags; + + /* 8 bytes of data. */ + union { + binder_uintptr_t binder; /* local object */ + __u32 handle; /* remote object */ + }; + + /* extra data associated with local object */ + binder_uintptr_t cookie; +}; + +/* + * On 64-bit platforms where user code may run in 32-bits the driver must + * translate the buffer (and local binder) addresses appropriately. + */ + +struct binder_write_read { + binder_size_t write_size; /* bytes to write */ + binder_size_t write_consumed; /* bytes consumed by driver */ + binder_uintptr_t write_buffer; + binder_size_t read_size; /* bytes to read */ + binder_size_t read_consumed; /* bytes consumed by driver */ + binder_uintptr_t read_buffer; +}; + +/* Use with BINDER_VERSION, driver fills in fields. */ +struct binder_version { + /* driver protocol version -- increment with incompatible change */ + __s32 protocol_version; +}; + +/* This is the current protocol version. */ +#ifdef BINDER_IPC_32BIT +#define BINDER_CURRENT_PROTOCOL_VERSION 7 +#else +#define BINDER_CURRENT_PROTOCOL_VERSION 8 +#endif + +#define BINDER_WRITE_READ _IOWR('b', 1, struct binder_write_read) +#define BINDER_SET_IDLE_TIMEOUT _IOW('b', 3, __s64) +#define BINDER_SET_MAX_THREADS _IOW('b', 5, __u32) +#define BINDER_SET_IDLE_PRIORITY _IOW('b', 6, __s32) +#define BINDER_SET_CONTEXT_MGR _IOW('b', 7, __s32) +#define BINDER_THREAD_EXIT _IOW('b', 8, __s32) +#define BINDER_VERSION _IOWR('b', 9, struct binder_version) + +/* + * NOTE: Two special error codes you should check for when calling + * in to the driver are: + * + * EINTR -- The operation has been interupted. This should be + * handled by retrying the ioctl() until a different error code + * is returned. + * + * ECONNREFUSED -- The driver is no longer accepting operations + * from your process. That is, the process is being destroyed. + * You should handle this by exiting from your process. Note + * that once this error code is returned, all further calls to + * the driver from any thread will return this same code. + */ + +enum transaction_flags { + TF_ONE_WAY = 0x01, /* this is a one-way call: async, no return */ + TF_ROOT_OBJECT = 0x04, /* contents are the component's root object */ + TF_STATUS_CODE = 0x08, /* contents are a 32-bit status code */ + TF_ACCEPT_FDS = 0x10, /* allow replies with file descriptors */ +}; + +struct binder_transaction_data { + /* The first two are only used for bcTRANSACTION and brTRANSACTION, + * identifying the target and contents of the transaction. + */ + union { + /* target descriptor of command transaction */ + __u32 handle; + /* target descriptor of return transaction */ + binder_uintptr_t ptr; + } target; + binder_uintptr_t cookie; /* target object cookie */ + __u32 code; /* transaction command */ + + /* General information about the transaction. */ + __u32 flags; + pid_t sender_pid; + uid_t sender_euid; + binder_size_t data_size; /* number of bytes of data */ + binder_size_t offsets_size; /* number of bytes of offsets */ + + /* If this transaction is inline, the data immediately + * follows here; otherwise, it ends with a pointer to + * the data buffer. + */ + union { + struct { + /* transaction data */ + binder_uintptr_t buffer; + /* offsets from buffer to flat_binder_object structs */ + binder_uintptr_t offsets; + } ptr; + __u8 buf[8]; + } data; +}; + +struct binder_ptr_cookie { + binder_uintptr_t ptr; + binder_uintptr_t cookie; +}; + +struct binder_handle_cookie { + __u32 handle; + binder_uintptr_t cookie; +} __packed; + +struct binder_pri_desc { + __s32 priority; + __u32 desc; +}; + +struct binder_pri_ptr_cookie { + __s32 priority; + binder_uintptr_t ptr; + binder_uintptr_t cookie; +}; + +enum binder_driver_return_protocol { + BR_ERROR = _IOR('r', 0, __s32), + /* + * int: error code + */ + + BR_OK = _IO('r', 1), + /* No parameters! */ + + BR_TRANSACTION = _IOR('r', 2, struct binder_transaction_data), + BR_REPLY = _IOR('r', 3, struct binder_transaction_data), + /* + * binder_transaction_data: the received command. + */ + + BR_ACQUIRE_RESULT = _IOR('r', 4, __s32), + /* + * not currently supported + * int: 0 if the last bcATTEMPT_ACQUIRE was not successful. + * Else the remote object has acquired a primary reference. + */ + + BR_DEAD_REPLY = _IO('r', 5), + /* + * The target of the last transaction (either a bcTRANSACTION or + * a bcATTEMPT_ACQUIRE) is no longer with us. No parameters. + */ + + BR_TRANSACTION_COMPLETE = _IO('r', 6), + /* + * No parameters... always refers to the last transaction requested + * (including replies). Note that this will be sent even for + * asynchronous transactions. + */ + + BR_INCREFS = _IOR('r', 7, struct binder_ptr_cookie), + BR_ACQUIRE = _IOR('r', 8, struct binder_ptr_cookie), + BR_RELEASE = _IOR('r', 9, struct binder_ptr_cookie), + BR_DECREFS = _IOR('r', 10, struct binder_ptr_cookie), + /* + * void *: ptr to binder + * void *: cookie for binder + */ + + BR_ATTEMPT_ACQUIRE = _IOR('r', 11, struct binder_pri_ptr_cookie), + /* + * not currently supported + * int: priority + * void *: ptr to binder + * void *: cookie for binder + */ + + BR_NOOP = _IO('r', 12), + /* + * No parameters. Do nothing and examine the next command. It exists + * primarily so that we can replace it with a BR_SPAWN_LOOPER command. + */ + + BR_SPAWN_LOOPER = _IO('r', 13), + /* + * No parameters. The driver has determined that a process has no + * threads waiting to service incoming transactions. When a process + * receives this command, it must spawn a new service thread and + * register it via bcENTER_LOOPER. + */ + + BR_FINISHED = _IO('r', 14), + /* + * not currently supported + * stop threadpool thread + */ + + BR_DEAD_BINDER = _IOR('r', 15, binder_uintptr_t), + /* + * void *: cookie + */ + BR_CLEAR_DEATH_NOTIFICATION_DONE = _IOR('r', 16, binder_uintptr_t), + /* + * void *: cookie + */ + + BR_FAILED_REPLY = _IO('r', 17), + /* + * The the last transaction (either a bcTRANSACTION or + * a bcATTEMPT_ACQUIRE) failed (e.g. out of memory). No parameters. + */ +}; + +enum binder_driver_command_protocol { + BC_TRANSACTION = _IOW('c', 0, struct binder_transaction_data), + BC_REPLY = _IOW('c', 1, struct binder_transaction_data), + /* + * binder_transaction_data: the sent command. + */ + + BC_ACQUIRE_RESULT = _IOW('c', 2, __s32), + /* + * not currently supported + * int: 0 if the last BR_ATTEMPT_ACQUIRE was not successful. + * Else you have acquired a primary reference on the object. + */ + + BC_FREE_BUFFER = _IOW('c', 3, binder_uintptr_t), + /* + * void *: ptr to transaction data received on a read + */ + + BC_INCREFS = _IOW('c', 4, __u32), + BC_ACQUIRE = _IOW('c', 5, __u32), + BC_RELEASE = _IOW('c', 6, __u32), + BC_DECREFS = _IOW('c', 7, __u32), + /* + * int: descriptor + */ + + BC_INCREFS_DONE = _IOW('c', 8, struct binder_ptr_cookie), + BC_ACQUIRE_DONE = _IOW('c', 9, struct binder_ptr_cookie), + /* + * void *: ptr to binder + * void *: cookie for binder + */ + + BC_ATTEMPT_ACQUIRE = _IOW('c', 10, struct binder_pri_desc), + /* + * not currently supported + * int: priority + * int: descriptor + */ + + BC_REGISTER_LOOPER = _IO('c', 11), + /* + * No parameters. + * Register a spawned looper thread with the device. + */ + + BC_ENTER_LOOPER = _IO('c', 12), + BC_EXIT_LOOPER = _IO('c', 13), + /* + * No parameters. + * These two commands are sent as an application-level thread + * enters and exits the binder loop, respectively. They are + * used so the binder can have an accurate count of the number + * of looping threads it has available. + */ + + BC_REQUEST_DEATH_NOTIFICATION = _IOW('c', 14, struct binder_handle_cookie), + /* + * int: handle + * void *: cookie + */ + + BC_CLEAR_DEATH_NOTIFICATION = _IOW('c', 15, struct binder_handle_cookie), + /* + * int: handle + * void *: cookie + */ + + BC_DEAD_BINDER_DONE = _IOW('c', 16, binder_uintptr_t), + /* + * void *: cookie + */ +}; + +#endif /* _UAPI_LINUX_BINDER_H */ + -- cgit v1.2.3 From 0983897d00a541f725375d00604dccb3e716e128 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 16 Oct 2014 15:26:51 +0200 Subject: android: binder: remove binder.h binder.h isn't needed to just include a uapi file and set a single define, so move it into binder.c to save a few lines of code. Change-Id: Idcd0aba576295bbe0ddf5d18c4b1d1e8efdc8c84 Signed-off-by: Greg Kroah-Hartman --- drivers/android/binder.c | 6 +++++- drivers/android/binder.h | 30 ------------------------------ 2 files changed, 5 insertions(+), 31 deletions(-) delete mode 100644 drivers/android/binder.h diff --git a/drivers/android/binder.c b/drivers/android/binder.c index b44b9398810c..52aae90eb1fd 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -38,7 +38,11 @@ #include #include -#include "binder.h" +#ifdef CONFIG_ANDROID_BINDER_IPC_32BIT +#define BINDER_IPC_32BIT 1 +#endif + +#include #include "binder_trace.h" static DEFINE_MUTEX(binder_main_lock); diff --git a/drivers/android/binder.h b/drivers/android/binder.h deleted file mode 100644 index 5dc6a66b0665..000000000000 --- a/drivers/android/binder.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2008 Google, Inc. - * - * Based on, but no longer compatible with, the original - * OpenBinder.org binder driver interface, which is: - * - * Copyright (c) 2005 Palmsource, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#ifndef _LINUX_BINDER_H -#define _LINUX_BINDER_H - -#ifdef CONFIG_ANDROID_BINDER_IPC_32BIT -#define BINDER_IPC_32BIT 1 -#endif - -#include - -#endif /* _LINUX_BINDER_H */ - -- cgit v1.2.3 From 3ce49f5ed6c76a7772aeba7419d4976f225e1b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arve=20Hj=C3=B8nnev=C3=A5g?= Date: Thu, 13 Feb 2014 20:25:06 -0800 Subject: android: binder: More offset validation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure offsets don't point to overlapping flat_binder_object structs. Change-Id: I85c759b9c6395492474b177625dc6b0b289fd6b0 Signed-off-by: Arve HjønnevÃ¥g --- drivers/android/binder.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 52aae90eb1fd..b9242c6905ba 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -1322,6 +1322,7 @@ static void binder_transaction(struct binder_proc *proc, struct binder_transaction *t; struct binder_work *tcomplete; binder_size_t *offp, *off_end; + binder_size_t off_min; struct binder_proc *target_proc; struct binder_thread *target_thread = NULL; struct binder_node *target_node = NULL; @@ -1518,18 +1519,24 @@ static void binder_transaction(struct binder_proc *proc, goto err_bad_offset; } off_end = (void *)offp + tr->offsets_size; + off_min = 0; for (; offp < off_end; offp++) { struct flat_binder_object *fp; if (*offp > t->buffer->data_size - sizeof(*fp) || + *offp < off_min || t->buffer->data_size < sizeof(*fp) || !IS_ALIGNED(*offp, sizeof(u32))) { - binder_user_error("%d:%d got transaction with invalid offset, %lld\n", - proc->pid, thread->pid, (u64)*offp); + binder_user_error("%d:%d got transaction with invalid offset, %lld (min %lld, max %lld)\n", + proc->pid, thread->pid, (u64)*offp, + (u64)off_min, + (u64)(t->buffer->data_size - + sizeof(*fp))); return_error = BR_FAILED_REPLY; goto err_bad_offset; } fp = (struct flat_binder_object *)(t->buffer->data + *offp); + off_min = *offp + sizeof(struct flat_binder_object); switch (fp->type) { case BINDER_TYPE_BINDER: case BINDER_TYPE_WEAK_BINDER: { -- cgit v1.2.3 From ebce7cd30b3e00232c8be23d0f04d7c0599089b5 Mon Sep 17 00:00:00 2001 From: Riley Andrews Date: Fri, 26 Sep 2014 17:36:36 -0700 Subject: android: binder: Change binder mutex to rtmutex. Surfaceflinger uses binder heavily to receive/send frames from applications while compositing the screen. Change the binder mutex to an rt mutex to minimize instances where high priority surfaceflinger binder work is blocked by lower priority binder ipc. Change-Id: If7429040641d6e463f20301ec14f02ecf6b0da36 Signed-off-by: Riley Andrews --- drivers/android/binder.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index b9242c6905ba..3d0750780217 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -45,7 +46,7 @@ #include #include "binder_trace.h" -static DEFINE_MUTEX(binder_main_lock); +static DEFINE_RT_MUTEX(binder_main_lock); static DEFINE_MUTEX(binder_deferred_lock); static DEFINE_MUTEX(binder_mmap_lock); @@ -425,14 +426,14 @@ static long task_close_fd(struct binder_proc *proc, unsigned int fd) static inline void binder_lock(const char *tag) { trace_binder_lock(tag); - mutex_lock(&binder_main_lock); + rt_mutex_lock(&binder_main_lock); trace_binder_locked(tag); } static inline void binder_unlock(const char *tag) { trace_binder_unlock(tag); - mutex_unlock(&binder_main_lock); + rt_mutex_unlock(&binder_main_lock); } static void binder_set_nice(long nice) -- cgit v1.2.3 From c76b9f83bfc0551c065229c13c785675d6da9825 Mon Sep 17 00:00:00 2001 From: Stephen Smalley Date: Mon, 5 Nov 2012 08:15:34 -0500 Subject: Add security hooks to binder and implement the hooks for SELinux. Add security hooks to the binder and implement the hooks for SELinux. The security hooks enable security modules such as SELinux to implement controls over binder IPC. The security hooks include support for controlling what process can become the binder context manager (binder_set_context_mgr), controlling the ability of a process to invoke a binder transaction/IPC to another process (binder_transaction), controlling the ability a process to transfer a binder reference to another process (binder_transfer_binder), and controlling the ability of a process to transfer an open file to another process (binder_transfer_file). This support is used by SE Android, http://selinuxproject.org/page/SEAndroid. Change-Id: I34266b66320b6a3df9ac01833d7f94daf742920e Signed-off-by: Stephen Smalley --- drivers/android/binder.c | 21 ++++++++++++ include/linux/security.h | 29 ++++++++++++++++ security/capability.c | 24 ++++++++++++++ security/security.c | 20 +++++++++++ security/selinux/hooks.c | 66 +++++++++++++++++++++++++++++++++++++ security/selinux/include/classmap.h | 1 + 6 files changed, 161 insertions(+) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 3d0750780217..c048624fc6b7 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -38,6 +38,7 @@ #include #include #include +#include #ifdef CONFIG_ANDROID_BINDER_IPC_32BIT #define BINDER_IPC_32BIT 1 @@ -1404,6 +1405,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_DEAD_REPLY; goto err_dead_binder; } + if (security_binder_transaction(proc->tsk, target_proc->tsk) < 0) { + return_error = BR_FAILED_REPLY; + goto err_invalid_target_handle; + } if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) { struct binder_transaction *tmp; @@ -1561,6 +1566,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_for_node_failed; } + if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_for_node_failed; + } ref = binder_get_ref_for_node(target_proc, node); if (ref == NULL) { return_error = BR_FAILED_REPLY; @@ -1591,6 +1600,10 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_failed; } + if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { + return_error = BR_FAILED_REPLY; + goto err_binder_get_ref_failed; + } if (ref->node->proc == target_proc) { if (fp->type == BINDER_TYPE_HANDLE) fp->type = BINDER_TYPE_BINDER; @@ -1648,6 +1661,11 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_fget_failed; } + if (security_binder_transfer_file(proc->tsk, target_proc->tsk, file) < 0) { + fput(file); + return_error = BR_FAILED_REPLY; + goto err_get_unused_fd_failed; + } target_fd = task_get_unused_fd_flags(target_proc, O_CLOEXEC); if (target_fd < 0) { fput(file); @@ -2685,6 +2703,9 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp) ret = -EBUSY; goto out; } + ret = security_binder_set_context_mgr(proc->tsk); + if (ret < 0) + goto out; if (uid_valid(binder_context_mgr_uid)) { if (!uid_eq(binder_context_mgr_uid, curr_euid)) { pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n", diff --git a/include/linux/security.h b/include/linux/security.h index 4686491852a7..17e1888ff505 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1402,6 +1402,11 @@ static inline void security_free_mnt_opts(struct security_mnt_opts *opts) struct security_operations { char name[SECURITY_NAME_MAX + 1]; + int (*binder_set_context_mgr) (struct task_struct *mgr); + int (*binder_transaction) (struct task_struct *from, struct task_struct *to); + int (*binder_transfer_binder) (struct task_struct *from, struct task_struct *to); + int (*binder_transfer_file) (struct task_struct *from, struct task_struct *to, struct file *file); + int (*ptrace_access_check) (struct task_struct *child, unsigned int mode); int (*ptrace_traceme) (struct task_struct *parent); int (*capget) (struct task_struct *target, @@ -1690,6 +1695,10 @@ extern void __init security_fixup_ops(struct security_operations *ops); /* Security operations */ +int security_binder_set_context_mgr(struct task_struct *mgr); +int security_binder_transaction(struct task_struct *from, struct task_struct *to); +int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to); +int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file); int security_ptrace_access_check(struct task_struct *child, unsigned int mode); int security_ptrace_traceme(struct task_struct *parent); int security_capget(struct task_struct *target, @@ -1869,6 +1878,26 @@ static inline int security_init(void) return 0; } +static inline int security_binder_set_context_mgr(struct task_struct *mgr) +{ + return 0; +} + +static inline int security_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static inline int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static inline int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return 0; +} + static inline int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { diff --git a/security/capability.c b/security/capability.c index 1728d4e375db..6e4fc776badf 100644 --- a/security/capability.c +++ b/security/capability.c @@ -12,6 +12,26 @@ #include +static int cap_binder_set_context_mgr(struct task_struct *mgr) +{ + return 0; +} + +static int cap_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static int cap_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return 0; +} + +static int cap_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return 0; +} + static int cap_syslog(int type) { return 0; @@ -903,6 +923,10 @@ static void cap_audit_rule_free(void *lsmrule) void __init security_fixup_ops(struct security_operations *ops) { + set_to_cap_if_null(ops, binder_set_context_mgr); + set_to_cap_if_null(ops, binder_transaction); + set_to_cap_if_null(ops, binder_transfer_binder); + set_to_cap_if_null(ops, binder_transfer_file); set_to_cap_if_null(ops, ptrace_access_check); set_to_cap_if_null(ops, ptrace_traceme); set_to_cap_if_null(ops, capget); diff --git a/security/security.c b/security/security.c index a3dce87d1aef..d6ab2d2b363d 100644 --- a/security/security.c +++ b/security/security.c @@ -134,6 +134,26 @@ int __init register_security(struct security_operations *ops) /* Security operations */ +int security_binder_set_context_mgr(struct task_struct *mgr) +{ + return security_ops->binder_set_context_mgr(mgr); +} + +int security_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + return security_ops->binder_transaction(from, to); +} + +int security_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + return security_ops->binder_transfer_binder(from, to); +} + +int security_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + return security_ops->binder_transfer_file(from, to, file); +} + int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { #ifdef CONFIG_SECURITY_YAMA_STACKED diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 8908c367425b..a8de30bd733a 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1844,6 +1844,67 @@ static inline u32 open_file_to_av(struct file *file) /* Hook functions begin here. */ +static int selinux_binder_set_context_mgr(struct task_struct *mgr) +{ + u32 mysid = current_sid(); + u32 mgrsid = task_sid(mgr); + + return avc_has_perm(mysid, mgrsid, SECCLASS_BINDER, BINDER__SET_CONTEXT_MGR, NULL); +} + +static int selinux_binder_transaction(struct task_struct *from, struct task_struct *to) +{ + u32 mysid = current_sid(); + u32 fromsid = task_sid(from); + u32 tosid = task_sid(to); + int rc; + + if (mysid != fromsid) { + rc = avc_has_perm(mysid, fromsid, SECCLASS_BINDER, BINDER__IMPERSONATE, NULL); + if (rc) + return rc; + } + + return avc_has_perm(fromsid, tosid, SECCLASS_BINDER, BINDER__CALL, NULL); +} + +static int selinux_binder_transfer_binder(struct task_struct *from, struct task_struct *to) +{ + u32 fromsid = task_sid(from); + u32 tosid = task_sid(to); + return avc_has_perm(fromsid, tosid, SECCLASS_BINDER, BINDER__TRANSFER, NULL); +} + +static int selinux_binder_transfer_file(struct task_struct *from, struct task_struct *to, struct file *file) +{ + u32 sid = task_sid(to); + struct file_security_struct *fsec = file->f_security; + struct inode *inode = file->f_path.dentry->d_inode; + struct inode_security_struct *isec = inode->i_security; + struct common_audit_data ad; + struct selinux_audit_data sad = {0,}; + int rc; + + ad.type = LSM_AUDIT_DATA_PATH; + ad.u.path = file->f_path; + ad.selinux_audit_data = &sad; + + if (sid != fsec->sid) { + rc = avc_has_perm(sid, fsec->sid, + SECCLASS_FD, + FD__USE, + &ad); + if (rc) + return rc; + } + + if (unlikely(IS_PRIVATE(inode))) + return 0; + + return avc_has_perm(sid, isec->sid, isec->sclass, file_to_av(file), + &ad); +} + static int selinux_ptrace_access_check(struct task_struct *child, unsigned int mode) { @@ -5556,6 +5617,11 @@ static int selinux_key_getsecurity(struct key *key, char **_buffer) static struct security_operations selinux_ops = { .name = "selinux", + .binder_set_context_mgr = selinux_binder_set_context_mgr, + .binder_transaction = selinux_binder_transaction, + .binder_transfer_binder = selinux_binder_transfer_binder, + .binder_transfer_file = selinux_binder_transfer_file, + .ptrace_access_check = selinux_ptrace_access_check, .ptrace_traceme = selinux_ptrace_traceme, .capget = selinux_capget, diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h index 14d04e63b1f0..c32ff7bde81a 100644 --- a/security/selinux/include/classmap.h +++ b/security/selinux/include/classmap.h @@ -151,5 +151,6 @@ struct security_class_mapping secclass_map[] = { { "kernel_service", { "use_as_override", "create_files_as", NULL } }, { "tun_socket", { COMMON_SOCK_PERMS, "attach_queue", NULL } }, + { "binder", { "impersonate", "call", "set_context_mgr", "transfer", NULL } }, { NULL } }; -- cgit v1.2.3 From e1a60cbe0312adbf471ee057c5d7b917ae5e80c5 Mon Sep 17 00:00:00 2001 From: Micha Kalfon Date: Tue, 13 Jan 2015 11:47:20 +0200 Subject: prctl: fix misplaced PR_SET_TIMERSLACK_PID case The case clause for the PR_SET_TIMERSLACK_PID option was placed inside the an internal switch statement for PR_MCE_KILL (see commits 37a591d4 and 8ae872f1) . This commit moves it to the right place. Change-Id: I63251669d7e2f2aa843d1b0900e7df61518c3dea Signed-off-by: Micha Kalfon Acked-by: Oren Laadan --- kernel/sys.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/kernel/sys.c b/kernel/sys.c index 0b08c9f000f3..33192c2785b2 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2377,26 +2377,6 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, else return -EINVAL; break; - case PR_SET_TIMERSLACK_PID: - if (current->pid != (pid_t)arg3 && - !capable(CAP_SYS_NICE)) - return -EPERM; - rcu_read_lock(); - tsk = find_task_by_pid_ns((pid_t)arg3, &init_pid_ns); - if (tsk == NULL) { - rcu_read_unlock(); - return -EINVAL; - } - get_task_struct(tsk); - rcu_read_unlock(); - if (arg2 <= 0) - tsk->timer_slack_ns = - tsk->default_timer_slack_ns; - else - tsk->timer_slack_ns = arg2; - put_task_struct(tsk); - error = 0; - break; default: return -EINVAL; } @@ -2416,6 +2396,26 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, case PR_GET_TID_ADDRESS: error = prctl_get_tid_address(me, (int __user **)arg2); break; + case PR_SET_TIMERSLACK_PID: + if (current->pid != (pid_t)arg3 && + !capable(CAP_SYS_NICE)) + return -EPERM; + rcu_read_lock(); + tsk = find_task_by_pid_ns((pid_t)arg3, &init_pid_ns); + if (tsk == NULL) { + rcu_read_unlock(); + return -EINVAL; + } + get_task_struct(tsk); + rcu_read_unlock(); + if (arg2 <= 0) + tsk->timer_slack_ns = + tsk->default_timer_slack_ns; + else + tsk->timer_slack_ns = arg2; + put_task_struct(tsk); + error = 0; + break; case PR_SET_CHILD_SUBREAPER: me->signal->is_child_subreaper = !!arg2; break; -- cgit v1.2.3 From 2e09fe094acd1fb26b9ce242df045ceef042a2a0 Mon Sep 17 00:00:00 2001 From: Micha Kalfon Date: Tue, 13 Jan 2015 12:12:57 +0200 Subject: prctl: make PR_SET_TIMERSLACK_PID pid namespace aware Make PR_SET_TIMERSLACK_PID consider pid namespace and resolve the target pid in the caller's namespace. Otherwise, calls from pid namespace other than init would fail or affect the wrong task. Change-Id: I1da15196abc4096536713ce03714e99d2e63820a Signed-off-by: Micha Kalfon Acked-by: Oren Laadan --- kernel/sys.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sys.c b/kernel/sys.c index 33192c2785b2..a3bef5bd452b 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2397,11 +2397,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = prctl_get_tid_address(me, (int __user **)arg2); break; case PR_SET_TIMERSLACK_PID: - if (current->pid != (pid_t)arg3 && + if (task_pid_vnr(current) != (pid_t)arg3 && !capable(CAP_SYS_NICE)) return -EPERM; rcu_read_lock(); - tsk = find_task_by_pid_ns((pid_t)arg3, &init_pid_ns); + tsk = find_task_by_vpid((pid_t)arg3); if (tsk == NULL) { rcu_read_unlock(); return -EINVAL; -- cgit v1.2.3 From 6a148888b00dbcb8e76c4ff33b062f6ca5f0cdd9 Mon Sep 17 00:00:00 2001 From: Liu ShuoX Date: Mon, 17 Mar 2014 11:24:49 +1100 Subject: pstore: clarify clearing of _read_cnt in ramoops_context *_read_cnt in ramoops_context need to be cleared during pstore ->open to support mutli times getting the records. The patch added missed ftrace_read_cnt clearing and removed duplicate clearing in ramoops_probe. Signed-off-by: Liu ShuoX Cc: "Zhang, Yanmin" Cc: Colin Cross Cc: Kees Cook Signed-off-by: Andrew Morton Signed-off-by: Tony Luck Change-Id: I98622b2587ed661884e74a7273cfc92ee99eaae3 --- fs/pstore/ram.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c index 058f17f0b466..1107bc1edd92 100644 --- a/fs/pstore/ram.c +++ b/fs/pstore/ram.c @@ -86,6 +86,7 @@ struct ramoops_context { struct persistent_ram_ecc_info ecc_info; unsigned int max_dump_cnt; unsigned int dump_write_cnt; + /* _read_cnt need clear on ramoops_pstore_open */ unsigned int dump_read_cnt; unsigned int console_read_cnt; unsigned int ftrace_read_cnt; @@ -101,6 +102,7 @@ static int ramoops_pstore_open(struct pstore_info *psi) cxt->dump_read_cnt = 0; cxt->console_read_cnt = 0; + cxt->ftrace_read_cnt = 0; return 0; } @@ -414,7 +416,6 @@ static int ramoops_probe(struct platform_device *pdev) if (!is_power_of_2(pdata->ftrace_size)) pdata->ftrace_size = rounddown_pow_of_two(pdata->ftrace_size); - cxt->dump_read_cnt = 0; cxt->size = pdata->mem_size; cxt->phys_addr = pdata->mem_address; cxt->record_size = pdata->record_size; -- cgit v1.2.3 From 3797dc2dca25d4a89ff171990c4e6dba827b8e3d Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Mon, 8 Dec 2014 15:06:52 -0800 Subject: [PATCH v4 1/5] pstore: use snprintf No guarantees that the names will not exceed the name buffer with future adjustments. Signed-off-by: Mark Salyzyn Acked-by: Joe Perches v2: switch from snprintf to scnprintf v3: remove embedded space v4: renumber pmsg series dependencies Change-Id: I161fe8cadc967d74e18cc09b7d60b5b398c92c86 --- fs/pstore/inode.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fs/pstore/inode.c b/fs/pstore/inode.c index e4bcb2cf055a..ae977dd0e426 100644 --- a/fs/pstore/inode.c +++ b/fs/pstore/inode.c @@ -313,22 +313,24 @@ int pstore_mkfile(enum pstore_type_id type, char *psname, u64 id, int count, switch (type) { case PSTORE_TYPE_DMESG: - sprintf(name, "dmesg-%s-%lld", psname, id); + scnprintf(name, sizeof(name), "dmesg-%s-%lld", + psname, id); break; case PSTORE_TYPE_CONSOLE: - sprintf(name, "console-%s", psname); + scnprintf(name, sizeof(name), "console-%s", psname); break; case PSTORE_TYPE_FTRACE: - sprintf(name, "ftrace-%s", psname); + scnprintf(name, sizeof(name), "ftrace-%s", psname); break; case PSTORE_TYPE_MCE: - sprintf(name, "mce-%s-%lld", psname, id); + scnprintf(name, sizeof(name), "mce-%s-%lld", psname, id); break; case PSTORE_TYPE_UNKNOWN: - sprintf(name, "unknown-%s-%lld", psname, id); + scnprintf(name, sizeof(name), "unknown-%s-%lld", psname, id); break; default: - sprintf(name, "type%d-%s-%lld", type, psname, id); + scnprintf(name, sizeof(name), "type%d-%s-%lld", + type, psname, id); break; } -- cgit v1.2.3 From b8cbc429050e3f9dd93bfee8461627d37d048e18 Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Mon, 8 Dec 2014 15:06:52 -0800 Subject: [PATCH v2 2/5] pstore: remove superfluous memory size check All previous checks will fail with error if memory size is not sufficient to register a zone, so this legacy check has become redundant. Signed-off-by: Mark Salyzyn Acked-by: Kees Cook v2: renumber pmsg series dependencies Change-Id: Ie21c988ae0b1ebb0dafa6c0c0b069e9cfe1e8506 --- fs/pstore/ram.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c index 1107bc1edd92..a05dc23d954c 100644 --- a/fs/pstore/ram.c +++ b/fs/pstore/ram.c @@ -441,14 +441,6 @@ static int ramoops_probe(struct platform_device *pdev) if (err) goto fail_init_fprz; - if (!cxt->przs && !cxt->cprz && !cxt->fprz) { - pr_err("memory size too small, minimum is %zu\n", - cxt->console_size + cxt->record_size + - cxt->ftrace_size); - err = -EINVAL; - goto fail_cnt; - } - cxt->pstore.data = cxt; /* * Console can handle any buffer size, so prefer LOG_LINE_MAX. If we @@ -493,7 +485,6 @@ fail_buf: fail_clear: cxt->pstore.bufsize = 0; cxt->max_dump_cnt = 0; -fail_cnt: kfree(cxt->fprz); fail_init_fprz: kfree(cxt->cprz); -- cgit v1.2.3 From 2d416e95a31472d2654e964369f99d2fc352852f Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Mon, 8 Dec 2014 15:06:52 -0800 Subject: [PATCH 3/5] pstore: handle zero-sized prz in series Corrects a problem wih ramoops_pstore_read failing to return the next in a prz series after first zero-sized entry, not venturing to the next non-zero entry. Signed-off-by: Mark Salyzyn Change-Id: Iedce3b94c13917da33be44e1d80811757774c793 --- fs/pstore/ram.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c index a05dc23d954c..585422d70581 100644 --- a/fs/pstore/ram.c +++ b/fs/pstore/ram.c @@ -133,6 +133,12 @@ ramoops_get_next_prz(struct persistent_ram_zone *przs[], uint *c, uint max, return prz; } +static bool prz_ok(struct persistent_ram_zone *prz) +{ + return !!prz && !!(persistent_ram_old_size(prz) + + persistent_ram_ecc_string(prz, NULL, 0)); +} + static ssize_t ramoops_pstore_read(u64 *id, enum pstore_type_id *type, int *count, struct timespec *time, char **buf, struct pstore_info *psi) @@ -145,13 +151,13 @@ static ssize_t ramoops_pstore_read(u64 *id, enum pstore_type_id *type, prz = ramoops_get_next_prz(cxt->przs, &cxt->dump_read_cnt, cxt->max_dump_cnt, id, type, PSTORE_TYPE_DMESG, 1); - if (!prz) + if (!prz_ok(prz)) prz = ramoops_get_next_prz(&cxt->cprz, &cxt->console_read_cnt, 1, id, type, PSTORE_TYPE_CONSOLE, 0); - if (!prz) + if (!prz_ok(prz)) prz = ramoops_get_next_prz(&cxt->fprz, &cxt->ftrace_read_cnt, 1, id, type, PSTORE_TYPE_FTRACE, 0); - if (!prz) + if (!prz_ok(prz)) return 0; /* TODO(kees): Bogus time for the moment. */ -- cgit v1.2.3 From 53d8e1c7141617e8656420afb44f9536e7cf0519 Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Mon, 8 Dec 2014 15:06:52 -0800 Subject: [PATCH v4 4/5] pstore: add pmsg A secured user-space accessible pstore object. Writes to /dev/pmsg0 are appended to the buffer, on reboot the persistent contents are available in /sys/fs/pstore/pmsg-ramoops-[ID]. One possible use is syslogd, or other daemon, can write messages, then on reboot provides a means to triage user-space activities leading up to a panic as a companion to the pstore dmesg or console logs. Signed-off-by: Mark Salyzyn v2: switch from snprintf to scnprintf v3: split out prz_ok checking into PATCH 3/5 replace pmsg_lseek with noop_llseek use pr_fmt() macro make write atomic and use a vmalloc'd bounce buffer v4: use mutex_lock instead of spin_lock. Change-Id: I82a2a9a989d7583c5fcb65ff520027dc3a034a4c --- fs/pstore/Kconfig | 10 ++++ fs/pstore/Makefile | 2 + fs/pstore/inode.c | 3 ++ fs/pstore/internal.h | 6 +++ fs/pstore/platform.c | 1 + fs/pstore/pmsg.c | 114 +++++++++++++++++++++++++++++++++++++++++++++ fs/pstore/ram.c | 34 +++++++++++++- include/linux/pstore.h | 1 + include/linux/pstore_ram.h | 1 + 9 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 fs/pstore/pmsg.c diff --git a/fs/pstore/Kconfig b/fs/pstore/Kconfig index ca71db69da07..da12fd4e3d47 100644 --- a/fs/pstore/Kconfig +++ b/fs/pstore/Kconfig @@ -19,6 +19,16 @@ config PSTORE_CONSOLE When the option is enabled, pstore will log all kernel messages, even if no oops or panic happened. +config PSTORE_PMSG + bool "Log user space messages" + depends on PSTORE + help + When the option is enabled, pstore will export a character + interface /dev/pmsg0 to log user space messages. On reboot + data can be retrieved from /sys/fs/pstore/pmsg-ramoops-[ID]. + + If unsure, say N. + config PSTORE_FTRACE bool "Persistent function tracer" depends on PSTORE diff --git a/fs/pstore/Makefile b/fs/pstore/Makefile index 4c9095c2781e..e647d8e81712 100644 --- a/fs/pstore/Makefile +++ b/fs/pstore/Makefile @@ -7,5 +7,7 @@ obj-y += pstore.o pstore-objs += inode.o platform.o obj-$(CONFIG_PSTORE_FTRACE) += ftrace.o +obj-$(CONFIG_PSTORE_PMSG) += pmsg.o + ramoops-objs += ram.o ram_core.o obj-$(CONFIG_PSTORE_RAM) += ramoops.o diff --git a/fs/pstore/inode.c b/fs/pstore/inode.c index ae977dd0e426..57bbfeaab663 100644 --- a/fs/pstore/inode.c +++ b/fs/pstore/inode.c @@ -325,6 +325,9 @@ int pstore_mkfile(enum pstore_type_id type, char *psname, u64 id, int count, case PSTORE_TYPE_MCE: scnprintf(name, sizeof(name), "mce-%s-%lld", psname, id); break; + case PSTORE_TYPE_PMSG: + scnprintf(name, sizeof(name), "pmsg-%s-%lld", psname, id); + break; case PSTORE_TYPE_UNKNOWN: scnprintf(name, sizeof(name), "unknown-%s-%lld", psname, id); break; diff --git a/fs/pstore/internal.h b/fs/pstore/internal.h index 937d820f273c..fd8d248c285c 100644 --- a/fs/pstore/internal.h +++ b/fs/pstore/internal.h @@ -45,6 +45,12 @@ extern void pstore_register_ftrace(void); static inline void pstore_register_ftrace(void) {} #endif +#ifdef CONFIG_PSTORE_PMSG +extern void pstore_register_pmsg(void); +#else +static inline void pstore_register_pmsg(void) {} +#endif + extern struct pstore_info *psinfo; extern void pstore_set_kmsg_bytes(int); diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c index 86d1038b5a12..4f11f2382e24 100644 --- a/fs/pstore/platform.c +++ b/fs/pstore/platform.c @@ -267,6 +267,7 @@ int pstore_register(struct pstore_info *psi) kmsg_dump_register(&pstore_dumper); pstore_register_console(); pstore_register_ftrace(); + pstore_register_pmsg(); if (pstore_update_ms >= 0) { pstore_timer.expires = jiffies + diff --git a/fs/pstore/pmsg.c b/fs/pstore/pmsg.c new file mode 100644 index 000000000000..db47810a92d8 --- /dev/null +++ b/fs/pstore/pmsg.c @@ -0,0 +1,114 @@ +/* + * Copyright 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include "internal.h" + +static DEFINE_MUTEX(pmsg_lock); +#define PMSG_MAX_BOUNCE_BUFFER_SIZE (2*PAGE_SIZE) + +static ssize_t write_pmsg(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + size_t i, buffer_size; + char *buffer; + + if (!count) + return 0; + + if (!access_ok(VERIFY_READ, buf, count)) + return -EFAULT; + + buffer_size = count; + if (buffer_size > PMSG_MAX_BOUNCE_BUFFER_SIZE) + buffer_size = PMSG_MAX_BOUNCE_BUFFER_SIZE; + buffer = vmalloc(buffer_size); + + mutex_lock(&pmsg_lock); + for (i = 0; i < count; ) { + size_t c = min(count - i, buffer_size); + u64 id; + long ret; + + ret = __copy_from_user(buffer, buf + i, c); + if (unlikely(ret != 0)) { + mutex_unlock(&pmsg_lock); + vfree(buffer); + return -EFAULT; + } + psinfo->write_buf(PSTORE_TYPE_PMSG, 0, &id, 0, buffer, c, + psinfo); + + i += c; + } + + mutex_unlock(&pmsg_lock); + vfree(buffer); + return count; +} + +static const struct file_operations pmsg_fops = { + .owner = THIS_MODULE, + .llseek = noop_llseek, + .write = write_pmsg, +}; + +static struct class *pmsg_class; +static int pmsg_major; +#define PMSG_NAME "pmsg" +#undef pr_fmt +#define pr_fmt(fmt) PMSG_NAME ": " fmt + +static char *pmsg_devnode(struct device *dev, umode_t *mode) +{ + if (mode) + *mode = 0220; + return NULL; +} + +void pstore_register_pmsg(void) +{ + struct device *pmsg_device; + + pmsg_major = register_chrdev(0, PMSG_NAME, &pmsg_fops); + if (pmsg_major < 0) { + pr_err("register_chrdev failed\n"); + goto err; + } + + pmsg_class = class_create(THIS_MODULE, PMSG_NAME); + if (IS_ERR(pmsg_class)) { + pr_err("device class file already in use\n"); + goto err_class; + } + pmsg_class->devnode = pmsg_devnode; + + pmsg_device = device_create(pmsg_class, NULL, MKDEV(pmsg_major, 0), + NULL, "%s%d", PMSG_NAME, 0); + if (IS_ERR(pmsg_device)) { + pr_err("failed to create device\n"); + goto err_device; + } + return; + +err_device: + class_destroy(pmsg_class); +err_class: + unregister_chrdev(pmsg_major, PMSG_NAME); +err: + return; +} diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c index 585422d70581..3a28d468b458 100644 --- a/fs/pstore/ram.c +++ b/fs/pstore/ram.c @@ -51,6 +51,10 @@ static ulong ramoops_ftrace_size = MIN_MEM_SIZE; module_param_named(ftrace_size, ramoops_ftrace_size, ulong, 0400); MODULE_PARM_DESC(ftrace_size, "size of ftrace log"); +static ulong ramoops_pmsg_size = MIN_MEM_SIZE; +module_param_named(pmsg_size, ramoops_pmsg_size, ulong, 0400); +MODULE_PARM_DESC(pmsg_size, "size of user space message log"); + static ulong mem_address; module_param(mem_address, ulong, 0400); MODULE_PARM_DESC(mem_address, @@ -77,11 +81,13 @@ struct ramoops_context { struct persistent_ram_zone **przs; struct persistent_ram_zone *cprz; struct persistent_ram_zone *fprz; + struct persistent_ram_zone *mprz; phys_addr_t phys_addr; unsigned long size; size_t record_size; size_t console_size; size_t ftrace_size; + size_t pmsg_size; int dump_oops; struct persistent_ram_ecc_info ecc_info; unsigned int max_dump_cnt; @@ -90,6 +96,7 @@ struct ramoops_context { unsigned int dump_read_cnt; unsigned int console_read_cnt; unsigned int ftrace_read_cnt; + unsigned int pmsg_read_cnt; struct pstore_info pstore; }; @@ -103,6 +110,7 @@ static int ramoops_pstore_open(struct pstore_info *psi) cxt->dump_read_cnt = 0; cxt->console_read_cnt = 0; cxt->ftrace_read_cnt = 0; + cxt->pmsg_read_cnt = 0; return 0; } @@ -157,6 +165,9 @@ static ssize_t ramoops_pstore_read(u64 *id, enum pstore_type_id *type, if (!prz_ok(prz)) prz = ramoops_get_next_prz(&cxt->fprz, &cxt->ftrace_read_cnt, 1, id, type, PSTORE_TYPE_FTRACE, 0); + if (!prz_ok(prz)) + prz = ramoops_get_next_prz(&cxt->mprz, &cxt->pmsg_read_cnt, + 1, id, type, PSTORE_TYPE_PMSG, 0); if (!prz_ok(prz)) return 0; @@ -220,6 +231,11 @@ static int notrace ramoops_pstore_write_buf(enum pstore_type_id type, return -ENOMEM; persistent_ram_write(cxt->fprz, buf, size); return 0; + } else if (type == PSTORE_TYPE_PMSG) { + if (!cxt->mprz) + return -ENOMEM; + persistent_ram_write(cxt->mprz, buf, size); + return 0; } if (type != PSTORE_TYPE_DMESG) @@ -277,6 +293,9 @@ static int ramoops_pstore_erase(enum pstore_type_id type, u64 id, int count, case PSTORE_TYPE_FTRACE: prz = cxt->fprz; break; + case PSTORE_TYPE_PMSG: + prz = cxt->mprz; + break; default: return -EINVAL; } @@ -407,7 +426,7 @@ static int ramoops_probe(struct platform_device *pdev) goto fail_out; if (!pdata->mem_size || (!pdata->record_size && !pdata->console_size && - !pdata->ftrace_size)) { + !pdata->ftrace_size && !pdata->pmsg_size)) { pr_err("The memory size and the record/console size must be " "non-zero\n"); goto fail_out; @@ -421,18 +440,22 @@ static int ramoops_probe(struct platform_device *pdev) pdata->console_size = rounddown_pow_of_two(pdata->console_size); if (!is_power_of_2(pdata->ftrace_size)) pdata->ftrace_size = rounddown_pow_of_two(pdata->ftrace_size); + if (pdata->pmsg_size && !is_power_of_2(pdata->pmsg_size)) + pdata->pmsg_size = rounddown_pow_of_two(pdata->pmsg_size); cxt->size = pdata->mem_size; cxt->phys_addr = pdata->mem_address; cxt->record_size = pdata->record_size; cxt->console_size = pdata->console_size; cxt->ftrace_size = pdata->ftrace_size; + cxt->pmsg_size = pdata->pmsg_size; cxt->dump_oops = pdata->dump_oops; cxt->ecc_info = pdata->ecc_info; paddr = cxt->phys_addr; - dump_mem_sz = cxt->size - cxt->console_size - cxt->ftrace_size; + dump_mem_sz = cxt->size - cxt->console_size - cxt->ftrace_size + - cxt->pmsg_size; err = ramoops_init_przs(dev, cxt, &paddr, dump_mem_sz); if (err) goto fail_out; @@ -447,6 +470,10 @@ static int ramoops_probe(struct platform_device *pdev) if (err) goto fail_init_fprz; + err = ramoops_init_prz(dev, cxt, &cxt->mprz, &paddr, cxt->pmsg_size, 0); + if (err) + goto fail_init_mprz; + cxt->pstore.data = cxt; /* * Console can handle any buffer size, so prefer LOG_LINE_MAX. If we @@ -491,6 +518,8 @@ fail_buf: fail_clear: cxt->pstore.bufsize = 0; cxt->max_dump_cnt = 0; + kfree(cxt->mprz); +fail_init_mprz: kfree(cxt->fprz); fail_init_fprz: kfree(cxt->cprz); @@ -548,6 +577,7 @@ static void ramoops_register_dummy(void) dummy_data->record_size = record_size; dummy_data->console_size = ramoops_console_size; dummy_data->ftrace_size = ramoops_ftrace_size; + dummy_data->pmsg_size = ramoops_pmsg_size; dummy_data->dump_oops = dump_oops; /* * For backwards compatibility ramoops.ecc=1 means 16 bytes ECC diff --git a/include/linux/pstore.h b/include/linux/pstore.h index 75d01760c911..289884121d30 100644 --- a/include/linux/pstore.h +++ b/include/linux/pstore.h @@ -35,6 +35,7 @@ enum pstore_type_id { PSTORE_TYPE_MCE = 1, PSTORE_TYPE_CONSOLE = 2, PSTORE_TYPE_FTRACE = 3, + PSTORE_TYPE_PMSG = 4, /* Backport: 7 in upstream 3.19.0-rc3 */ PSTORE_TYPE_UNKNOWN = 255 }; diff --git a/include/linux/pstore_ram.h b/include/linux/pstore_ram.h index 9e370618352a..2c8e5dd7a762 100644 --- a/include/linux/pstore_ram.h +++ b/include/linux/pstore_ram.h @@ -81,6 +81,7 @@ struct ramoops_platform_data { unsigned long record_size; unsigned long console_size; unsigned long ftrace_size; + unsigned long pmsg_size; int dump_oops; struct persistent_ram_ecc_info ecc_info; }; -- cgit v1.2.3 From 3084b294e95259aa8cfded700deba4cc3da50c95 Mon Sep 17 00:00:00 2001 From: Richard Haines Date: Tue, 19 Nov 2013 17:34:23 -0500 Subject: SELinux: Update policy version to support constraints info Update the policy version (POLICYDB_VERSION_CONSTRAINT_NAMES) to allow holding of policy source info for constraints. Signed-off-by: Richard Haines Acked-by: Stephen Smalley Signed-off-by: Paul Moore --- security/selinux/include/security.h | 3 +- security/selinux/ss/constraint.h | 1 + security/selinux/ss/policydb.c | 96 +++++++++++++++++++++++++++++++++---- security/selinux/ss/policydb.h | 11 +++++ 4 files changed, 101 insertions(+), 10 deletions(-) diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h index 6d3885165d14..b214ef5f86e1 100644 --- a/security/selinux/include/security.h +++ b/security/selinux/include/security.h @@ -33,13 +33,14 @@ #define POLICYDB_VERSION_ROLETRANS 26 #define POLICYDB_VERSION_NEW_OBJECT_DEFAULTS 27 #define POLICYDB_VERSION_DEFAULT_TYPE 28 +#define POLICYDB_VERSION_CONSTRAINT_NAMES 29 /* Range of policy versions we understand*/ #define POLICYDB_VERSION_MIN POLICYDB_VERSION_BASE #ifdef CONFIG_SECURITY_SELINUX_POLICYDB_VERSION_MAX #define POLICYDB_VERSION_MAX CONFIG_SECURITY_SELINUX_POLICYDB_VERSION_MAX_VALUE #else -#define POLICYDB_VERSION_MAX POLICYDB_VERSION_DEFAULT_TYPE +#define POLICYDB_VERSION_MAX POLICYDB_VERSION_CONSTRAINT_NAMES #endif /* Mask for just the mount related flags */ diff --git a/security/selinux/ss/constraint.h b/security/selinux/ss/constraint.h index 149dda731fd3..96fd947c494b 100644 --- a/security/selinux/ss/constraint.h +++ b/security/selinux/ss/constraint.h @@ -48,6 +48,7 @@ struct constraint_expr { u32 op; /* operator */ struct ebitmap names; /* names */ + struct type_set *type_names; struct constraint_expr *next; /* next expression */ }; diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index 9cd9b7c661ec..60af34a1c80c 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -143,6 +143,11 @@ static struct policydb_compat_info policydb_compat[] = { .sym_num = SYM_NUM, .ocon_num = OCON_NUM, }, + { + .version = POLICYDB_VERSION_CONSTRAINT_NAMES, + .sym_num = SYM_NUM, + .ocon_num = OCON_NUM, + }, }; static struct policydb_compat_info *policydb_lookup_compat(int version) @@ -613,6 +618,19 @@ static int common_destroy(void *key, void *datum, void *p) return 0; } +static void constraint_expr_destroy(struct constraint_expr *expr) +{ + if (expr) { + ebitmap_destroy(&expr->names); + if (expr->type_names) { + ebitmap_destroy(&expr->type_names->types); + ebitmap_destroy(&expr->type_names->negset); + kfree(expr->type_names); + } + kfree(expr); + } +} + static int cls_destroy(void *key, void *datum, void *p) { struct class_datum *cladatum; @@ -628,10 +646,9 @@ static int cls_destroy(void *key, void *datum, void *p) while (constraint) { e = constraint->expr; while (e) { - ebitmap_destroy(&e->names); etmp = e; e = e->next; - kfree(etmp); + constraint_expr_destroy(etmp); } ctemp = constraint; constraint = constraint->next; @@ -642,16 +659,14 @@ static int cls_destroy(void *key, void *datum, void *p) while (constraint) { e = constraint->expr; while (e) { - ebitmap_destroy(&e->names); etmp = e; e = e->next; - kfree(etmp); + constraint_expr_destroy(etmp); } ctemp = constraint; constraint = constraint->next; kfree(ctemp); } - kfree(cladatum->comkey); } kfree(datum); @@ -1156,8 +1171,34 @@ bad: return rc; } -static int read_cons_helper(struct constraint_node **nodep, int ncons, - int allowxtarget, void *fp) +static void type_set_init(struct type_set *t) +{ + ebitmap_init(&t->types); + ebitmap_init(&t->negset); +} + +static int type_set_read(struct type_set *t, void *fp) +{ + __le32 buf[1]; + int rc; + + if (ebitmap_read(&t->types, fp)) + return -EINVAL; + if (ebitmap_read(&t->negset, fp)) + return -EINVAL; + + rc = next_entry(buf, fp, sizeof(u32)); + if (rc < 0) + return -EINVAL; + t->flags = le32_to_cpu(buf[0]); + + return 0; +} + + +static int read_cons_helper(struct policydb *p, + struct constraint_node **nodep, + int ncons, int allowxtarget, void *fp) { struct constraint_node *c, *lc; struct constraint_expr *e, *le; @@ -1225,6 +1266,18 @@ static int read_cons_helper(struct constraint_node **nodep, int ncons, rc = ebitmap_read(&e->names, fp); if (rc) return rc; + if (p->policyvers >= + POLICYDB_VERSION_CONSTRAINT_NAMES) { + e->type_names = kzalloc(sizeof + (*e->type_names), + GFP_KERNEL); + if (!e->type_names) + return -ENOMEM; + type_set_init(e->type_names); + rc = type_set_read(e->type_names, fp); + if (rc) + return rc; + } break; default: return -EINVAL; @@ -1301,7 +1354,7 @@ static int class_read(struct policydb *p, struct hashtab *h, void *fp) goto bad; } - rc = read_cons_helper(&cladatum->constraints, ncons, 0, fp); + rc = read_cons_helper(p, &cladatum->constraints, ncons, 0, fp); if (rc) goto bad; @@ -1311,7 +1364,8 @@ static int class_read(struct policydb *p, struct hashtab *h, void *fp) if (rc) goto bad; ncons = le32_to_cpu(buf[0]); - rc = read_cons_helper(&cladatum->validatetrans, ncons, 1, fp); + rc = read_cons_helper(p, &cladatum->validatetrans, + ncons, 1, fp); if (rc) goto bad; } @@ -2750,6 +2804,24 @@ static int common_write(void *vkey, void *datum, void *ptr) return 0; } +static int type_set_write(struct type_set *t, void *fp) +{ + int rc; + __le32 buf[1]; + + if (ebitmap_write(&t->types, fp)) + return -EINVAL; + if (ebitmap_write(&t->negset, fp)) + return -EINVAL; + + buf[0] = cpu_to_le32(t->flags); + rc = put_entry(buf, sizeof(u32), 1, fp); + if (rc) + return -EINVAL; + + return 0; +} + static int write_cons_helper(struct policydb *p, struct constraint_node *node, void *fp) { @@ -2781,6 +2853,12 @@ static int write_cons_helper(struct policydb *p, struct constraint_node *node, rc = ebitmap_write(&e->names, fp); if (rc) return rc; + if (p->policyvers >= + POLICYDB_VERSION_CONSTRAINT_NAMES) { + rc = type_set_write(e->type_names, fp); + if (rc) + return rc; + } break; default: break; diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h index da637471d4ce..725d5945a97e 100644 --- a/security/selinux/ss/policydb.h +++ b/security/selinux/ss/policydb.h @@ -153,6 +153,17 @@ struct cond_bool_datum { struct cond_node; +/* + * type set preserves data needed to determine constraint info from + * policy source. This is not used by the kernel policy but allows + * utilities such as audit2allow to determine constraint denials. + */ +struct type_set { + struct ebitmap types; + struct ebitmap negset; + u32 flags; +}; + /* * The configuration data includes security contexts for * initial SIDs, unlabeled file systems, TCP and UDP port numbers, -- cgit v1.2.3 From 2294d499b7969df3838becf5e58bf16b0e3c86c8 Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Wed, 7 Jan 2015 09:27:15 -0800 Subject: [PATCH 5/5] pstore: selinux: add security in-core xattr support for pstore and debugfs - add "pstore" and "debugfs" to list of in-core exceptions - change fstype checks to boolean equation - change from strncmp to strcmp for checking Signed-off-by: Mark Salyzyn Bug: 18917345 Bug: 18935184 Change-Id: Ib648f30ce4b5d6c96f11465836d6fee89bec1c72 --- security/selinux/hooks.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index a8de30bd733a..0fa3195626a9 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -418,15 +418,11 @@ static int sb_finish_set_opts(struct super_block *sb) sbsec->behavior > ARRAY_SIZE(labeling_behaviors)) sbsec->flags &= ~SE_SBLABELSUPP; - /* Special handling for sysfs. Is genfs but also has setxattr handler*/ - if (strncmp(sb->s_type->name, "sysfs", sizeof("sysfs")) == 0) - sbsec->flags |= SE_SBLABELSUPP; - - /* - * Special handling for rootfs. Is genfs but supports - * setting SELinux context on in-core inodes. - */ - if (strncmp(sb->s_type->name, "rootfs", sizeof("rootfs")) == 0) + /* Special handling. Is genfs but also has in-core setxattr handler*/ + if (!strcmp(sb->s_type->name, "sysfs") || + !strcmp(sb->s_type->name, "pstore") || + !strcmp(sb->s_type->name, "debugfs") || + !strcmp(sb->s_type->name, "rootfs")) sbsec->flags |= SE_SBLABELSUPP; /* Initialize the root inode. */ -- cgit v1.2.3 From 5b5b6febcab05ef9e8972ead4cc3cf8381d45a95 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Thu, 26 Jun 2014 14:33:56 -0400 Subject: selinux: reduce the number of calls to synchronize_net() when flushing caches commit 615e51fdda6f274e94b1e905fcaf6111e0d9aa20 upstream. When flushing the AVC, such as during a policy load, the various network caches are also flushed, with each making a call to synchronize_net() which has shown to be expensive in some cases. This patch consolidates the network cache flushes into a single AVC callback which only calls synchronize_net() once for each AVC cache flush. Change-Id: I2a7f020748d1adf2b68246f6ef86d0c871adffb7 Reported-by: Jaejyn Shin Signed-off-by: Paul Moore --- security/selinux/hooks.c | 14 ++++++++++++++ security/selinux/include/netif.h | 2 ++ security/selinux/include/netnode.h | 2 ++ security/selinux/include/netport.h | 2 ++ security/selinux/netif.c | 15 +-------------- security/selinux/netnode.c | 15 +-------------- security/selinux/netport.c | 15 +-------------- 7 files changed, 23 insertions(+), 42 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 0fa3195626a9..8923208b4be7 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -146,6 +146,17 @@ static int selinux_secmark_enabled(void) return (atomic_read(&selinux_secmark_refcount) > 0); } +static int selinux_netcache_avc_callback(u32 event) +{ + if (event == AVC_CALLBACK_RESET) { + sel_netif_flush(); + sel_netnode_flush(); + sel_netport_flush(); + synchronize_net(); + } + return 0; +} + /* * initialise the security for the init task */ @@ -5844,6 +5855,9 @@ static __init int selinux_init(void) if (register_security(&selinux_ops)) panic("SELinux: Unable to register with kernel.\n"); + if (avc_add_callback(selinux_netcache_avc_callback, AVC_CALLBACK_RESET)) + panic("SELinux: Unable to register AVC netcache callback\n"); + if (selinux_enforcing) printk(KERN_DEBUG "SELinux: Starting in enforcing mode\n"); else diff --git a/security/selinux/include/netif.h b/security/selinux/include/netif.h index 43d507242b42..57c6eae81eac 100644 --- a/security/selinux/include/netif.h +++ b/security/selinux/include/netif.h @@ -17,6 +17,8 @@ #ifndef _SELINUX_NETIF_H_ #define _SELINUX_NETIF_H_ +void sel_netif_flush(void); + int sel_netif_sid(int ifindex, u32 *sid); #endif /* _SELINUX_NETIF_H_ */ diff --git a/security/selinux/include/netnode.h b/security/selinux/include/netnode.h index df7a5ed6c694..937668dd3024 100644 --- a/security/selinux/include/netnode.h +++ b/security/selinux/include/netnode.h @@ -27,6 +27,8 @@ #ifndef _SELINUX_NETNODE_H #define _SELINUX_NETNODE_H +void sel_netnode_flush(void); + int sel_netnode_sid(void *addr, u16 family, u32 *sid); #endif diff --git a/security/selinux/include/netport.h b/security/selinux/include/netport.h index 4d965b83d735..d1ce896b2cb0 100644 --- a/security/selinux/include/netport.h +++ b/security/selinux/include/netport.h @@ -26,6 +26,8 @@ #ifndef _SELINUX_NETPORT_H #define _SELINUX_NETPORT_H +void sel_netport_flush(void); + int sel_netport_sid(u8 protocol, u16 pnum, u32 *sid); #endif diff --git a/security/selinux/netif.c b/security/selinux/netif.c index 47a49d1a6f6a..6c814ec4b01a 100644 --- a/security/selinux/netif.c +++ b/security/selinux/netif.c @@ -240,7 +240,7 @@ static void sel_netif_kill(int ifindex) * Remove all entries from the network interface table. * */ -static void sel_netif_flush(void) +void sel_netif_flush(void) { int idx; struct sel_netif *netif; @@ -252,15 +252,6 @@ static void sel_netif_flush(void) spin_unlock_bh(&sel_netif_lock); } -static int sel_netif_avc_callback(u32 event) -{ - if (event == AVC_CALLBACK_RESET) { - sel_netif_flush(); - synchronize_net(); - } - return 0; -} - static int sel_netif_netdev_notifier_handler(struct notifier_block *this, unsigned long event, void *ptr) { @@ -291,10 +282,6 @@ static __init int sel_netif_init(void) register_netdevice_notifier(&sel_netif_netdev_notifier); - err = avc_add_callback(sel_netif_avc_callback, AVC_CALLBACK_RESET); - if (err) - panic("avc_add_callback() failed, error %d\n", err); - return err; } diff --git a/security/selinux/netnode.c b/security/selinux/netnode.c index c5454c0477c3..bb8de9d9b429 100644 --- a/security/selinux/netnode.c +++ b/security/selinux/netnode.c @@ -281,7 +281,7 @@ int sel_netnode_sid(void *addr, u16 family, u32 *sid) * Remove all entries from the network address table. * */ -static void sel_netnode_flush(void) +void sel_netnode_flush(void) { unsigned int idx; struct sel_netnode *node, *node_tmp; @@ -298,15 +298,6 @@ static void sel_netnode_flush(void) spin_unlock_bh(&sel_netnode_lock); } -static int sel_netnode_avc_callback(u32 event) -{ - if (event == AVC_CALLBACK_RESET) { - sel_netnode_flush(); - synchronize_net(); - } - return 0; -} - static __init int sel_netnode_init(void) { int iter; @@ -320,10 +311,6 @@ static __init int sel_netnode_init(void) sel_netnode_hash[iter].size = 0; } - ret = avc_add_callback(sel_netnode_avc_callback, AVC_CALLBACK_RESET); - if (ret != 0) - panic("avc_add_callback() failed, error %d\n", ret); - return ret; } diff --git a/security/selinux/netport.c b/security/selinux/netport.c index d35379781c2c..73ac6784d091 100644 --- a/security/selinux/netport.c +++ b/security/selinux/netport.c @@ -217,7 +217,7 @@ int sel_netport_sid(u8 protocol, u16 pnum, u32 *sid) * Remove all entries from the network address table. * */ -static void sel_netport_flush(void) +void sel_netport_flush(void) { unsigned int idx; struct sel_netport *port, *port_tmp; @@ -234,15 +234,6 @@ static void sel_netport_flush(void) spin_unlock_bh(&sel_netport_lock); } -static int sel_netport_avc_callback(u32 event) -{ - if (event == AVC_CALLBACK_RESET) { - sel_netport_flush(); - synchronize_net(); - } - return 0; -} - static __init int sel_netport_init(void) { int iter; @@ -256,10 +247,6 @@ static __init int sel_netport_init(void) sel_netport_hash[iter].size = 0; } - ret = avc_add_callback(sel_netport_avc_callback, AVC_CALLBACK_RESET); - if (ret != 0) - panic("avc_add_callback() failed, error %d\n", ret); - return ret; } -- cgit v1.2.3 From 3d14663dd6a16ae50ed9f3d893e2be04c39b39a5 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Fri, 28 Feb 2014 07:23:24 -0500 Subject: selinux: put the mmap() DAC controls before the MAC controls commit 0909c0ae999c325b9d34c6f4710f40730ae3bc24 upstream. It turns out that doing the SELinux MAC checks for mmap() before the DAC checks was causing users and the SELinux policy folks headaches as users were seeing a lot of SELinux AVC denials for the memprotect:mmap_zero permission that would have also been denied by the normal DAC capability checks (CAP_SYS_RAWIO). Example: # cat mmap_test.c #include #include #include #include int main(int argc, char *argv[]) { int rc; void *mem; mem = mmap(0x0, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); if (mem == MAP_FAILED) return errno; printf("mem = %p\n", mem); munmap(mem, 4096); return 0; } # gcc -g -O0 -o mmap_test mmap_test.c # ./mmap_test mem = (nil) # ausearch -m AVC | grep mmap_zero type=AVC msg=audit(...): avc: denied { mmap_zero } for pid=1025 comm="mmap_test" scontext=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 tcontext=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 tclass=memprotect This patch corrects things so that when the above example is run by a user without CAP_SYS_RAWIO the SELinux AVC is no longer generated as the DAC capability check fails before the SELinux permission check. Change-Id: Ic3b2ef30d13c15ca7c60adbd3c3b93ebe251c7bc Signed-off-by: Paul Moore Acked-by: Stephen Smalley --- security/selinux/hooks.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 8923208b4be7..0436fe9bbe0e 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3178,24 +3178,20 @@ error: static int selinux_mmap_addr(unsigned long addr) { - int rc = 0; - u32 sid = current_sid(); + int rc; + + /* do DAC check on address space usage */ + rc = cap_mmap_addr(addr); + if (rc) + return rc; - /* - * notice that we are intentionally putting the SELinux check before - * the secondary cap_file_mmap check. This is such a likely attempt - * at bad behaviour/exploit that we always want to get the AVC, even - * if DAC would have also denied the operation. - */ if (addr < CONFIG_LSM_MMAP_MIN_ADDR) { + u32 sid = current_sid(); rc = avc_has_perm(sid, sid, SECCLASS_MEMPROTECT, MEMPROTECT__MMAP_ZERO, NULL); - if (rc) - return rc; } - /* do DAC check on address space usage */ - return cap_mmap_addr(addr); + return rc; } static int selinux_mmap_file(struct file *file, unsigned long reqprot, -- cgit v1.2.3 From dfba58cbad044ecd135eee6351b6695aa0e3ef05 Mon Sep 17 00:00:00 2001 From: Stephen Smalley Date: Mon, 6 Oct 2014 16:32:52 -0400 Subject: selinux: fix inode security list corruption commit 923190d32de4428afbea5e5773be86bea60a9925 upstream. sb_finish_set_opts() can race with inode_free_security() when initializing inode security structures for inodes created prior to initial policy load or by the filesystem during ->mount(). This appears to have always been a possible race, but commit 3dc91d4 ("SELinux: Fix possible NULL pointer dereference in selinux_inode_permission()") made it more evident by immediately reusing the unioned list/rcu element of the inode security structure for call_rcu() upon an inode_free_security(). But the underlying issue was already present before that commit as a possible use-after-free of isec. Shivnandan Kumar reported the list corruption and proposed a patch to split the list and rcu elements out of the union as separate fields of the inode_security_struct so that setting the rcu element would not affect the list element. However, this would merely hide the issue and not truly fix the code. This patch instead moves up the deletion of the list entry prior to dropping the sbsec->isec_lock initially. Then, if the inode is dropped subsequently, there will be no further references to the isec. Change-Id: I7c56670bddbb896f159701651758d2e7f739dff8 Reported-by: Shivnandan Kumar Signed-off-by: Stephen Smalley Signed-off-by: Paul Moore Signed-off-by: Greg Kroah-Hartman --- security/selinux/hooks.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 0436fe9bbe0e..68ae54b0616c 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -450,6 +450,7 @@ next_inode: list_entry(sbsec->isec_head.next, struct inode_security_struct, list); struct inode *inode = isec->inode; + list_del_init(&isec->list); spin_unlock(&sbsec->isec_lock); inode = igrab(inode); if (inode) { @@ -458,7 +459,6 @@ next_inode: iput(inode); } spin_lock(&sbsec->isec_lock); - list_del_init(&isec->list); goto next_inode; } spin_unlock(&sbsec->isec_lock); -- cgit v1.2.3 From 17e7d5a0dd16cbfc4e1996b963678611ec4a8623 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 19 Mar 2014 16:46:18 -0400 Subject: selinux: correctly label /proc inodes in use before the policy is loaded commit f64410ec665479d7b4b77b7519e814253ed0f686 upstream. This patch is based on an earlier patch by Eric Paris, he describes the problem below: "If an inode is accessed before policy load it will get placed on a list of inodes to be initialized after policy load. After policy load we call inode_doinit() which calls inode_doinit_with_dentry() on all inodes accessed before policy load. In the case of inodes in procfs that means we'll end up at the bottom where it does: /* Default to the fs superblock SID. */ isec->sid = sbsec->sid; if ((sbsec->flags & SE_SBPROC) && !S_ISLNK(inode->i_mode)) { if (opt_dentry) { isec->sclass = inode_mode_to_security_class(...) rc = selinux_proc_get_sid(opt_dentry, isec->sclass, &sid); if (rc) goto out_unlock; isec->sid = sid; } } Since opt_dentry is null, we'll never call selinux_proc_get_sid() and will leave the inode labeled with the label on the superblock. I believe a fix would be to mimic the behavior of xattrs. Look for an alias of the inode. If it can't be found, just leave the inode uninitialized (and pick it up later) if it can be found, we should be able to call selinux_proc_get_sid() ..." On a system exhibiting this problem, you will notice a lot of files in /proc with the generic "proc_t" type (at least the ones that were accessed early in the boot), for example: # ls -Z /proc/sys/kernel/shmmax | awk '{ print $4 " " $5 }' system_u:object_r:proc_t:s0 /proc/sys/kernel/shmmax However, with this patch in place we see the expected result: # ls -Z /proc/sys/kernel/shmmax | awk '{ print $4 " " $5 }' system_u:object_r:sysctl_kernel_t:s0 /proc/sys/kernel/shmmax Change-Id: I7742b4b7e53b45e4dd13d99c39553a927aa4a7e9 Cc: Eric Paris Signed-off-by: Paul Moore Acked-by: Eric Paris --- security/selinux/hooks.c | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 68ae54b0616c..65d7aa40adf3 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1374,15 +1374,33 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent isec->sid = sbsec->sid; if ((sbsec->flags & SE_SBPROC) && !S_ISLNK(inode->i_mode)) { - if (opt_dentry) { - isec->sclass = inode_mode_to_security_class(inode->i_mode); - rc = selinux_proc_get_sid(opt_dentry, - isec->sclass, - &sid); - if (rc) - goto out_unlock; - isec->sid = sid; - } + /* We must have a dentry to determine the label on + * procfs inodes */ + if (opt_dentry) + /* Called from d_instantiate or + * d_splice_alias. */ + dentry = dget(opt_dentry); + else + /* Called from selinux_complete_init, try to + * find a dentry. */ + dentry = d_find_alias(inode); + /* + * This can be hit on boot when a file is accessed + * before the policy is loaded. When we load policy we + * may find inodes that have no dentry on the + * sbsec->isec_head list. No reason to complain as + * these will get fixed up the next time we go through + * inode_doinit() with a dentry, before these inodes + * could be used again by userspace. + */ + if (!dentry) + goto out_unlock; + isec->sclass = inode_mode_to_security_class(inode->i_mode); + rc = selinux_proc_get_sid(dentry, isec->sclass, &sid); + dput(dentry); + if (rc) + goto out_unlock; + isec->sid = sid; } break; } -- cgit v1.2.3 From 872ecde0211c30e7ef8cdf13ee0600bbb43fed75 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 10 Sep 2014 17:09:57 -0400 Subject: selinux: make the netif cache namespace aware While SELinux largely ignores namespaces, for good reason, there are some places where it needs to at least be aware of namespaces in order to function correctly. Network namespaces are one example. Basic awareness of network namespaces are necessary in order to match a network interface's index number to an actual network device. This patch corrects a problem with network interfaces added to a non-init namespace, and can be reproduced with the following commands: [NOTE: the NetLabel configuration is here only to active the dynamic networking controls ] # netlabelctl unlbl add default address:0.0.0.0/0 \ label:system_u:object_r:unlabeled_t:s0 # netlabelctl unlbl add default address:::/0 \ label:system_u:object_r:unlabeled_t:s0 # netlabelctl cipsov4 add pass doi:100 tags:1 # netlabelctl map add domain:lspp_test_netlabel_t \ protocol:cipsov4,100 # ip link add type veth # ip netns add myns # ip link set veth1 netns myns # ip a add dev veth0 10.250.13.100/24 # ip netns exec myns ip a add dev veth1 10.250.13.101/24 # ip l set veth0 up # ip netns exec myns ip l set veth1 up # ping -c 1 10.250.13.101 # ip netns exec myns ping -c 1 10.250.13.100 Reported-by: Jiri Jaburek Signed-off-by: Paul Moore --- security/selinux/hooks.c | 33 ++++++++++++++++-------------- security/selinux/include/netif.h | 4 +++- security/selinux/include/objsec.h | 2 ++ security/selinux/netif.c | 43 +++++++++++++++++++++------------------ 4 files changed, 46 insertions(+), 36 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 65d7aa40adf3..d5cba5989f63 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -4200,15 +4200,15 @@ static int selinux_socket_unix_may_send(struct socket *sock, &ad); } -static int selinux_inet_sys_rcv_skb(int ifindex, char *addrp, u16 family, - u32 peer_sid, +static int selinux_inet_sys_rcv_skb(struct net *ns, int ifindex, + char *addrp, u16 family, u32 peer_sid, struct common_audit_data *ad) { int err; u32 if_sid; u32 node_sid; - err = sel_netif_sid(ifindex, &if_sid); + err = sel_netif_sid(ns, ifindex, &if_sid); if (err) return err; err = avc_has_perm(peer_sid, if_sid, @@ -4301,8 +4301,8 @@ static int selinux_socket_sock_rcv_skb(struct sock *sk, struct sk_buff *skb) err = selinux_skb_peerlbl_sid(skb, family, &peer_sid); if (err) return err; - err = selinux_inet_sys_rcv_skb(skb->skb_iif, addrp, family, - peer_sid, &ad); + err = selinux_inet_sys_rcv_skb(sock_net(sk), skb->skb_iif, + addrp, family, peer_sid, &ad); if (err) { selinux_netlbl_err(skb, err, 0); return err; @@ -4648,7 +4648,8 @@ out: #ifdef CONFIG_NETFILTER -static unsigned int selinux_ip_forward(struct sk_buff *skb, int ifindex, +static unsigned int selinux_ip_forward(struct sk_buff *skb, + const struct net_device *indev, u16 family) { int err; @@ -4674,14 +4675,14 @@ static unsigned int selinux_ip_forward(struct sk_buff *skb, int ifindex, ad.type = LSM_AUDIT_DATA_NET; ad.u.net = &net; - ad.u.net->netif = ifindex; + ad.u.net->netif = indev->ifindex; ad.u.net->family = family; if (selinux_parse_skb(skb, &ad, &addrp, 1, NULL) != 0) return NF_DROP; if (peerlbl_active) { - err = selinux_inet_sys_rcv_skb(ifindex, addrp, family, - peer_sid, &ad); + err = selinux_inet_sys_rcv_skb(dev_net(indev), indev->ifindex, + addrp, family, peer_sid, &ad); if (err) { selinux_netlbl_err(skb, err, 1); return NF_DROP; @@ -4710,7 +4711,7 @@ static unsigned int selinux_ipv4_forward(unsigned int hooknum, const struct net_device *out, int (*okfn)(struct sk_buff *)) { - return selinux_ip_forward(skb, in->ifindex, PF_INET); + return selinux_ip_forward(skb, in, PF_INET); } #if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) @@ -4720,7 +4721,7 @@ static unsigned int selinux_ipv6_forward(unsigned int hooknum, const struct net_device *out, int (*okfn)(struct sk_buff *)) { - return selinux_ip_forward(skb, in->ifindex, PF_INET6); + return selinux_ip_forward(skb, in, PF_INET6); } #endif /* IPV6 */ @@ -4788,11 +4789,13 @@ static unsigned int selinux_ip_postroute_compat(struct sk_buff *skb, return NF_ACCEPT; } -static unsigned int selinux_ip_postroute(struct sk_buff *skb, int ifindex, +static unsigned int selinux_ip_postroute(struct sk_buff *skb, + const struct net_device *outdev, u16 family) { u32 secmark_perm; u32 peer_sid; + int ifindex = outdev->ifindex; struct sock *sk; struct common_audit_data ad; struct lsm_network_audit net = {0,}; @@ -4857,7 +4860,7 @@ static unsigned int selinux_ip_postroute(struct sk_buff *skb, int ifindex, u32 if_sid; u32 node_sid; - if (sel_netif_sid(ifindex, &if_sid)) + if (sel_netif_sid(dev_net(outdev), ifindex, &if_sid)) return NF_DROP; if (avc_has_perm(peer_sid, if_sid, SECCLASS_NETIF, NETIF__EGRESS, &ad)) @@ -4879,7 +4882,7 @@ static unsigned int selinux_ipv4_postroute(unsigned int hooknum, const struct net_device *out, int (*okfn)(struct sk_buff *)) { - return selinux_ip_postroute(skb, out->ifindex, PF_INET); + return selinux_ip_postroute(skb, out, PF_INET); } #if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) @@ -4889,7 +4892,7 @@ static unsigned int selinux_ipv6_postroute(unsigned int hooknum, const struct net_device *out, int (*okfn)(struct sk_buff *)) { - return selinux_ip_postroute(skb, out->ifindex, PF_INET6); + return selinux_ip_postroute(skb, out, PF_INET6); } #endif /* IPV6 */ diff --git a/security/selinux/include/netif.h b/security/selinux/include/netif.h index 57c6eae81eac..c72145444090 100644 --- a/security/selinux/include/netif.h +++ b/security/selinux/include/netif.h @@ -17,9 +17,11 @@ #ifndef _SELINUX_NETIF_H_ #define _SELINUX_NETIF_H_ +#include + void sel_netif_flush(void); -int sel_netif_sid(int ifindex, u32 *sid); +int sel_netif_sid(struct net *ns, int ifindex, u32 *sid); #endif /* _SELINUX_NETIF_H_ */ diff --git a/security/selinux/include/objsec.h b/security/selinux/include/objsec.h index 6fd9dd256a62..2a3be7adc071 100644 --- a/security/selinux/include/objsec.h +++ b/security/selinux/include/objsec.h @@ -24,6 +24,7 @@ #include #include #include +#include #include "flask.h" #include "avc.h" @@ -78,6 +79,7 @@ struct ipc_security_struct { }; struct netif_security_struct { + struct net *ns; /* network namespace */ int ifindex; /* device index */ u32 sid; /* SID for this interface */ }; diff --git a/security/selinux/netif.c b/security/selinux/netif.c index 6c814ec4b01a..548651a73876 100644 --- a/security/selinux/netif.c +++ b/security/selinux/netif.c @@ -45,6 +45,7 @@ static struct list_head sel_netif_hash[SEL_NETIF_HASH_SIZE]; /** * sel_netif_hashfn - Hashing function for the interface table + * @ns: the network namespace * @ifindex: the network interface * * Description: @@ -52,13 +53,14 @@ static struct list_head sel_netif_hash[SEL_NETIF_HASH_SIZE]; * bucket number for the given interface. * */ -static inline u32 sel_netif_hashfn(int ifindex) +static inline u32 sel_netif_hashfn(const struct net *ns, int ifindex) { - return (ifindex & (SEL_NETIF_HASH_SIZE - 1)); + return (((uintptr_t)ns + ifindex) & (SEL_NETIF_HASH_SIZE - 1)); } /** * sel_netif_find - Search for an interface record + * @ns: the network namespace * @ifindex: the network interface * * Description: @@ -66,15 +68,15 @@ static inline u32 sel_netif_hashfn(int ifindex) * If an entry can not be found in the table return NULL. * */ -static inline struct sel_netif *sel_netif_find(int ifindex) +static inline struct sel_netif *sel_netif_find(const struct net *ns, + int ifindex) { - int idx = sel_netif_hashfn(ifindex); + int idx = sel_netif_hashfn(ns, ifindex); struct sel_netif *netif; list_for_each_entry_rcu(netif, &sel_netif_hash[idx], list) - /* all of the devices should normally fit in the hash, so we - * optimize for that case */ - if (likely(netif->nsec.ifindex == ifindex)) + if (net_eq(netif->nsec.ns, ns) && + netif->nsec.ifindex == ifindex) return netif; return NULL; @@ -96,7 +98,7 @@ static int sel_netif_insert(struct sel_netif *netif) if (sel_netif_total >= SEL_NETIF_HASH_MAX) return -ENOSPC; - idx = sel_netif_hashfn(netif->nsec.ifindex); + idx = sel_netif_hashfn(netif->nsec.ns, netif->nsec.ifindex); list_add_rcu(&netif->list, &sel_netif_hash[idx]); sel_netif_total++; @@ -120,6 +122,7 @@ static void sel_netif_destroy(struct sel_netif *netif) /** * sel_netif_sid_slow - Lookup the SID of a network interface using the policy + * @ns: the network namespace * @ifindex: the network interface * @sid: interface SID * @@ -130,7 +133,7 @@ static void sel_netif_destroy(struct sel_netif *netif) * failure. * */ -static int sel_netif_sid_slow(int ifindex, u32 *sid) +static int sel_netif_sid_slow(struct net *ns, int ifindex, u32 *sid) { int ret; struct sel_netif *netif; @@ -140,7 +143,7 @@ static int sel_netif_sid_slow(int ifindex, u32 *sid) /* NOTE: we always use init's network namespace since we don't * currently support containers */ - dev = dev_get_by_index(&init_net, ifindex); + dev = dev_get_by_index(ns, ifindex); if (unlikely(dev == NULL)) { printk(KERN_WARNING "SELinux: failure in sel_netif_sid_slow()," @@ -149,7 +152,7 @@ static int sel_netif_sid_slow(int ifindex, u32 *sid) } spin_lock_bh(&sel_netif_lock); - netif = sel_netif_find(ifindex); + netif = sel_netif_find(ns, ifindex); if (netif != NULL) { *sid = netif->nsec.sid; ret = 0; @@ -163,6 +166,7 @@ static int sel_netif_sid_slow(int ifindex, u32 *sid) ret = security_netif_sid(dev->name, &new->nsec.sid); if (ret != 0) goto out; + new->nsec.ns = ns; new->nsec.ifindex = ifindex; ret = sel_netif_insert(new); if (ret != 0) @@ -184,6 +188,7 @@ out: /** * sel_netif_sid - Lookup the SID of a network interface + * @ns: the network namespace * @ifindex: the network interface * @sid: interface SID * @@ -195,12 +200,12 @@ out: * on failure. * */ -int sel_netif_sid(int ifindex, u32 *sid) +int sel_netif_sid(struct net *ns, int ifindex, u32 *sid) { struct sel_netif *netif; rcu_read_lock(); - netif = sel_netif_find(ifindex); + netif = sel_netif_find(ns, ifindex); if (likely(netif != NULL)) { *sid = netif->nsec.sid; rcu_read_unlock(); @@ -208,11 +213,12 @@ int sel_netif_sid(int ifindex, u32 *sid) } rcu_read_unlock(); - return sel_netif_sid_slow(ifindex, sid); + return sel_netif_sid_slow(ns, ifindex, sid); } /** * sel_netif_kill - Remove an entry from the network interface table + * @ns: the network namespace * @ifindex: the network interface * * Description: @@ -220,13 +226,13 @@ int sel_netif_sid(int ifindex, u32 *sid) * table if it exists. * */ -static void sel_netif_kill(int ifindex) +static void sel_netif_kill(const struct net *ns, int ifindex) { struct sel_netif *netif; rcu_read_lock(); spin_lock_bh(&sel_netif_lock); - netif = sel_netif_find(ifindex); + netif = sel_netif_find(ns, ifindex); if (netif) sel_netif_destroy(netif); spin_unlock_bh(&sel_netif_lock); @@ -257,11 +263,8 @@ static int sel_netif_netdev_notifier_handler(struct notifier_block *this, { struct net_device *dev = ptr; - if (dev_net(dev) != &init_net) - return NOTIFY_DONE; - if (event == NETDEV_DOWN) - sel_netif_kill(dev->ifindex); + sel_netif_kill(dev_net(dev), dev->ifindex); return NOTIFY_DONE; } -- cgit v1.2.3 From b3df6b14db2cb6919081441789e86ca2efceccca Mon Sep 17 00:00:00 2001 From: Stephen Smalley Date: Wed, 21 Jan 2015 13:59:39 -0500 Subject: selinux: Remove obsolete selinux_audit_data initialization. Commit 899838b25f063a94594b1df6e0100aea1ec57fac eliminated the need to initialize selinux_audit_data except in the slow path, when it is handled by slow_avc_audit(). That commit removed all other initializations of selinux_audit_data but this one remained since the binder security hooks are not yet upstream (posted them to linux-kernel today). Change-Id: I735e4500cde23275686cb3208068cbf8dd7bccd7 Signed-off-by: Stephen Smalley --- security/selinux/hooks.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index d5cba5989f63..63b7a331906a 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1907,12 +1907,10 @@ static int selinux_binder_transfer_file(struct task_struct *from, struct task_st struct inode *inode = file->f_path.dentry->d_inode; struct inode_security_struct *isec = inode->i_security; struct common_audit_data ad; - struct selinux_audit_data sad = {0,}; int rc; ad.type = LSM_AUDIT_DATA_PATH; ad.u.path = file->f_path; - ad.selinux_audit_data = &sad; if (sid != fsec->sid) { rc = avc_has_perm(sid, fsec->sid, -- cgit v1.2.3 From ea262b409baefe776613660d6b7fdf029f5c96be Mon Sep 17 00:00:00 2001 From: Jack Pham Date: Tue, 18 Feb 2014 22:16:19 -0800 Subject: usb: gadget: android: Save/restore ep0 completion function The android_setup() function currently gives the f_accessory setup function first opportunity to handle control requests in order to support Android Open Accessory (AOA) hosts. That function makes use of cdev->req and overrides its completion function, but not in all cases. Thus, if a later request uses the same request pointer but doesn't (re)set req->complete it could result in the wrong completion function being called and causing invalid memory access. One way to fix this would be to explicitly set req->complete in all cases but that might require auditing all function drivers that have ep0 handling. Instead, note that the composite device had already initially set cdev->req->complete and simply cache and restore that pointer at the start of android_setup(). Change-Id: I33bcd17bd20687a349d537d1013b52a2afef6996 Signed-off-by: Jack Pham --- drivers/usb/gadget/android.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/usb/gadget/android.c b/drivers/usb/gadget/android.c index 04cbeb134814..83b1240d9b19 100644 --- a/drivers/usb/gadget/android.c +++ b/drivers/usb/gadget/android.c @@ -90,6 +90,9 @@ struct android_dev { struct usb_composite_dev *cdev; struct device *dev; + void (*setup_complete)(struct usb_ep *ep, + struct usb_request *req); + bool enabled; int disable_depth; struct mutex mutex; @@ -1312,6 +1315,9 @@ static int android_bind(struct usb_composite_dev *cdev) struct usb_gadget *gadget = cdev->gadget; int id, ret; + /* Save the default handler */ + dev->setup_complete = cdev->req->complete; + /* * Start disconnected. Userspace will connect the gadget once * it is done configuring the functions. @@ -1378,6 +1384,7 @@ android_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *c) req->zero = 0; req->length = 0; + req->complete = dev->setup_complete; gadget->ep0->driver_data = cdev; list_for_each_entry(f, &dev->enabled_functions, enabled_list) { -- cgit v1.2.3 From 0d7c7ba9f6428e694130a40e270516618f27ce14 Mon Sep 17 00:00:00 2001 From: Rom Lemarchand Date: Sat, 24 Jan 2015 15:08:42 -0800 Subject: staging: android: ashmem: add missing include Include into ashmem.h to ensure referenced types are defined Signed-off-by: Rom Lemarchand Change-Id: Iac18ed86dd47296d02a269607b1514724aaa9958 --- drivers/staging/android/uapi/ashmem.h | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/staging/android/uapi/ashmem.h b/drivers/staging/android/uapi/ashmem.h index ba4743c71d6b..13df42d200b7 100644 --- a/drivers/staging/android/uapi/ashmem.h +++ b/drivers/staging/android/uapi/ashmem.h @@ -13,6 +13,7 @@ #define _UAPI_LINUX_ASHMEM_H #include +#include #define ASHMEM_NAME_LEN 256 -- cgit v1.2.3 From 41ba06281471b7ade8782204079f92633caf7285 Mon Sep 17 00:00:00 2001 From: Amit Pundir Date: Fri, 16 Jan 2015 05:41:10 +0530 Subject: usb: gadget: check for accessory device before disconnecting HIDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While disabling ConfigFS Android gadget, android_disconnect() calls kill_all_hid_devices(), if CONFIG_USB_CONFIGFS_F_ACC is enabled, to free the registered HIDs without checking whether the USB accessory device really exist or not. If USB accessory device doesn't exist then we run into following kernel panic: ----8<---- [  136.724761] Unable to handle kernel NULL pointer dereference at virtual address 00000064 [  136.724809] pgd = c0204000 [  136.731924] [00000064] *pgd=00000000 [  136.737830] Internal error: Oops: 5 [#1] SMP ARM [  136.738108] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 3.18.0-rc4-00400-gf75300e-dirty #76 [  136.742788] task: c0fb19d8 ti: c0fa4000 task.ti: c0fa4000 [  136.750890] PC is at _raw_spin_lock_irqsave+0x24/0x60 [  136.756246] LR is at kill_all_hid_devices+0x24/0x114 ---->8---- This patch adds a test to check if USB Accessory device exists before freeing HIDs. Change-Id: Ie229feaf0de3f4f7a151fcaa9a994e34e15ff73b Signed-off-by: Amit Pundir (cherry picked from commit 32a71bce154cb89a549b9b7d28e8cf03b889d849) --- drivers/usb/gadget/f_accessory.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/usb/gadget/f_accessory.c b/drivers/usb/gadget/f_accessory.c index a401acdceb4d..0237f1e059b4 100644 --- a/drivers/usb/gadget/f_accessory.c +++ b/drivers/usb/gadget/f_accessory.c @@ -951,6 +951,10 @@ kill_all_hid_devices(struct acc_dev *dev) struct list_head *entry, *temp; unsigned long flags; + /* do nothing if usb accessory device doesn't exist */ + if (!dev) + return; + spin_lock_irqsave(&dev->lock, flags); list_for_each_safe(entry, temp, &dev->hid_list) { hid = list_entry(entry, struct acc_hid_dev, list); -- cgit v1.2.3 From 86a47ad60de5221c3869821d3552dcd1c89199f5 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 10 Sep 2014 23:23:02 +0200 Subject: ipv6: clean up anycast when an interface is destroyed If we try to rmmod the driver for an interface while sockets with setsockopt(JOIN_ANYCAST) are alive, some refcounts aren't cleaned up and we get stuck on: unregister_netdevice: waiting for ens3 to become free. Usage count = 1 If we LEAVE_ANYCAST/close everything before rmmod'ing, there is no problem. We need to perform a cleanup similar to the one for multicast in addrconf_ifdown(how == 1). BUG: 18902601 Change-Id: I6d51aed5755eb5738fcba91950e7773a1c985d2e Signed-off-by: Sabrina Dubroca Acked-by: Hannes Frederic Sowa Signed-off-by: David S. Miller --- include/net/addrconf.h | 1 + net/ipv6/addrconf.c | 8 +++++--- net/ipv6/anycast.c | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 96a8afe33c52..25d5a98f206c 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -196,6 +196,7 @@ extern int ipv6_dev_ac_inc(struct net_device *dev, const struct in6_addr *addr); extern int __ipv6_dev_ac_dec(struct inet6_dev *idev, const struct in6_addr *addr); extern bool ipv6_chk_acast_addr(struct net *net, struct net_device *dev, const struct in6_addr *addr); +extern void ipv6_ac_destroy_dev(struct inet6_dev *idev); /* Device notifier */ diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 165cee964e75..adffa4747532 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -3155,11 +3155,13 @@ static int addrconf_ifdown(struct net_device *dev, int how) write_unlock_bh(&idev->lock); - /* Step 5: Discard multicast list */ - if (how) + /* Step 5: Discard anycast and multicast list */ + if (how) { + ipv6_ac_destroy_dev(idev); ipv6_mc_destroy_dev(idev); - else + } else { ipv6_mc_down(idev); + } idev->tstamp = jiffies; diff --git a/net/ipv6/anycast.c b/net/ipv6/anycast.c index 5a80f15a9de2..1e97d0886597 100644 --- a/net/ipv6/anycast.c +++ b/net/ipv6/anycast.c @@ -341,6 +341,27 @@ static int ipv6_dev_ac_dec(struct net_device *dev, const struct in6_addr *addr) return __ipv6_dev_ac_dec(idev, addr); } +void ipv6_ac_destroy_dev(struct inet6_dev *idev) +{ + struct ifacaddr6 *aca; + + write_lock_bh(&idev->lock); + while ((aca = idev->ac_list) != NULL) { + idev->ac_list = aca->aca_next; + write_unlock_bh(&idev->lock); + + addrconf_leave_solict(idev, &aca->aca_addr); + + dst_hold(&aca->aca_rt->dst); + ip6_del_rt(aca->aca_rt); + + aca_put(aca); + + write_lock_bh(&idev->lock); + } + write_unlock_bh(&idev->lock); +} + /* * check if the interface has this anycast address * called with rcu_read_lock() -- cgit v1.2.3 From aac55d5ed474e8b6b08d85e742727e5dc0b7893c Mon Sep 17 00:00:00 2001 From: Mohamad Ayyash Date: Tue, 13 Jan 2015 19:20:44 -0800 Subject: xt_qtaguid: Use sk_callback_lock read locks before reading sk->sk_socket It prevents a kernel panic when accessing sk->sk_socket fields due to NULLing sk->sk_socket when sock_orphan is called through sk_common_release. Change-Id: I4aa46b4e2d8600e4d4ef8dcdd363aa4e6e5f8433 Signed-off-by: Mohamad Ayyash (cherry picked from commit cdea0ebcb8bcfe57688f6cb692b49e550ebd9796) --- net/netfilter/xt_qtaguid.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/net/netfilter/xt_qtaguid.c b/net/netfilter/xt_qtaguid.c index 9128933a5b21..bcc622e64b16 100644 --- a/net/netfilter/xt_qtaguid.c +++ b/net/netfilter/xt_qtaguid.c @@ -1658,6 +1658,7 @@ static bool qtaguid_mt(const struct sk_buff *skb, struct xt_action_param *par) struct sock *sk; uid_t sock_uid; bool res; + bool set_sk_callback_lock = false; if (unlikely(module_passive)) return (info->match ^ info->invert) == 0; @@ -1715,6 +1716,8 @@ static bool qtaguid_mt(const struct sk_buff *skb, struct xt_action_param *par) MT_DEBUG("qtaguid[%d]: sk=%p got_sock=%d fam=%d proto=%d\n", par->hooknum, sk, got_sock, par->family, ipx_proto(skb, par)); if (sk != NULL) { + set_sk_callback_lock = true; + read_lock_bh(&sk->sk_callback_lock); MT_DEBUG("qtaguid[%d]: sk=%p->sk_socket=%p->file=%p\n", par->hooknum, sk, sk->sk_socket, sk->sk_socket ? sk->sk_socket->file : (void *)-1LL); @@ -1794,6 +1797,8 @@ static bool qtaguid_mt(const struct sk_buff *skb, struct xt_action_param *par) put_sock_ret_res: if (got_sock) xt_socket_put_sk(sk); + if (set_sk_callback_lock) + read_unlock_bh(&sk->sk_callback_lock); ret_res: MT_DEBUG("qtaguid[%d]: left %d\n", par->hooknum, res); return res; -- cgit v1.2.3 From f5aa3045b11aeff79687236a2cebf1e4b068b995 Mon Sep 17 00:00:00 2001 From: Rom Lemarchand Date: Thu, 5 Feb 2015 16:07:59 -0800 Subject: kbuild: make it possible to specify the module output dir Make modinst_dir user-defined on the command line. This allows to do things like: make MODLIB=output/ modinst_dir=. modules_install to ensure all the .ko are in the output/ directory. Change-Id: I2bc007eea27ee744d35289e26e4a8ac43ba04151 Signed-off-by: Rom Lemarchand (cherry picked from commit ae7f28e5aa168a008c153ccff7cc437066d016c3) --- scripts/Makefile.modinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst index ecbb44797e28..8900c5c5390d 100644 --- a/scripts/Makefile.modinst +++ b/scripts/Makefile.modinst @@ -24,7 +24,7 @@ quiet_cmd_modules_install = INSTALL $@ INSTALL_MOD_DIR ?= extra ext-mod-dir = $(INSTALL_MOD_DIR)$(subst $(patsubst %/,%,$(KBUILD_EXTMOD)),,$(@D)) -modinst_dir = $(if $(KBUILD_EXTMOD),$(ext-mod-dir),kernel/$(@D)) +modinst_dir ?= $(if $(KBUILD_EXTMOD),$(ext-mod-dir),kernel/$(@D)) $(modules): $(call cmd,modules_install,$(MODLIB)/$(modinst_dir)) -- cgit v1.2.3