ppc: fix endian check (#1029)

* Remove `big_endian` field of `cs_struct`

Added a helper macro `MODE_IS_BIG_ENDIAN()` to check if
`CS_MODE_BIG_ENDIAN` is set.

Refactored `cs_open()` check for valid mode out of arch-specific code
into arch-independent code. Also added a valid mode check to
`cs_option()`.  The checks use a new global array
`arch_disallowed_mode_mask[]`, which is initialized in the arch-specific
`*_enable()` functions.

Fixes bug where endianness could not be set for ppc.

* Fix Mac OS brew for Travis CI

.travis.yml

@@ -18,9 +18,9 @@ matrix:
   include:
     - if: branch = master
       os: osx
-      script: brew install --HEAD capstone && brew test capstone
+      script: brew update && brew install --HEAD capstone && brew test capstone
       compiler: gcc
     - if: branch = master
       os: osx
-      script: brew install --HEAD capstone && brew test capstone
+      script: brew update && brew install --HEAD capstone && brew test capstone
       compiler: clang

arch/AArch64/AArch64Disassembler.c

@@ -238,7 +238,7 @@ static DecodeStatus _getInstruction(cs_struct *ud, MCInst *MI,
 			MI->flat_insn->detail->arm64.operands[i].vector_index = -1;
 	}
 
-	if (ud->big_endian)
+	if (MODE_IS_BIG_ENDIAN(ud->mode))
 		insn = (code[3] << 0) | (code[2] << 8) |
 			(code[1] <<  16) | (code[0] <<  24);
 	else

arch/AArch64/AArch64Module.c

@@ -12,11 +12,6 @@
 static cs_err init(cs_struct *ud)
 {
 	MCRegisterInfo *mri;
-
-	// verify if requested mode is valid
-	if (ud->mode & ~(CS_MODE_LITTLE_ENDIAN | CS_MODE_ARM | CS_MODE_BIG_ENDIAN))
-		return CS_ERR_MODE;
-
 	mri = cs_mem_malloc(sizeof(*mri));
 
 	AArch64_init(mri);
@@ -36,7 +31,7 @@ static cs_err init(cs_struct *ud)
 static cs_err option(cs_struct *handle, cs_opt_type type, size_t value)
 {
 	if (type == CS_OPT_MODE) {
-		handle->big_endian = (((cs_mode)value & CS_MODE_BIG_ENDIAN) != 0);
+		handle->mode = (cs_mode)value;
 	}
 
 	return CS_ERR_OK;
@@ -51,6 +46,7 @@ void AArch64_enable(void)
 	arch_init[CS_ARCH_ARM64] = init;
 	arch_option[CS_ARCH_ARM64] = option;
 	arch_destroy[CS_ARCH_ARM64] = destroy;
+	arch_disallowed_mode_mask[CS_ARCH_ARM64] = ~(CS_MODE_LITTLE_ENDIAN | CS_MODE_ARM | CS_MODE_BIG_ENDIAN);
 
 	// support this arch
 	all_arch |= (1 << CS_ARCH_ARM64);

arch/ARM/ARMDisassembler.c

@@ -462,7 +462,7 @@ static DecodeStatus _ARM_getInstruction(cs_struct *ud, MCInst *MI, const uint8_t
 
 	memcpy(bytes, code, 4);
 
-	if (ud->big_endian)
+	if (MODE_IS_BIG_ENDIAN(ud->mode))
 		insn = (bytes[3] << 0) |
 			(bytes[2] << 8) |
 			(bytes[1] <<  16) |
@@ -704,7 +704,7 @@ static DecodeStatus _Thumb_getInstruction(cs_struct *ud, MCInst *MI, const uint8
 
 	memcpy(bytes, code, 2);
 
-	if (ud->big_endian)
+	if (MODE_IS_BIG_ENDIAN(ud->mode))
 		insn16 = (bytes[0] << 8) | bytes[1];
 	else
 		insn16 = (bytes[1] << 8) | bytes[0];
@@ -757,7 +757,7 @@ static DecodeStatus _Thumb_getInstruction(cs_struct *ud, MCInst *MI, const uint8
 
 	memcpy(bytes, code, 4);
 
-	if (ud->big_endian)
+	if (MODE_IS_BIG_ENDIAN(ud->mode))
 		insn32 = (bytes[3] <<  0) |
 			(bytes[2] <<  8) |
 			(bytes[1] << 16) |

arch/ARM/ARMModule.c

@@ -12,12 +12,6 @@
 static cs_err init(cs_struct *ud)
 {
 	MCRegisterInfo *mri;
-
-	// verify if requested mode is valid
-	if (ud->mode & ~(CS_MODE_LITTLE_ENDIAN | CS_MODE_ARM | CS_MODE_V8 |
-				CS_MODE_MCLASS | CS_MODE_THUMB | CS_MODE_BIG_ENDIAN))
-		return CS_ERR_MODE;
-
 	mri = cs_mem_malloc(sizeof(*mri));
 
 	ARM_init(mri);
@@ -49,7 +43,6 @@ static cs_err option(cs_struct *handle, cs_opt_type type, size_t value)
 				handle->disasm = ARM_getInstruction;
 
 			handle->mode = (cs_mode)value;
-			handle->big_endian = ((handle->mode & CS_MODE_BIG_ENDIAN) != 0);
 
 			break;
 		case CS_OPT_SYNTAX:
@@ -72,6 +65,9 @@ void ARM_enable(void)
 	arch_init[CS_ARCH_ARM] = init;
 	arch_option[CS_ARCH_ARM] = option;
 	arch_destroy[CS_ARCH_ARM] = destroy;
+	arch_disallowed_mode_mask[CS_ARCH_ARM] = ~(CS_MODE_LITTLE_ENDIAN |
+		CS_MODE_ARM | CS_MODE_V8 | CS_MODE_MCLASS | CS_MODE_THUMB |
+		CS_MODE_BIG_ENDIAN);
 
 	// support this arch
 	all_arch |= (1 << CS_ARCH_ARM);

arch/Mips/MipsDisassembler.c

@@ -398,7 +398,7 @@ bool Mips_getInstruction(csh ud, const uint8_t *code, size_t code_len, MCInst *i
 	DecodeStatus status = MipsDisassembler_getInstruction(handle->mode, instr,
 			code, code_len,
 			size,
-			address, handle->big_endian, (MCRegisterInfo *)info);
+			address, MODE_IS_BIG_ENDIAN(handle->mode), (MCRegisterInfo *)info);
 
 	return status == MCDisassembler_Success;
 }
@@ -450,9 +450,8 @@ bool Mips64_getInstruction(csh ud, const uint8_t *code, size_t code_len, MCInst
 	cs_struct *handle = (cs_struct *)(uintptr_t)ud;
 
 	DecodeStatus status = Mips64Disassembler_getInstruction(handle->mode, instr,
-			code, code_len,
+			code, code_len, size, address,
-			size,
+			MODE_IS_BIG_ENDIAN(handle->mode), (MCRegisterInfo *)info);
-			address, handle->big_endian, (MCRegisterInfo *)info);
 
 	return status == MCDisassembler_Success;
 }

arch/Mips/MipsModule.c

@@ -22,13 +22,6 @@ static inline cs_mode updated_mode(cs_mode mode)
 static cs_err init(cs_struct *ud)
 {
 	MCRegisterInfo *mri;
-
-	// verify if requested mode is valid
-	if (ud->mode & ~(CS_MODE_LITTLE_ENDIAN | CS_MODE_32 | CS_MODE_64 |
-				CS_MODE_MICRO | CS_MODE_MIPS32R6 |
-				CS_MODE_MIPSGP64 | CS_MODE_BIG_ENDIAN))
-		return CS_ERR_MODE;
-
 	mri = cs_mem_malloc(sizeof(*mri));
 
 	Mips_init(mri);
@@ -59,7 +52,6 @@ static cs_err option(cs_struct *handle, cs_opt_type type, size_t value)
 			handle->disasm = Mips64_getInstruction;
 
 		handle->mode = (cs_mode)value;
-		handle->big_endian = ((handle->mode & CS_MODE_BIG_ENDIAN) != 0);
 	}
 	return CS_ERR_OK;
 }
@@ -73,6 +65,9 @@ void Mips_enable(void)
 	arch_init[CS_ARCH_MIPS] = init;
 	arch_option[CS_ARCH_MIPS] = option;
 	arch_destroy[CS_ARCH_MIPS] = destroy;
+	arch_disallowed_mode_mask[CS_ARCH_MIPS] = ~(CS_MODE_LITTLE_ENDIAN |
+		CS_MODE_32 | CS_MODE_64 | CS_MODE_MICRO | CS_MODE_MIPS32R6 |
+		CS_MODE_MIPSGP64 | CS_MODE_BIG_ENDIAN);
 
 	// support this arch
 	all_arch |= (1 << CS_ARCH_MIPS);

arch/PowerPC/PPCDisassembler.c

@@ -342,7 +342,7 @@ static DecodeStatus getInstruction(MCInst *MI,
 	}
 
 	// The instruction is big-endian encoded.
-	if (MI->csh->mode & CS_MODE_BIG_ENDIAN)
+	if (MODE_IS_BIG_ENDIAN(MI->csh->mode))
 		insn = (code[0] << 24) | (code[1] << 16) |
 			(code[2] <<  8) | (code[3] <<  0);
 	else

arch/PowerPC/PPCModule.c

@@ -12,12 +12,6 @@
 static cs_err init(cs_struct *ud)
 {
 	MCRegisterInfo *mri;
-
-	// verify if requested mode is valid
-	if (ud->mode & ~(CS_MODE_LITTLE_ENDIAN | CS_MODE_32 | CS_MODE_64 |
-				CS_MODE_BIG_ENDIAN))
-		return CS_ERR_MODE;
-
 	mri = (MCRegisterInfo *) cs_mem_malloc(sizeof(*mri));
 
 	PPC_init(mri);
@@ -41,7 +35,7 @@ static cs_err option(cs_struct *handle, cs_opt_type type, size_t value)
 		handle->syntax = (int) value;
 
 	if (type == CS_OPT_MODE) {
-		handle->big_endian = (((cs_mode)value & CS_MODE_BIG_ENDIAN) != 0);
+		handle->mode = (cs_mode)value;
 	}
 
 	return CS_ERR_OK;
@@ -56,6 +50,8 @@ void PPC_enable(void)
 	arch_init[CS_ARCH_PPC] = init;
 	arch_option[CS_ARCH_PPC] = option;
 	arch_destroy[CS_ARCH_PPC] = destroy;
+	arch_disallowed_mode_mask[CS_ARCH_PPC] = ~(CS_MODE_LITTLE_ENDIAN |
+		CS_MODE_32 | CS_MODE_64 | CS_MODE_BIG_ENDIAN);
 
 	// support this arch
 	all_arch |= (1 << CS_ARCH_PPC);

arch/Sparc/SparcModule.c

@@ -12,11 +12,6 @@
 static cs_err init(cs_struct *ud)
 {
 	MCRegisterInfo *mri;
-
-	// verify if requested mode is valid
-	if (ud->mode & ~(CS_MODE_BIG_ENDIAN | CS_MODE_V9))
-		return CS_ERR_MODE;
-
 	mri = cs_mem_malloc(sizeof(*mri));
 
 	Sparc_init(mri);
@@ -40,7 +35,7 @@ static cs_err option(cs_struct *handle, cs_opt_type type, size_t value)
 		handle->syntax = (int) value;
 
 	if (type == CS_OPT_MODE) {
-		handle->big_endian = (((cs_mode)value & CS_MODE_BIG_ENDIAN) != 0);
+		handle->mode = (cs_mode)value;
 	}
 
 	return CS_ERR_OK;
@@ -55,6 +50,8 @@ void Sparc_enable(void)
 	arch_init[CS_ARCH_SPARC] = init;
 	arch_option[CS_ARCH_SPARC] = option;
 	arch_destroy[CS_ARCH_SPARC] = destroy;
+	arch_disallowed_mode_mask[CS_ARCH_SPARC] =
+		~(CS_MODE_BIG_ENDIAN | CS_MODE_V9);
 
 	// support this arch
 	all_arch |= (1 << CS_ARCH_SPARC);

arch/SystemZ/SystemZModule.c

@@ -12,7 +12,6 @@
 static cs_err init(cs_struct *ud)
 {
 	MCRegisterInfo *mri;
-
 	mri = cs_mem_malloc(sizeof(*mri));
 
 	SystemZ_init(mri);
@@ -35,6 +34,9 @@ static cs_err option(cs_struct *handle, cs_opt_type type, size_t value)
 	if (type == CS_OPT_SYNTAX)
 		handle->syntax = (int) value;
 
+	// Do not set mode because only CS_MODE_BIG_ENDIAN is valid; we cannot
+	// test for CS_MODE_LITTLE_ENDIAN because it is 0
+
 	return CS_ERR_OK;
 }
 
@@ -47,6 +49,7 @@ void SystemZ_enable(void)
 	arch_init[CS_ARCH_SYSZ] = init;
 	arch_option[CS_ARCH_SYSZ] = option;
 	arch_destroy[CS_ARCH_SYSZ] = destroy;
+	arch_disallowed_mode_mask[CS_ARCH_SYSZ] = ~CS_MODE_BIG_ENDIAN;
 
 	// support this arch
 	all_arch |= (1 << CS_ARCH_SYSZ);

arch/X86/X86Module.c

@@ -12,11 +12,6 @@
 static cs_err init(cs_struct *ud)
 {
 	MCRegisterInfo *mri;
-
-	// verify if requested mode is valid
-	if (ud->mode & ~(CS_MODE_LITTLE_ENDIAN | CS_MODE_32 | CS_MODE_64 | CS_MODE_16))
-		return CS_ERR_MODE;
-
 	mri = cs_mem_malloc(sizeof(*mri));
 
 	X86_init(mri);
@@ -96,6 +91,8 @@ void X86_enable(void)
 	arch_init[CS_ARCH_X86] = init;
 	arch_option[CS_ARCH_X86] = option;
 	arch_destroy[CS_ARCH_X86] = destroy;
+	arch_disallowed_mode_mask[CS_ARCH_X86] = ~(CS_MODE_LITTLE_ENDIAN |
+		CS_MODE_32 | CS_MODE_64 | CS_MODE_16);
 
 	// support this arch
 	all_arch |= (1 << CS_ARCH_X86);

arch/XCore/XCoreModule.c

@@ -12,7 +12,6 @@
 static cs_err init(cs_struct *ud)
 {
 	MCRegisterInfo *mri;
-
 	mri = cs_mem_malloc(sizeof(*mri));
 
 	XCore_init(mri);
@@ -32,6 +31,9 @@ static cs_err init(cs_struct *ud)
 
 static cs_err option(cs_struct *handle, cs_opt_type type, size_t value)
 {
+	// Do not set mode because only CS_MODE_BIG_ENDIAN is valid; we cannot
+	// test for CS_MODE_LITTLE_ENDIAN because it is 0
+
 	return CS_ERR_OK;
 }
 
@@ -44,6 +46,7 @@ void XCore_enable(void)
 	arch_init[CS_ARCH_XCORE] = init;
 	arch_option[CS_ARCH_XCORE] = option;
 	arch_destroy[CS_ARCH_XCORE] = destroy;
+	arch_disallowed_mode_mask[CS_ARCH_XCORE] = ~CS_MODE_BIG_ENDIAN;
 
 	// support this arch
 	all_arch |= (1 << CS_ARCH_XCORE);

cs.c

@@ -54,6 +54,7 @@
 cs_err (*arch_init[MAX_ARCH])(cs_struct *) = { NULL };
 cs_err (*arch_option[MAX_ARCH]) (cs_struct *, cs_opt_type, size_t value) = { NULL };
 void (*arch_destroy[MAX_ARCH]) (cs_struct *) = { NULL };
+cs_mode arch_disallowed_mode_mask[MAX_ARCH] = { 0 };
 
 extern void ARM_enable(void);
 extern void AArch64_enable(void);
@@ -244,6 +245,12 @@ cs_err CAPSTONE_API cs_open(cs_arch arch, cs_mode mode, csh *handle)
 	archs_enable();
 
 	if (arch < CS_ARCH_MAX && arch_init[arch]) {
+		// verify if requested mode is valid
+		if (mode & arch_disallowed_mode_mask[arch]) {
+			*handle = 0;
+			return CS_ERR_MODE;
+		}
+
 		ud = cs_mem_calloc(1, sizeof(*ud));
 		if (!ud) {
 			// memory insufficient
@@ -253,7 +260,6 @@ cs_err CAPSTONE_API cs_open(cs_arch arch, cs_mode mode, csh *handle)
 		ud->errnum = CS_ERR_OK;
 		ud->arch = arch;
 		ud->mode = mode;
-		ud->big_endian = (mode & CS_MODE_BIG_ENDIAN) != 0;
 		// by default, do not break instruction into details
 		ud->detail = CS_OPT_OFF;
 
@@ -429,6 +435,12 @@ cs_err CAPSTONE_API cs_option(csh ud, cs_opt_type type, size_t value)
 			if (value)
 				handle->skipdata_setup = *((cs_opt_skipdata *)value);
 			return CS_ERR_OK;
+		case CS_OPT_MODE:
+			// verify if requested mode is valid
+			if (value & arch_disallowed_mode_mask[handle->arch]) {
+				return CS_ERR_OPTION;
+			}
+			break;
 	}
 
 	return arch_option[handle->arch](handle, type, value);

cs_priv.h

@@ -37,7 +37,6 @@ struct cs_struct {
 	void *printer_info; // aux info for printer
 	Disasm_t disasm;	// disassembler
 	void *getinsn_info; // auxiliary info for printer
-	bool big_endian;
 	GetName_t reg_name;
 	GetName_t insn_name;
 	GetName_t group_name;
@@ -58,6 +57,9 @@ struct cs_struct {
 
 #define MAX_ARCH 8
 
+// Returns a bool (0 or 1) whether big endian is enabled for a mode
+#define MODE_IS_BIG_ENDIAN(mode) (((mode) & CS_MODE_BIG_ENDIAN) != 0)
+
 // constructor initialization for all archs
 extern cs_err (*arch_init[MAX_ARCH]) (cs_struct *);
 
@@ -67,6 +69,10 @@ extern cs_err (*arch_option[MAX_ARCH]) (cs_struct*, cs_opt_type, size_t value);
 // deinitialized functions: to be called when cs_close() is called
 extern void (*arch_destroy[MAX_ARCH]) (cs_struct*);
 
+// bitmask for finding disallowed modes for an arch:
+// to be called in cs_open()/cs_option()
+extern cs_mode arch_disallowed_mode_mask[MAX_ARCH];
+
 extern unsigned int all_arch;
 
 extern cs_malloc_t cs_mem_malloc;