diff --git a/.github/workflows/run_mypy.yaml b/.github/workflows/run_mypy.yaml index fc833fe5..e126f78f 100644 --- a/.github/workflows/run_mypy.yaml +++ b/.github/workflows/run_mypy.yaml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: run_mypy: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v1 diff --git a/chb/app/AppResultFunctionMetrics.py b/chb/app/AppResultFunctionMetrics.py index a83ad60f..a7002b6e 100644 --- a/chb/app/AppResultFunctionMetrics.py +++ b/chb/app/AppResultFunctionMetrics.py @@ -237,6 +237,7 @@ def metrics_to_string( self, shownocallees: bool = False, space: str = " ", + hide: List[str] = [], tags: List[str] = [], taglen: int = 0) -> str: callcount = '' @@ -251,13 +252,25 @@ def metrics_to_string( if self.unresolved_call_count > 0: unrc = str(self.unresolved_call_count) if len(tags) > 0: - ftags = (" [" + ",".join(tags) + "]").ljust(taglen) - - return (str(self.faddr).ljust(10) + space - + '{:6.1f}'.format(self.espp) + space - + '{:6.1f}'.format(self.readsp) + space - + '{:6.1f}'.format(self.writesp) + space - + unrc.rjust(6) + space - + str(self.block_count).rjust(6) + space - + str(self.instruction_count).rjust(6) + space - + '{:8.3f}'.format(self.time) + ftags + name + callcount) + ftags = space + ("[" + ",".join(tags) + "]").ljust(taglen) + espp = "" if "esp" in hide else space + '{:6.1f}'.format(self.espp) + readsp = "" if "reads" in hide else space + '{:6.1f}'.format(self.readsp) + writesp = ("" if "writes" in hide + else space + '{:6.1f}'.format(self.writesp)) + unrcp = ("" if "unrc" in hide else space + unrc.rjust(6)) + blocksp = ("" if "blocks" in hide + else space + str(self.block_count).rjust(6)) + instrsp = ("" if "instrs" in hide + else space + str(self.instruction_count).rjust(6)) + timep = "" if "time" in hide else space + '{:8.3f}'.format(self.time) + return (str(self.faddr).ljust(10) + + espp + + readsp + + writesp + + unrcp + + blocksp + + instrsp + + timep + + ftags + + name + + callcount) diff --git a/chb/app/AppResultMetrics.py b/chb/app/AppResultMetrics.py index eb43d355..4d62c7f0 100644 --- a/chb/app/AppResultMetrics.py +++ b/chb/app/AppResultMetrics.py @@ -518,24 +518,24 @@ def analysis_to_string(self) -> str: lines.append('-' * 80) return '\n'.join(lines) - def header_to_string(self, space: str = " ") -> str: + def header_to_string(self, space: str = " ", hide: List[str] = []) -> str: lines: List[str] = [] lines.append('-' * 80) + h_esp = "" if "esp" in hide else space + "esp".center(6) + h_reads = "" if "reads" in hide else space + "reads".center(6) + h_writes = "" if "writes" in hide else space + "writes".center(6) + h_unrc = "" if "unrc" in hide else space + "unrc".center(6) + h_blocks = "" if "blocks" in hide else space + "blocks".center(6) + h_instrs = "" if "instrs" in hide else space + "instrs".center(6) + h_time = "" if "time" in hide else space + "time".center(8) lines.append( - 'function ' - + space - + 'esp'.center(6) - + space - + 'reads'.center(6) - + space - + 'writes'.center(6) - + space - + 'unrc'.center(6) - + space - + 'blocks'.center(6) - + space - + 'instrs'.center(6) - + space - + 'time'.center(8)) + "function " + + h_esp + + h_reads + + h_writes + + h_unrc + + h_blocks + + h_instrs + + h_time) lines.append('-' * 80) return '\n'.join(lines) diff --git a/chb/app/CHVersion.py b/chb/app/CHVersion.py index 644f53e5..d0169546 100644 --- a/chb/app/CHVersion.py +++ b/chb/app/CHVersion.py @@ -1 +1 @@ -chbversion: str = "0.3.0-20250401" +chbversion: str = "0.3.0-20250404" diff --git a/chb/arm/ARMInstruction.py b/chb/arm/ARMInstruction.py index 60806350..e48ff8b4 100644 --- a/chb/arm/ARMInstruction.py +++ b/chb/arm/ARMInstruction.py @@ -106,7 +106,7 @@ def xdata(self) -> InstrXData: @property def mnemonic_stem(self) -> str: - return self.opcode.mnemonic + return self.opcode.mnemonic_stem @property def mnemonic(self) -> str: diff --git a/chb/arm/ARMOpcode.py b/chb/arm/ARMOpcode.py index 5e3b98ff..60ef415b 100644 --- a/chb/arm/ARMOpcode.py +++ b/chb/arm/ARMOpcode.py @@ -241,6 +241,10 @@ def __init__( def mnemonic(self) -> str: return self.tags[0] + @property + def mnemonic_stem(self) -> str: + return self.mnemonic + def annotation(self, xdata: InstrXData) -> str: return self.__str__() diff --git a/chb/arm/opcodes/ARMBitFieldClear.py b/chb/arm/opcodes/ARMBitFieldClear.py index 3a32fa12..5b3fa22f 100644 --- a/chb/arm/opcodes/ARMBitFieldClear.py +++ b/chb/arm/opcodes/ARMBitFieldClear.py @@ -4,7 +4,7 @@ # ------------------------------------------------------------------------------ # The MIT License (MIT) # -# Copyright (c) 2021-2023 Aarno Labs LLC +# Copyright (c) 2021-2025 Aarno Labs LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -30,7 +30,7 @@ from chb.app.InstrXData import InstrXData from chb.arm.ARMDictionaryRecord import armregistry -from chb.arm.ARMOpcode import ARMOpcode, simplify_result +from chb.arm.ARMOpcode import ARMOpcode, ARMOpcodeXData, simplify_result from chb.arm.ARMOperand import ARMOperand import chb.arm.ARMPseudoCode as APC @@ -41,11 +41,34 @@ import chb.invariants.XXprUtil as XU import chb.util.fileutil as UF - from chb.util.IndexedTable import IndexedTableValue +from chb.util.loggingutil import chklogger if TYPE_CHECKING: from chb.arm.ARMDictionary import ARMDictionary + from chb.invariants.XVariable import XVariable + from chb.invariants.XXpr import XXpr + + +class ARMBitFieldClearXData(ARMOpcodeXData): + """Data format: + - variables: + 0: vrd + + - expressions: + 0: xrd + """ + + def __init__(self, xdata: InstrXData) -> None: + ARMOpcodeXData.__init__(self, xdata) + + @property + def vrd(self) -> "XVariable": + return self.var(0, "vrd") + + @property + def xrd(self) -> "XXpr": + return self.xpr(0, "xrd") @armregistry.register_tag("BFC", ARMOpcode) @@ -60,10 +83,8 @@ class ARMBitFieldClear(ARMOpcode): args[2]: width args[3]: msb position - xdata format: a:vxrdh - --------------------- - vars[0]: lhs (Rd) - xprs[0]: rhs (Rd) + xdata format + ------------ rdefs[0]: rhs uses[0]: lhs useshigh[0]: lhs @@ -102,8 +123,9 @@ def width(self) -> int: return self.args[2] def annotation(self, xdata: InstrXData) -> str: - lhs = str(xdata.vars[0]) - rhs = str(xdata.xprs[0]) + xd = ARMBitFieldClearXData(xdata) + lhs = str(xd.vrd) + rhs = str(xd.xrd) assignment = ( lhs + " := bit-field-clear(" @@ -112,6 +134,8 @@ def annotation(self, xdata: InstrXData) -> str: + str(self.lsb) + ", " + str(self.width) + ")") + return xd.add_instruction_condition(assignment) + ''' if xdata.has_unknown_instruction_condition(): return "if ? then " + assignment elif xdata.has_instruction_condition(): @@ -119,6 +143,7 @@ def annotation(self, xdata: InstrXData) -> str: return "if " + c + " then " + assignment else: return assignment + ''' def ast_prov( self, @@ -149,6 +174,22 @@ def ast_prov( bytestring=bytestring, annotations=annotations) + rdefs = xdata.reachingdefs + astree.add_expr_reachingdefs(ll_op1, [rdefs[0]]) + + # high-level assignment + + xd = ARMBitFieldClearXData(xdata) + + if not xd.is_ok: + chklogger.logger.error( + "BFC: Encountered error value for rhs at address %s", iaddr) + return ([], [ll_assign]) + + lhs = xd.vrd + rhsop = xd.xrd + + ''' lhsasts = XU.xvariable_to_ast_lvals(lhs, xdata, astree) if len(lhsasts) == 0: raise UF.CHBError("BitFieldClear (BFC): no lval found") @@ -170,8 +211,11 @@ def ast_prov( + ", ".join(str(v) for v in rhsasts)) hl_rhs1 = rhsasts[0] + ''' - hl_rhs = astree.mk_binary_op("band", hl_rhs1, maskconst) + hl_lhs = XU.xvariable_to_ast_lval(lhs, xdata, iaddr, astree) + hl_rhsop = XU.xxpr_to_ast_def_expr(rhsop, xdata, iaddr, astree) + hl_rhs = astree.mk_binary_op("band", hl_rhsop, maskconst) hl_assign = astree.mk_assign( hl_lhs, @@ -184,7 +228,7 @@ def ast_prov( astree.add_instr_mapping(hl_assign, ll_assign) astree.add_instr_address(hl_assign, [iaddr]) astree.add_expr_mapping(hl_rhs, ll_rhs) - astree.add_expr_mapping(hl_rhs1, ll_op1) + astree.add_expr_mapping(hl_rhsop, ll_op1) astree.add_lval_mapping(hl_lhs, ll_lhs) astree.add_expr_reachingdefs(ll_rhs, [rdefs[0]]) astree.add_expr_reachingdefs(ll_op1, [rdefs[0]]) diff --git a/chb/arm/opcodes/ARMLoadRegisterByte.py b/chb/arm/opcodes/ARMLoadRegisterByte.py index c4eeb59b..9d11095d 100644 --- a/chb/arm/opcodes/ARMLoadRegisterByte.py +++ b/chb/arm/opcodes/ARMLoadRegisterByte.py @@ -53,6 +53,21 @@ class ARMLoadRegisterByteXData(ARMOpcodeXData): + """Data format: + - variables: + 0: vrt (lhs) + + - expressions: + 0: xrn + 1: xrm + 2: xmem (rhs) + 3: xrmem (rhs, rewritten) + 4: xaddr (address of rhs) + + - c expressions: + 0: cxrmem (rhs) + 1: cxaddr (address of rhs) + """ def __init__(self, xdata: InstrXData) -> None: ARMOpcodeXData.__init__(self, xdata) @@ -61,10 +76,6 @@ def __init__(self, xdata: InstrXData) -> None: def vrt(self) -> "XVariable": return self.var(0, "vrt") - @property - def vmem(self) -> "XVariable": - return self.var(1, "vmem") - @property def xrn(self) -> "XXpr": return self.xpr(0, "xrn") @@ -77,29 +88,68 @@ def xrm(self) -> "XXpr": def xmem(self) -> "XXpr": return self.xpr(2, "xmem") + @property + def is_xmem_ok(self) -> bool: + return self.is_xpr_ok(2) + @property def xrmem(self) -> "XXpr": return self.xpr(3, "xrmem") @property - def is_xrmem_unknown(self) -> bool: - return self.xdata.xprs_r[3] is None + def is_xrmem_ok(self) -> bool: + return self.is_xpr_ok(3) + + @property + def cxrmem(self) -> "XXpr": + return self.cxpr(0, "cxrmem") @property - def is_address_known(self) -> bool: - return self.xdata.xprs_r[4] is not None + def is_cxrmem_ok(self) -> bool: + return self.is_cxpr_ok(0) @property def xaddr(self) -> "XXpr": return self.xpr(4, "xaddr") + @property + def is_xaddr_ok(self) -> bool: + return self.is_xpr_ok(4) + + @property + def xxaddr(self) -> "XXpr": + return self.xpr(5, "xxaddr") + + @property + def is_xxaddr_ok(self) -> bool: + return self.is_xpr_ok(5) + + @property + def cxaddr(self) -> "XXpr": + return self.cxpr(1, "cxaddr") + + @property + def is_cxaddr_ok(self) -> bool: + return self.is_cxpr_ok(1) + @property def annotation(self) -> str: wbu = self.writeback_update() - if self.is_ok: - assignment = str(self.vrt) + " := " + str(self.xrmem) - elif self.is_xrmem_unknown and self.is_address_known: - assignment = str(self.vrt) + " := *(" + str(self.xaddr) + ")" + if self.is_cxrmem_ok: + crhs = str(self.cxrmem) + elif self.is_cxaddr_ok: + crhs = "*(" + str(self.cxaddr) + ")" + else: + crhs = "None" + cx = " (C: " + crhs + ")" + addr = str(self.xxaddr if self.is_xxaddr_ok else self.xaddr) + caddr = str(self.cxaddr if self.is_cxaddr_ok else "None") + caddr = " (addr: " + addr + "; C: " + caddr + ")" + if self.is_ok or self.is_xrmem_ok: + assignment = str(self.vrt) + " := " + str(self.xrmem) + cx + caddr + elif self.is_xaddr_ok: + assignment = ( + str(self.vrt) + " := *(" + str(self.xaddr) + ")" + cx + caddr) else: assignment = "Error value" return self.add_instruction_condition(assignment) + wbu @@ -118,30 +168,14 @@ class ARMLoadRegisterByte(ARMOpcode): args[3]: index of memory location in armdictionary args[4]: is-wide (thumb) - xdata format: a:vxxxxrrrdh - -------------------------- - vars[0]: lhs - vars[1]: memory location expressed as a variable - xprs[0]: value in rn - xprs[1]: value in rm - xprs[2]: value in memory location - xprs[3]: value in memory location (simplified) - xprs[4]: address of memory location + xdata format: + ------------- rdefs[0]: reaching definitions rn rdefs[1]: reaching definitions rm rdefs[2]: reaching definitions memory location rdefs[3..]: reaching definitions for memory value uses[0]: use of lhs useshigh[0]: use of lhs at high level - - optional: - vars[2]: lhs base register (if base update) - - xprs[.]: instruction condition (if has condition) - xprs[.]: new address for base register - - uses[1]: use of updated base register - useshigh[1]: use of updated base register """ def __init__( @@ -165,13 +199,20 @@ def opargs(self) -> List[ARMOperand]: return [self.armd.arm_operand(self.args[i]) for i in [0, 1, 2, 3]] def lhs(self, xdata: InstrXData) -> List[XVariable]: - return [xdata.vars[0]] + xd = ARMLoadRegisterByteXData(xdata) + return [xd.vrt] def is_load_instruction(self, xdata: InstrXData) -> bool: return True def rhs(self, xdata: InstrXData) -> List[XXpr]: - return [xdata.xprs[1]] + xd = ARMLoadRegisterByteXData(xdata) + if xd.is_xrmem_ok: + return [xd.xrmem] + elif xd.is_xmem_ok: + return [xd.xmem] + else: + return [] def annotation(self, xdata: InstrXData) -> str: """lhs, rhs, with optional instr condition and base update.""" @@ -188,9 +229,7 @@ def ast_prov( xd = ARMLoadRegisterByteXData(xdata) - memaddr = xd.xaddr - - annotations: List[str] = [iaddr, "LDRB", "addr:" + str(memaddr)] + annotations: List[str] = [iaddr, "LDRB"] # low-level assignment @@ -208,26 +247,42 @@ def ast_prov( # high-level assignment + def has_cast() -> bool: + return ( + astree.has_register_variable_intro(iaddr) + and astree.get_register_variable_intro(iaddr).has_cast()) + lhs = xd.vrt if xd.is_ok: - rhs = xd.xrmem - xaddr = xd.xaddr - hl_lhs = XU.xvariable_to_ast_lval(lhs, xdata, iaddr, astree, rhs=rhs) - hl_rhs = XU.xxpr_to_ast_def_expr( - rhs, xdata, iaddr, astree, size=1, memaddr=xaddr) + rhs = xd.cxrmem + rhsval = None if has_cast() else xd.cxrmem + hl_lhs = XU.xvariable_to_ast_lval( + lhs, xdata, iaddr, astree, rhs=rhsval) + hl_rhs = XU.xxpr_to_ast_def_expr(rhs, xdata, iaddr, astree) + + elif xd.is_cxaddr_ok: + cxaddr = xd.cxaddr + hl_lhs = XU.xvariable_to_ast_lval(lhs, xdata, iaddr, astree) + hl_rhs = XU.xmemory_dereference_lval_expr( + cxaddr, xdata, iaddr, astree) - elif xd.is_xrmem_unknown and xd.is_address_known: + elif xd.is_xaddr_ok: xaddr = xd.xaddr hl_lhs = XU.xvariable_to_ast_lval(lhs, xdata, iaddr, astree) hl_rhs = XU.xmemory_dereference_lval_expr( - xaddr, xdata, iaddr, astree, size=1) + xaddr, xdata, iaddr, astree) + + chklogger.logger.warning( + "LDRB: Unable to use a C expression for rhs. Fall back to " + + "native byte-based address: %s to form rhs %s at address %s", + str(xaddr), str(hl_rhs), iaddr) else: chklogger.logger.error( "LDRB: both memory value and address values are error values " + "at address %s: ", iaddr) - return ([], []) + return ([], (ll_pre + [ll_assign] + ll_post)) rdefs = xdata.reachingdefs defuses = xdata.defuses @@ -240,6 +295,9 @@ def ast_prov( bytestring=bytestring, annotations=annotations) + if has_cast(): + astree.add_expose_instruction(hl_assign.instrid) + astree.add_instr_mapping(hl_assign, ll_assign) astree.add_instr_address(hl_assign, [iaddr]) astree.add_expr_mapping(hl_rhs, ll_rhs) diff --git a/chb/arm/opcodes/ARMLogicalShiftLeft.py b/chb/arm/opcodes/ARMLogicalShiftLeft.py index 397f3777..1bb5ef1a 100644 --- a/chb/arm/opcodes/ARMLogicalShiftLeft.py +++ b/chb/arm/opcodes/ARMLogicalShiftLeft.py @@ -135,6 +135,10 @@ def mnemonic_extension(self) -> str: def opargs(self) -> List[ARMOperand]: return [self.armd.arm_operand(i) for i in self.args[1:-1]] + @property + def mnemonic_stem(self) -> str: + return self.tags[0] + @property def mnemonic(self) -> str: mnem = self.tags[0] diff --git a/chb/arm/opcodes/ARMLogicalShiftRight.py b/chb/arm/opcodes/ARMLogicalShiftRight.py index 5295bd30..c59a8b5f 100644 --- a/chb/arm/opcodes/ARMLogicalShiftRight.py +++ b/chb/arm/opcodes/ARMLogicalShiftRight.py @@ -127,6 +127,10 @@ def operands(self) -> List[ARMOperand]: def opargs(self) -> List[ARMOperand]: return [self.armd.arm_operand(i) for i in self.args[1:-1]] + @property + def mnemonic_stem(self) -> str: + return self.tags[0] + @property def mnemonic(self) -> str: mnem = self.tags[0] diff --git a/chb/arm/opcodes/ARMMove.py b/chb/arm/opcodes/ARMMove.py index 5423288f..df3c3e9e 100644 --- a/chb/arm/opcodes/ARMMove.py +++ b/chb/arm/opcodes/ARMMove.py @@ -124,6 +124,10 @@ def __init__(self, d: "ARMDictionary", ixval: IndexedTableValue) -> None: ARMOpcode.__init__(self, d, ixval) self.check_key(2, 5, "Move") + @property + def mnemonic_stem(self) -> str: + return self.tags[0] + @property def mnemonic(self) -> str: return ( diff --git a/chb/arm/opcodes/ARMPop.py b/chb/arm/opcodes/ARMPop.py index 770fcec6..f488c1ce 100644 --- a/chb/arm/opcodes/ARMPop.py +++ b/chb/arm/opcodes/ARMPop.py @@ -99,6 +99,10 @@ def rspresult(self) -> "XXpr": def rrhsexprs(self) -> List["XXpr"]: return [self.xpr(i, "rhsexpr") for i in range(3, self.regcount + 3)] + @property + def are_rrhsexprs_ok(self) -> bool: + return all(self.is_xpr_ok(i) for i in range(3, self.regcount + 3)) + @property def xaddrs(self) -> List["XXpr"]: return [self.xpr(i, "xaddr") @@ -127,18 +131,22 @@ def r0(self) -> Optional["XXpr"]: @property def annotation(self) -> str: - pairs = zip(self.lhsvars, self.rrhsexprs) - spassign = str(self.splhs) + " := " + str(self.rspresult) - assigns = "; ".join(str(v) + " := " + str(x) for (v, x) in pairs) - assigns = spassign + "; " + assigns - if self.has_return_xpr(): - cxpr = ( - " (C: " - + (str(self.creturnval()) if self.has_creturnval() else "None") - + ")") - rxpr = "; return " + str(self.rreturnval()) + cxpr + if self.are_rrhsexprs_ok: + pairs = zip(self.lhsvars, self.rrhsexprs) + spassign = str(self.splhs) + " := " + str(self.rspresult) + assigns = "; ".join(str(v) + " := " + str(x) for (v, x) in pairs) + assigns = spassign + "; " + assigns + if self.has_return_xpr(): + cxpr = ( + " (C: " + + (str(self.creturnval()) if self.has_creturnval() else "None") + + ")") + rxpr = "; return " + str(self.rreturnval()) + cxpr + else: + rxpr = "" else: - rxpr = "" + assigns = "rhs error value" + rxpr = "?" return self.add_instruction_condition(assigns + rxpr) diff --git a/chb/arm/opcodes/ARMReverseSubtract.py b/chb/arm/opcodes/ARMReverseSubtract.py index 8ed30064..2d8e600b 100644 --- a/chb/arm/opcodes/ARMReverseSubtract.py +++ b/chb/arm/opcodes/ARMReverseSubtract.py @@ -158,6 +158,10 @@ def operands(self) -> List[ARMOperand]: def opargs(self) -> List[ARMOperand]: return [self.armd.arm_operand(i) for i in self.args[1: -1]] + @property + def mnemonic_stem(self) -> str: + return self.tags[0] + @property def mnemonic(self) -> str: mnem = self.tags[0] diff --git a/chb/arm/opcodes/ARMSwapByte.py b/chb/arm/opcodes/ARMSwapByte.py index 35ea03bf..72611642 100644 --- a/chb/arm/opcodes/ARMSwapByte.py +++ b/chb/arm/opcodes/ARMSwapByte.py @@ -56,7 +56,7 @@ def __init__( d: "ARMDictionary", ixval: IndexedTableValue) -> None: ARMOpcode.__init__(self, d, ixval) - self.check_key(2, 3, "SwapByte") + self.check_key(2, 4, "SwapByte") @property def operands(self) -> List[ARMOperand]: diff --git a/chb/arm/opcodes/opcodes_covered.json b/chb/arm/opcodes/opcodes_covered.json index 39f680af..dc2edd79 100644 --- a/chb/arm/opcodes/opcodes_covered.json +++ b/chb/arm/opcodes/opcodes_covered.json @@ -1,6 +1,7 @@ { "legend": { "AST": "ast support", + "ASTC": "support for lifting to C code", "D": "disassembled", "DA": "destination abstracted", "E": "encoding in bits", @@ -162,6 +163,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -195,6 +197,7 @@ } ] }, + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -314,6 +317,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -357,6 +361,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -397,6 +402,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "BFC": { @@ -477,6 +483,7 @@ ] }, "A": "Y", + "ASTC": "Y", "TC": "DA" }, "BKPT": { @@ -511,6 +518,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -545,6 +553,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -565,6 +574,7 @@ } ] }, + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -710,6 +720,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -794,6 +805,7 @@ } ] }, + "ASTC": "Y", "TC": "DA" }, "FLDMIAX": { @@ -926,6 +938,7 @@ } ] }, + "ASTC": "Y", "TC": "Y" }, "LDMDA": { @@ -1038,6 +1051,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "LDRB": { @@ -1084,6 +1098,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y", "A": "Y" }, @@ -1186,6 +1201,7 @@ ] }, "AST": "Y", + "ASTC":" Y", "TC": "Y" }, "LDRSB": { @@ -1464,6 +1480,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "MOVT": { @@ -1588,6 +1605,7 @@ } ] }, + "ASTC": "Y", "TC": "Y" }, "MVN": { @@ -1723,6 +1741,7 @@ } ] }, + "ASTC": "Y", "TC": "DA", "A": "Y" }, @@ -1805,6 +1824,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "PUSH": { @@ -2067,6 +2087,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "RSC": { @@ -2707,6 +2728,7 @@ } ] }, + "ASTC": "Y", "TC": "Y" }, "STMDA": { @@ -2809,6 +2831,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "STRB": { @@ -2855,6 +2878,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "STRD": { @@ -2949,6 +2973,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "SUB": { @@ -3028,6 +3053,7 @@ ] }, "AST": "Y", + "ASTC": "Y", "TC": "Y" }, "SVC": { diff --git a/chb/ast/ASTCPrettyPrinter.py b/chb/ast/ASTCPrettyPrinter.py index 8fe5f643..bec70ba3 100644 --- a/chb/ast/ASTCPrettyPrinter.py +++ b/chb/ast/ASTCPrettyPrinter.py @@ -495,8 +495,21 @@ def visit_variable(self, var: AST.ASTVariable) -> None: def visit_memref(self, memref: AST.ASTMemRef) -> None: if memref.memexp.is_ast_addressof: - memexp = cast(AST.ASTAddressOf, memref.memexp) - memexp.lval.accept(self) + memexpa = cast(AST.ASTAddressOf, memref.memexp) + memexpa.lval.accept(self) + elif memref.memexp.is_ast_binary_op: + memexpb = cast(AST.ASTBinaryOp, memref.memexp) + exp1 = memexpb.exp1 + exp2 = memexpb.exp2 + if exp1.is_ast_lval_expr: + exp1.accept(self) + self.ccode.write("[") + exp2.accept(self) + self.ccode.write("]") + else: + self.ccode.write("(*(") + memref.memexp.accept(self) + self.ccode.write("))") else: self.ccode.write("(*(") memref.memexp.accept(self) diff --git a/chb/cmdline/chkx b/chb/cmdline/chkx index a9fa4c38..ec4507e6 100755 --- a/chb/cmdline/chkx +++ b/chb/cmdline/chkx @@ -376,8 +376,7 @@ def parse() -> argparse.Namespace: help="output intermediate information") analyzecmd.add_argument( "--save_asm", - default="yes", - choices=["yes", "no"], + action="store_true", help='save asm listing in analysis directory') analyzecmd.add_argument( "--save_asm_cfg_info", @@ -615,6 +614,12 @@ def parse() -> argparse.Namespace: resultsstats.add_argument( "--tagfile", help="name of json file that has function tags") + resultsstats.add_argument( + "--hide", + help="name(s) of columns to hide", + nargs="*", + choices=["time", "esp","reads","writes", "unrc", "blocks", "instrs"], + default=[]) resultsstats.add_argument( "--loglevel", "-log", choices=UL.LogLevel.options(), @@ -1356,6 +1361,10 @@ def parse() -> argparse.Namespace: "--save_asm", action="store_true", help="save the assembly file") + relationalprepare.add_argument( + "--collect_diagnostics", + action="store_true", + help="save diagnostic information in diagnostics log") relationalprepare.add_argument( "--hints", nargs="*", diff --git a/chb/cmdline/commandutil.py b/chb/cmdline/commandutil.py index 8ff2d11c..e179cb1e 100644 --- a/chb/cmdline/commandutil.py +++ b/chb/cmdline/commandutil.py @@ -396,7 +396,7 @@ def analyzecmd(args: argparse.Namespace) -> NoReturn: doextract: bool = args.extract verbose: bool = args.verbose collectdiagnostics: bool = args.collect_diagnostics - save_asm: str = args.save_asm + save_asm: bool = args.save_asm save_asm_cfg_info: bool = args.save_asm_cfg_info thumb: List[str] = args.thumb preamble_cutoff: int = args.preamble_cutoff @@ -549,7 +549,6 @@ def analyzecmd(args: argparse.Namespace) -> NoReturn: verbose=verbose, collectdiagnostics=collectdiagnostics, preamble_cutoff=preamble_cutoff, - save_asm=save_asm, save_asm_cfg_info=save_asm_cfg_info) except subprocess.CalledProcessError as e: print_error(str(e.output)) @@ -584,6 +583,7 @@ def analyzecmd(args: argparse.Namespace) -> NoReturn: analysisrepeats=analysisrepeats, iterations=iterations, verbose=verbose, + save_asm=save_asm, construct_all_functions=construct_all_functions, collectdiagnostics=collectdiagnostics, preamble_cutoff=preamble_cutoff) @@ -619,6 +619,7 @@ def results_stats(args: argparse.Namespace) -> NoReturn: sortby: str = args.sortby timeshare: int = args.timeshare opcodes: str = args.opcodes + hide: List[str] = args.hide tagfile: Optional[str] = args.tagfile loglevel: str = args.loglevel logfilename: Optional[str] = args.logfilename @@ -661,7 +662,7 @@ def results_stats(args: argparse.Namespace) -> NoReturn: app = get_app(path, xfile, xinfo) stats = app.result_metrics - print(stats.header_to_string()) + print(stats.header_to_string(hide=hide)) if sortby == "instrs": sortkey = lambda f: f.instruction_count elif sortby == "basicblocks": @@ -679,7 +680,7 @@ def results_stats(args: argparse.Namespace) -> NoReturn: fn_tags = [] if "hide" in fn_tags: continue - print(f.metrics_to_string(shownocallees=nocallees, + print(f.metrics_to_string(shownocallees=nocallees, hide=hide, tags=fn_tags, taglen=maxlen)) print(stats.disassembly_to_string()) @@ -697,10 +698,39 @@ def results_stats(args: argparse.Namespace) -> NoReturn: if opcodes: filename = opcodes + ".json" + opcstats = app.mnemonic_stats() + if xinfo.is_arm: + opccovered = Config().armopcodes + with open(opccovered, "r") as fp: + opcsupport: Dict[str, Any] = json.load(fp)["instructions"] + else: + opcsupport = {} + result: Dict[str, Any] = {} + result["name"] = xname + result["md5"] = xinfo.md5 + result["opcodes"] = {} + for (opc, count) in sorted(opcstats.items()): + opcrec: Dict[str, Any] = {} + opcrec["count"] = count + opcrec["support"] = [] + if opc in opcsupport: + if "ASTC" in opcsupport[opc]: + if opcsupport[opc]["ASTC"] == "Y": + opcrec["support"].append("ASTC") + result["opcodes"][opc] = opcrec with open(filename, "w") as fp: - json.dump(app.mnemonic_stats(), fp, sort_keys=True, indent=2) + json.dump(result, fp, sort_keys=True, indent=2) chklogger.logger.info("opcodes saved to " + filename) - + print("\nOpcode stats") + print("-" * 80) + for (opc, opcr) in sorted(result["opcodes"].items()): + p_opcsupport = "" + if len(opcr["support"]) > 0: + p_opcsupport = "(" + ", ".join(opcr["support"]) + ")" + print(opc.ljust(10) + ": " + + str(opcr["count"]).rjust(6) + + " " + p_opcsupport) + print("=" * 80) chklogger.logger.info("results stats completed") exit(0) diff --git a/chb/cmdline/relationalcmds.py b/chb/cmdline/relationalcmds.py index dec0b38d..5b262764 100644 --- a/chb/cmdline/relationalcmds.py +++ b/chb/cmdline/relationalcmds.py @@ -153,6 +153,7 @@ def relational_prepare_command(args: argparse.Namespace) -> NoReturn: xpatchresults: Optional[str] = args.patch_results_file xprint: bool = not args.json xssa: bool = args.ssa + collectdiagnostics = args.collect_diagnostics xconstruct_all_functions: bool = args.construct_all_functions loglevel: str = args.loglevel logfilename: Optional[str] = args.logfilename @@ -241,6 +242,9 @@ def relational_prepare_command(args: argparse.Namespace) -> NoReturn: lines.append(" - New code inserted in the following memory regions:") for (x, y) in xcomparison.newcode: lines.append(" * From " + x + " to " + y) + if len(xcomparison.messages) > 0: + lines.append("\nMessages:") + lines.append("\n".join(xcomparison.messages)) lines.append("=" * 80) lines.append( "||" + (str(datetime.datetime.now()) + " ").rjust(76) + "||") @@ -300,12 +304,13 @@ def relational_prepare_command(args: argparse.Namespace) -> NoReturn: fns_include=fns_include, fns_exclude=fns_exclude, use_ssa=xssa, - thumb=True) + thumb=xcomparison.is_thumb) try: am.analyze( iterations=10, save_asm=xsave_asm, + collectdiagnostics=collectdiagnostics, construct_all_functions=xconstruct_all_functions) except subprocess.CalledProcessError as e: print(e.output) diff --git a/chb/invariants/XXprUtil.py b/chb/invariants/XXprUtil.py index b899f93b..37e7d824 100644 --- a/chb/invariants/XXprUtil.py +++ b/chb/invariants/XXprUtil.py @@ -468,13 +468,9 @@ def memory_variable_to_lval_expression( offset = cast("VMemoryOffsetBasePtrArrayIndexOffset", offset) (ptroffset, astoffset) = base_ptr_array_offset_to_ast_offset( offset, xdata, iaddr, astree, anonymous=anonymous) - if ptroffset.is_integer_constant_zero: - return astree.mk_memref_expr( - astlval, offset=astoffset, anonymous=anonymous) - else: - ptrexpr = astree.mk_binary_op("plus", ptroffset, astlval) - return astree.mk_memref_expr( - ptrexpr, offset=astoffset, anonymous=anonymous) + ptrexpr = astree.mk_binary_op("plus", astlval, ptroffset) + return astree.mk_memref_expr( + ptrexpr, offset=astoffset, anonymous=anonymous) name = str(base) @@ -1664,10 +1660,17 @@ def global_variable_to_ast_lval( aindexoffset, xdata, iaddr, astree, anonymous=anonymous) return astree.mk_vinfo_lval(vinfo, astoffset, anonymous=anonymous) + if not anonymous: + chklogger.logger.error( + "Conversion of constant offset %s with suboffset %s at address " + + "%s not yet supported", + str(offset), str(offset.offset), iaddr) + return astree.mk_temp_lval() + if not anonymous: chklogger.logger.error( - ("Conversion of global ast lval for address %s at address %s " - + "not yet supported"), + ("Conversion of global ast lval for address %s " + + "at address %s not yet supported"), str(offset), iaddr) return astree.mk_temp_lval() diff --git a/chb/summaries/bchsummaries.jar b/chb/summaries/bchsummaries.jar index c4743265..b3ef4078 100644 Binary files a/chb/summaries/bchsummaries.jar and b/chb/summaries/bchsummaries.jar differ diff --git a/chb/util/Config.py b/chb/util/Config.py index 39434bc0..35d7153c 100644 --- a/chb/util/Config.py +++ b/chb/util/Config.py @@ -71,6 +71,11 @@ def __init__(self) -> None: self.macOSdir = os.path.join(self.binariesdir, "macOS") self.chx86_analyze = os.path.join(self.macOSdir, "chx86_analyze") + # instruction support + self.armdir = os.path.join(self.chbdir, "arm") + self.armopcdir = os.path.join(self.armdir, "opcodes") + self.armopcodes = os.path.join(self.armopcdir, "opcodes_covered.json") + # registered command-line options self.commandline_options: Dict[str, str] = {}