Enable C LTO for firmware

CMakeLists.txt

@@ -41,6 +41,15 @@ project(bitbox02 C)
 
 # nosys is set in arm.cmake so that `project(c)` above works. Remove it since it interferes with compile options
 if(CMAKE_CROSSCOMPILING)
+  if(NOT CMAKE_C_COMPILER_ID STREQUAL "GNU")
+    message(FATAL_ERROR
+      "Firmware C LTO currently requires GCC. The stack-protector symbols use "
+      "GCC's externally_visible attribute so GCC LTO does not internalize or "
+      "drop __stack_chk_fail/__stack_chk_guard. Clang does not support that "
+      "attribute; add and verify compiler-specific symbol-retention handling "
+      "before enabling firmware C LTO with a non-GNU compiler."
+    )
+  endif()
   string(REPLACE "--specs=nosys.specs" "" CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS})
 endif()
 

arm.cmake

@@ -2,6 +2,9 @@ set(CMAKE_SYSTEM_NAME "Generic")
 set(CMAKE_SYSTEM_PROCESSOR "arm")
 
 set(CMAKE_C_COMPILER "arm-none-eabi-gcc")
+set(CMAKE_AR "arm-none-eabi-gcc-ar")
+set(CMAKE_NM "arm-none-eabi-gcc-nm")
+set(CMAKE_RANLIB "arm-none-eabi-gcc-ranlib")
 
 # Search for programs in the build host directories
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

external/CMakeLists.txt

@@ -176,6 +176,7 @@ set_property(TARGET asf4-drivers PROPERTY INTERFACE_LINK_LIBRARIES "")
     ${CMAKE_CURRENT_SOURCE_DIR} # for the BitBox02-custom "atca_config.h"
     )
   target_compile_options(cryptoauthlib PRIVATE
+    -flto -ffat-lto-objects
     -Wno-pedantic -Wno-incompatible-pointer-types -Wno-unused-parameter -Wno-unused-variable -Wno-cast-qual
     -Wno-switch-default -Wno-format-nonliteral -Wno-missing-prototypes -Wno-missing-declarations
   )
@@ -218,7 +219,7 @@ add_library(optiga EXCLUDE_FROM_ALL
 )
 target_compile_definitions(optiga PRIVATE MBEDTLS_USER_CONFIG_FILE="mbedtls_config.h")
 # Ignore warnings in external lib.
-target_compile_options(optiga PRIVATE "-w")
+target_compile_options(optiga PRIVATE "-w" -flto -ffat-lto-objects)
 target_compile_definitions(optiga PRIVATE OPTIGA_LIB_EXTERNAL="optiga_config.h")
 target_include_directories(optiga SYSTEM PUBLIC
   optiga-trust-m/config

src/CMakeLists.txt

@@ -416,6 +416,7 @@ if(CMAKE_CROSSCOMPILING)
   foreach(bootloader ${BOOTLOADERS})
     set(elf ${bootloader}.elf)
     add_executable(${elf} ${BOOTLOADER-SOURCES} ${PLATFORM-BITBOX02-SOURCES})
+    target_compile_options(${elf} PRIVATE -fno-lto)
     target_link_libraries(${elf} PRIVATE c asf4-drivers-min samd51a-ds -Wl,-u,exception_table)
     target_include_directories(${elf} PRIVATE ${INCLUDES})
     target_compile_definitions(${elf} PRIVATE BOOTLOADER "APP_U2F=0")
@@ -483,6 +484,14 @@ if(CMAKE_CROSSCOMPILING)
   foreach(firmware ${FIRMWARES})
     set(elf ${firmware}.elf)
     add_executable(${elf} ${FIRMWARE-SOURCES})
+    # Static libraries are LTO-capable for firmware size, but only firmware images do the LTO link.
+    if(firmware STREQUAL "factory-setup")
+      target_compile_options(${elf} PRIVATE -fno-lto)
+      target_link_libraries(${elf} PRIVATE -fno-lto)
+    else()
+      target_compile_options(${elf} PRIVATE -flto -ffat-lto-objects)
+      target_link_libraries(${elf} PRIVATE -flto)
+    endif()
     # Must manually link against C so that malloc can find _sbrk
     target_link_libraries(${elf}
       PRIVATE

src/common_main.c

@@ -14,7 +14,9 @@
 #include <rust/rust.h>
 
 extern void __attribute__((noreturn)) __stack_chk_fail(void);
-void __attribute__((noreturn)) __stack_chk_fail(void)
+// GCC LTO needs externally_visible; clang-tidy parses with Clang and does not support it.
+// NOLINTNEXTLINE(clang-diagnostic-unknown-attributes)
+void __attribute__((noreturn, used, externally_visible)) __stack_chk_fail(void)
 {
     Abort("Stack smashing detected");
     while (1) {

src/firmware.c

@@ -21,7 +21,9 @@
     #include <u2f.h>
 #endif
 
-uint32_t __stack_chk_guard = 0;
+// GCC LTO needs externally_visible; clang-tidy parses with Clang and does not support it.
+// NOLINTNEXTLINE(clang-diagnostic-unknown-attributes)
+uint32_t __attribute__((used, externally_visible)) __stack_chk_guard = 0;
 
 int main(void)
 {