Skip to content

Fix C undefined behavior in integer arithmetic across interpreter and compiler #506

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nbeerbower wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Fix C undefined behavior in integer arithmetic across interpreter and compiler #506

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d3e62e2
Fix C undefined behavior in integer arithmetic across interpreter and…
claude Apr 8, 2026
4d613cf
Merge main and resolve conflict in codegen_expr_binary.c
claude Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions runtime/include/hemlock_value.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -617,16 +617,26 @@ static inline void hml_array_set_i32_fast(HmlArray *arr, int32_t index, HmlValue
hml_runtime_error("Array index %d out of bounds (length %d)", index, arr->length);
}

// Fast path: Increment i32 variable in-place
// Fast path: Increment i32 variable in-place (with overflow check)
// Returns the new value
static inline HmlValue hml_i32_inc(HmlValue val) {
return (HmlValue){ .type = HML_VAL_I32, .as.as_i32 = val.as.as_i32 + 1 };
int32_t result;
if (__builtin_add_overflow(val.as.as_i32, 1, &result)) {
extern __attribute__((noreturn)) void hml_runtime_error(const char *fmt, ...);
hml_runtime_error("Integer overflow: i32 increment");
}
return (HmlValue){ .type = HML_VAL_I32, .as.as_i32 = result };
}

// Fast path: Decrement i32 variable in-place
// Fast path: Decrement i32 variable in-place (with overflow check)
// Returns the new value
static inline HmlValue hml_i32_dec(HmlValue val) {
return (HmlValue){ .type = HML_VAL_I32, .as.as_i32 = val.as.as_i32 - 1 };
int32_t result;
if (__builtin_sub_overflow(val.as.as_i32, 1, &result)) {
extern __attribute__((noreturn)) void hml_runtime_error(const char *fmt, ...);
hml_runtime_error("Integer overflow: i32 decrement");
}
return (HmlValue){ .type = HML_VAL_I32, .as.as_i32 = result };
}

#endif // HEMLOCK_VALUE_H
99 changes: 86 additions & 13 deletions runtime/src/builtins_ops.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,30 @@ static inline HmlValueType promote_types(HmlValueType a, HmlValueType b) {
return hml_tk_to_val(result);
}

// Create an integer result value with overflow checking for signed narrowing
HmlValue make_int_result_checked(HmlValueType result_type, int64_t value) {
switch (result_type) {
case HML_VAL_I8:
if (value < INT8_MIN || value > INT8_MAX)
hml_runtime_error("Integer overflow: i8 arithmetic");
return hml_val_i8((int8_t)value);
case HML_VAL_I16:
if (value < INT16_MIN || value > INT16_MAX)
hml_runtime_error("Integer overflow: i16 arithmetic");
return hml_val_i16((int16_t)value);
case HML_VAL_I32:
if (value < INT32_MIN || value > INT32_MAX)
hml_runtime_error("Integer overflow: i32 arithmetic");
return hml_val_i32((int32_t)value);
case HML_VAL_I64: return hml_val_i64(value);
case HML_VAL_U8: return hml_val_u8((uint8_t)value);
case HML_VAL_U16: return hml_val_u16((uint16_t)value);
case HML_VAL_U32: return hml_val_u32((uint32_t)value);
case HML_VAL_U64: return hml_val_u64((uint64_t)value);
default: return hml_val_i64(value);
}
}

// Create an integer result value with the correct type
HmlValue make_int_result(HmlValueType result_type, int64_t value) {
switch (result_type) {
Expand Down Expand Up @@ -119,9 +143,21 @@ HmlValue hml_binary_op(HmlBinaryOp op, HmlValue left, HmlValue right) {
int32_t l = left.as.as_i32;
int32_t r = right.as.as_i32;
switch (op) {
case HML_OP_ADD: return hml_val_i32(l + r);
case HML_OP_SUB: return hml_val_i32(l - r);
case HML_OP_MUL: return hml_val_i32(l * r);
case HML_OP_ADD: {
int32_t result;
if (__builtin_add_overflow(l, r, &result)) hml_runtime_error("Integer overflow: i32 addition");
return hml_val_i32(result);
}
case HML_OP_SUB: {
int32_t result;
if (__builtin_sub_overflow(l, r, &result)) hml_runtime_error("Integer overflow: i32 subtraction");
return hml_val_i32(result);
}
case HML_OP_MUL: {
int32_t result;
if (__builtin_mul_overflow(l, r, &result)) hml_runtime_error("Integer overflow: i32 multiplication");
return hml_val_i32(result);
}
case HML_OP_MOD:
if (r == 0) hml_runtime_error("Division by zero");
return hml_val_i32(l % r);
Expand Down Expand Up @@ -149,9 +185,21 @@ HmlValue hml_binary_op(HmlBinaryOp op, HmlValue left, HmlValue right) {
int64_t l = left.as.as_i64;
int64_t r = right.as.as_i64;
switch (op) {
case HML_OP_ADD: return hml_val_i64(l + r);
case HML_OP_SUB: return hml_val_i64(l - r);
case HML_OP_MUL: return hml_val_i64(l * r);
case HML_OP_ADD: {
int64_t result;
if (__builtin_add_overflow(l, r, &result)) hml_runtime_error("Integer overflow: i64 addition");
return hml_val_i64(result);
}
case HML_OP_SUB: {
int64_t result;
if (__builtin_sub_overflow(l, r, &result)) hml_runtime_error("Integer overflow: i64 subtraction");
return hml_val_i64(result);
}
case HML_OP_MUL: {
int64_t result;
if (__builtin_mul_overflow(l, r, &result)) hml_runtime_error("Integer overflow: i64 multiplication");
return hml_val_i64(result);
}
case HML_OP_DIV:
if (r == 0) hml_runtime_error("Division by zero");
return hml_val_i64(l / r);
Expand Down Expand Up @@ -338,12 +386,24 @@ HmlValue hml_binary_op(HmlBinaryOp op, HmlValue left, HmlValue right) {
int64_t r = hml_to_i64(right);

switch (op) {
case HML_OP_ADD:
return make_int_result(result_type, l + r);
case HML_OP_SUB:
return make_int_result(result_type, l - r);
case HML_OP_MUL:
return make_int_result(result_type, l * r);
case HML_OP_ADD: {
int64_t result;
if (__builtin_add_overflow(l, r, &result))
hml_runtime_error("Integer overflow: i64 addition");
return make_int_result_checked(result_type, result);
}
case HML_OP_SUB: {
int64_t result;
if (__builtin_sub_overflow(l, r, &result))
hml_runtime_error("Integer overflow: i64 subtraction");
return make_int_result_checked(result_type, result);
}
case HML_OP_MUL: {
int64_t result;
if (__builtin_mul_overflow(l, r, &result))
hml_runtime_error("Integer overflow: i64 multiplication");
return make_int_result_checked(result_type, result);
}
case HML_OP_DIV:
if (r == 0) {
hml_runtime_error("Division by zero");
Expand Down Expand Up @@ -403,9 +463,22 @@ HmlValue hml_unary_op(HmlUnaryOp op, HmlValue operand) {
} else if (operand.type == HML_VAL_F32) {
return hml_val_f32(-operand.as.as_f32);
} else if (operand.type == HML_VAL_I64) {
if (operand.as.as_i64 == INT64_MIN)
hml_runtime_error("Integer overflow: i64 negation");
return hml_val_i64(-operand.as.as_i64);
} else if (operand.type == HML_VAL_I8) {
if (operand.as.as_i8 == INT8_MIN)
hml_runtime_error("Integer overflow: i8 negation");
return hml_val_i8(-operand.as.as_i8);
} else if (operand.type == HML_VAL_I16) {
if (operand.as.as_i16 == INT16_MIN)
hml_runtime_error("Integer overflow: i16 negation");
return hml_val_i16(-operand.as.as_i16);
} else {
return hml_val_i32(-hml_to_i32(operand));
int32_t v = hml_to_i32(operand);
if (v == INT32_MIN)
hml_runtime_error("Integer overflow: i32 negation");
return hml_val_i32(-v);
}

case HML_UNARY_BIT_NOT:
Expand Down
70 changes: 4 additions & 66 deletions src/backends/compiler/codegen_expr.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -937,21 +937,7 @@ char* codegen_expr(CodegenContext *ctx, Expr *expr) {
}
// Check if variable is unboxed (native C type)
// Skip if captured variable - captured vars are always HmlValue in closure env
if (ctx->optimize && ctx->type_ctx && !codegen_is_main_var(ctx, raw_var) &&
!is_captured_variable(ctx, raw_var)) {
CheckedTypeKind native_type = type_check_get_unboxable(ctx->type_ctx, raw_var);
if (native_type != CHECKED_UNKNOWN && checked_kind_is_numeric(native_type)) {
const char *box_func = checked_type_to_box_func(native_type);
if (box_func) {
// Unboxed variable - use simple C increment
codegen_writeln(ctx, "++%s;", var);
codegen_writeln(ctx, "HmlValue %s = %s(%s);", result, box_func, var);
if (safe_var) free(safe_var);
break;
}
}
}
// Fast path for i32, fallback to generic binary_op
// Fast path for i32 (with overflow check), fallback to generic binary_op
codegen_writeln(ctx, "%s = %s.type == HML_VAL_I32 ? hml_i32_inc(%s) : hml_binary_op(HML_OP_ADD, %s, hml_val_i32(1));", var, var, var, var);
// If captured variable, update closure environment
if (ctx->current_closure && ctx->current_closure->num_captured > 0) {
Expand Down Expand Up @@ -1023,23 +1009,7 @@ char* codegen_expr(CodegenContext *ctx, Expr *expr) {
safe_var = codegen_sanitize_ident(raw_var);
var = safe_var;
}
// Check if variable is unboxed (native C type)
// Skip if captured variable - captured vars are always HmlValue in closure env
if (ctx->optimize && ctx->type_ctx && !codegen_is_main_var(ctx, raw_var) &&
!is_captured_variable(ctx, raw_var)) {
CheckedTypeKind native_type = type_check_get_unboxable(ctx->type_ctx, raw_var);
if (native_type != CHECKED_UNKNOWN && checked_kind_is_numeric(native_type)) {
const char *box_func = checked_type_to_box_func(native_type);
if (box_func) {
// Unboxed variable - use simple C decrement
codegen_writeln(ctx, "--%s;", var);
codegen_writeln(ctx, "HmlValue %s = %s(%s);", result, box_func, var);
if (safe_var) free(safe_var);
break;
}
}
}
// Fast path for i32, fallback to generic binary_op
// Fast path for i32 (with overflow check), fallback to generic binary_op
codegen_writeln(ctx, "%s = %s.type == HML_VAL_I32 ? hml_i32_dec(%s) : hml_binary_op(HML_OP_SUB, %s, hml_val_i32(1));", var, var, var, var);
// If captured variable, update closure environment
if (ctx->current_closure && ctx->current_closure->num_captured > 0) {
Expand Down Expand Up @@ -1112,25 +1082,9 @@ char* codegen_expr(CodegenContext *ctx, Expr *expr) {
safe_var = codegen_sanitize_ident(raw_var);
var = safe_var;
}
// Check if variable is unboxed (native C type)
// Skip if captured variable - captured vars are always HmlValue in closure env
if (ctx->optimize && ctx->type_ctx && !codegen_is_main_var(ctx, raw_var) &&
!is_captured_variable(ctx, raw_var)) {
CheckedTypeKind native_type = type_check_get_unboxable(ctx->type_ctx, raw_var);
if (native_type != CHECKED_UNKNOWN && checked_kind_is_numeric(native_type)) {
const char *box_func = checked_type_to_box_func(native_type);
if (box_func) {
// Unboxed variable - return old value, then increment
codegen_writeln(ctx, "HmlValue %s = %s(%s);", result, box_func, var);
codegen_writeln(ctx, "%s++;", var);
if (safe_var) free(safe_var);
break;
}
}
}
codegen_writeln(ctx, "HmlValue %s = %s;", result, var);
codegen_writeln(ctx, "hml_retain_if_needed(&%s);", result);
// Fast path for i32, fallback to generic binary_op
// Fast path for i32 (with overflow check), fallback to generic binary_op
codegen_writeln(ctx, "%s = %s.type == HML_VAL_I32 ? hml_i32_inc(%s) : hml_binary_op(HML_OP_ADD, %s, hml_val_i32(1));", var, var, var, var);
// If captured variable, update closure environment
if (ctx->current_closure && ctx->current_closure->num_captured > 0) {
Expand Down Expand Up @@ -1200,25 +1154,9 @@ char* codegen_expr(CodegenContext *ctx, Expr *expr) {
safe_var = codegen_sanitize_ident(raw_var);
var = safe_var;
}
// Check if variable is unboxed (native C type)
// Skip if captured variable - captured vars are always HmlValue in closure env
if (ctx->optimize && ctx->type_ctx && !codegen_is_main_var(ctx, raw_var) &&
!is_captured_variable(ctx, raw_var)) {
CheckedTypeKind native_type = type_check_get_unboxable(ctx->type_ctx, raw_var);
if (native_type != CHECKED_UNKNOWN && checked_kind_is_numeric(native_type)) {
const char *box_func = checked_type_to_box_func(native_type);
if (box_func) {
// Unboxed variable - return old value, then decrement
codegen_writeln(ctx, "HmlValue %s = %s(%s);", result, box_func, var);
codegen_writeln(ctx, "%s--;", var);
if (safe_var) free(safe_var);
break;
}
}
}
codegen_writeln(ctx, "HmlValue %s = %s;", result, var);
codegen_writeln(ctx, "hml_retain_if_needed(&%s);", result);
// Fast path for i32, fallback to generic binary_op
// Fast path for i32 (with overflow check), fallback to generic binary_op
codegen_writeln(ctx, "%s = %s.type == HML_VAL_I32 ? hml_i32_dec(%s) : hml_binary_op(HML_OP_SUB, %s, hml_val_i32(1));", var, var, var, var);
// If captured variable, update closure environment
if (ctx->current_closure && ctx->current_closure->num_captured > 0) {
Expand Down
17 changes: 5 additions & 12 deletions src/backends/compiler/codegen_expr_binary.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ char* codegen_native_expr(CodegenContext *ctx, Expr *expr, CheckedTypeKind *out_
if (expr->as.binary.op == OP_AND || expr->as.binary.op == OP_OR) return NULL;
// Shifts need validation (negative/oversized) - use safe runtime path
if (expr->as.binary.op == OP_BIT_LSHIFT || expr->as.binary.op == OP_BIT_RSHIFT) return NULL;
// Arithmetic ops need overflow/zero checks - use safe runtime path
if (expr->as.binary.op == OP_ADD || expr->as.binary.op == OP_SUB ||
expr->as.binary.op == OP_MUL || expr->as.binary.op == OP_MOD) return NULL;

CheckedTypeKind left_type, right_type;
char *left = codegen_native_expr(ctx, expr->as.binary.left, &left_type);
Expand Down Expand Up @@ -517,21 +520,11 @@ char* codegen_expr_binary(CodegenContext *ctx, Expr *expr, char *result) {

switch (expr->as.binary.op) {
case OP_ADD:
codegen_writeln(ctx, "HmlValue %s = %s(%s + %s);", result, box_func, left_var, right_var);
break;
case OP_SUB:
codegen_writeln(ctx, "HmlValue %s = %s(%s - %s);", result, box_func, left_var, right_var);
break;
case OP_MUL:
codegen_writeln(ctx, "HmlValue %s = %s(%s * %s);", result, box_func, left_var, right_var);
break;
case OP_MOD:
if (checked_kind_is_integer(left_native)) {
codegen_writeln(ctx, "if (%s == 0) hml_runtime_error(\"Division by zero\");", right_var);
codegen_writeln(ctx, "HmlValue %s = %s(%s %% %s);", result, box_func, left_var, right_var);
} else {
codegen_writeln(ctx, "HmlValue %s = hml_val_f64(fmod(%s, %s));", result, left_var, right_var);
}
// Arithmetic ops need overflow/zero checks - fall through to safe runtime path
handled = 0;
break;
case OP_DIV:
// Division always returns float; check for zero with integer operands
Expand Down
Loading
Loading