deps: Clean up Snitch integration

.gitlab/sw-tests.yml

@@ -30,8 +30,8 @@
       - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: $SN_BUILD_DIR/redmule.elf }
       - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: $SN_BUILD_DIR/redmule_quant.elf }
       - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: $SN_BUILD_DIR/datamover.elf }
-      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/gemm_2d/build/gemm_2d.elf, VERIFY_PY: $SN_ROOT/sw/blas/gemm/scripts/verify.py, PRELMODE: 3 }
-      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/fused_concat_linear/build/fused_concat_linear.elf, VERIFY_PY: $SN_ROOT/sw/dnn/fused_concat_linear/scripts/verify.py, PRELMODE: 3 }
-      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/mha/build/mha.elf, VERIFY_PY: $SN_ROOT/sw/dnn/mha/scripts/verify.py, PRELMODE: 3 }
-      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/gemm/build/gemm.elf, VERIFY_PY: $SN_ROOT/sw/blas/gemm/scripts/verify.py, PRELMODE: 3 }
-      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/axpy/build/axpy.elf, VERIFY_PY: $SN_ROOT/sw/blas/axpy/scripts/verify.py, PRELMODE: 3 }
+      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/gemm_2d/build/gemm_2d.elf, VERIFY_PY: $SN_ROOT/sw/kernels/blas/gemm/scripts/verify.py, PRELMODE: 3 }
+      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/fused_concat_linear/build/fused_concat_linear.elf, VERIFY_PY: $SN_ROOT/sw/kernels/dnn/fused_concat_linear/scripts/verify.py, PRELMODE: 3 }
+      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/mha/build/mha.elf, VERIFY_PY: $SN_ROOT/sw/kernels/dnn/mha/scripts/verify.py, PRELMODE: 3 }
+      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/gemm/build/gemm.elf, VERIFY_PY: $SN_ROOT/sw/kernels/blas/gemm/scripts/verify.py, PRELMODE: 3 }
+      - { CHS_BINARY: $CHS_BUILD_DIR/simple_offload.spm.elf, SN_BINARY: sw/snitch/apps/axpy/build/axpy.elf, VERIFY_PY: $SN_ROOT/sw/kernels/blas/axpy/scripts/verify.py, PRELMODE: 3 }

Bender.lock

@@ -383,7 +383,7 @@ packages:
     - common_cells
     - register_interface
   snitch_cluster:
-    revision: ef3ece6c9e119fbfc25b26bb89a429ccdaacb5c6
+    revision: 5d6c957c70e3824e2330d8308eb904724cd41cbe
     version: null
     source:
       Git: https://github.com/pulp-platform/snitch_cluster.git

Bender.yml

@@ -12,7 +12,7 @@ dependencies:
   axi:                { git: "https://github.com/pulp-platform/axi.git",                version: "0.39.6"                               }
   common_cells:       { git: "https://github.com/pulp-platform/common_cells.git",       rev: "snitch"                                   }
   cheshire:           { git: "https://github.com/pulp-platform/cheshire.git",           rev: "picobello"                                }
-  snitch_cluster:     { git: "https://github.com/pulp-platform/snitch_cluster.git",     rev: "ef3ece6c9e119fbfc25b26bb89a429ccdaacb5c6" }
+  snitch_cluster:     { git: "https://github.com/pulp-platform/snitch_cluster.git",     rev: "5d6c957c70e3824e2330d8308eb904724cd41cbe" }
   floo_noc:           { git: "https://github.com/pulp-platform/FlooNoC.git",            rev: "develop"                                  }
   obi:                { git: "https://github.com/pulp-platform/obi.git",                rev: "acfcd0f80c7539aa8da7821a66d9acf2074a5b4e" }
   redmule:            { git: "https://github.com/pulp-platform/redmule.git",            rev: "picobello"                                }

Makefile

@@ -104,8 +104,8 @@ $(CHS_SLINK_DIR)/.generated2:	$(SLINK_CFG)
 ##################
 
 SN_GEN_DIR = $(PB_GEN_DIR)
-include $(SN_ROOT)/target/common/common.mk
-include $(SN_ROOT)/target/common/rtl.mk
+include $(SN_ROOT)/make/common.mk
+include $(SN_ROOT)/make/rtl.mk
 
 .PHONY: sn-hw-clean sn-hw-all
 
@@ -199,6 +199,12 @@ TB_DUT = tb_picobello_top
 include $(PB_ROOT)/target/sim/vsim/vsim.mk
 include $(PB_ROOT)/target/sim/traces.mk
 
+##################
+# Snitch cluster #
+##################
+
+$(call sn_include_deps)
+
 ########
 # Misc #
 ########

iis-env.sh

@@ -8,8 +8,8 @@ export VOPT="questa-2023.4 vopt"
 export VLIB="questa-2023.4 vlib"
 export BASE_PYTHON=/usr/local/anaconda3/bin/python3.11
 export CHS_SW_GCC_BINROOT=/usr/pack/riscv-1.0-kgf/riscv64-gcc-12.2.0/bin
-export LLVM_BINROOT=/usr/scratch2/vulcano/colluca/tools/riscv32-snitch-llvm-almalinux8-15.0.0-snitch-0.2.0/bin
 export VERIBLE_FMT="oseda -2025.03 verible-verilog-format"
+export SN_LLVM_BINROOT=/usr/scratch2/vulcano/colluca/tools/riscv32-snitch-llvm-almalinux8-15.0.0-snitch-0.2.0/bin
 
 # Create the python venv
 if [ ! -d ".venv" ]; then

sw/snitch/apps/axpy/app.mk

@@ -7,9 +7,9 @@
 APP              := axpy
 $(APP)_BUILD_DIR ?= $(PB_SNITCH_SW_DIR)/apps/$(APP)/build
 $(APP)_DATA_CFG  := $(PB_SNITCH_SW_DIR)/apps/$(APP)/data/params.json
-SRC_DIR          := $(SN_ROOT)/sw/blas/$(APP)/src
+SRC_DIR          := $(SN_ROOT)/sw/kernels/blas/$(APP)/src
 SRCS             := $(SRC_DIR)/main.c
-$(APP)_INCDIRS   := $(SN_ROOT)/sw/blas
+$(APP)_INCDIRS   := $(SN_ROOT)/sw/kernels/blas
 
-include $(SN_ROOT)/sw/apps/common.mk
-include $(SN_ROOT)/target/snitch_cluster/sw/apps/common.mk
+include $(SN_ROOT)/sw/kernels/datagen.mk
+include $(SN_ROOT)/sw/kernels/common.mk

sw/snitch/apps/fused_concat_linear/app.mk

@@ -7,9 +7,7 @@
 APP              := fused_concat_linear
 $(APP)_BUILD_DIR ?= $(PB_SNITCH_SW_DIR)/apps/$(APP)/build
 $(APP)_DATA_CFG  := $(PB_SNITCH_SW_DIR)/apps/$(APP)/data/params.json
-SRC_DIR          := $(SN_ROOT)/sw/dnn/$(APP)/src
+SRC_DIR          := $(SN_ROOT)/sw/kernels/dnn/$(APP)/src
 SRCS             := $(SRC_DIR)/main.c
-$(APP)_INCDIRS   := $(SN_ROOT)/sw/dnn/src $(SN_ROOT)/sw/blas
 
-include $(SN_ROOT)/sw/apps/common.mk
-include $(SN_ROOT)/target/snitch_cluster/sw/apps/common.mk
+include $(SN_ROOT)/sw/kernels/dnn/common.mk

sw/snitch/apps/gemm/app.mk

@@ -7,9 +7,9 @@
 APP              := gemm
 $(APP)_BUILD_DIR ?= $(PB_SNITCH_SW_DIR)/apps/$(APP)/build
 $(APP)_DATA_CFG  := $(PB_SNITCH_SW_DIR)/apps/$(APP)/data/params.json
-SRC_DIR          := $(SN_ROOT)/sw/blas/$(APP)/src
+SRC_DIR          := $(SN_ROOT)/sw/kernels/blas/$(APP)/src
 SRCS             := $(SRC_DIR)/main.c
-$(APP)_INCDIRS   := $(SN_ROOT)/sw/blas
+$(APP)_INCDIRS   := $(SN_ROOT)/sw/kernels/blas
 
-include $(SN_ROOT)/sw/apps/common.mk
-include $(SN_ROOT)/target/snitch_cluster/sw/apps/common.mk
+include $(SN_ROOT)/sw/kernels/datagen.mk
+include $(SN_ROOT)/sw/kernels/common.mk

sw/snitch/apps/gemm_2d/app.mk

@@ -8,10 +8,10 @@ APP              := gemm_2d
 $(APP)_BUILD_DIR ?= $(PB_SNITCH_SW_DIR)/apps/$(APP)/build
 SRC_DIR          := $(PB_SNITCH_SW_DIR)/apps/$(APP)/src
 SRCS             := $(SRC_DIR)/gemm_2d.c
-$(APP)_INCDIRS   := $(SN_ROOT)/sw/blas $(SN_ROOT)/sw/blas/gemm/src
+$(APP)_INCDIRS   := $(SN_ROOT)/sw/kernels/blas $(SN_ROOT)/sw/kernels/blas/gemm/src
 
 # Refer to Snitch scripts
-$(APP)_SCRIPT_DIR :=  $(SN_ROOT)/sw/blas/gemm/scripts
+$(APP)_SCRIPT_DIR :=  $(SN_ROOT)/sw/kernels/blas/gemm/scripts
 
-include $(SN_ROOT)/sw/apps/common.mk
-include $(SN_ROOT)/target/snitch_cluster/sw/apps/common.mk
+include $(SN_ROOT)/sw/kernels/datagen.mk
+include $(SN_ROOT)/sw/kernels/common.mk

sw/snitch/apps/mha/app.mk

@@ -7,9 +7,7 @@
 APP              := mha
 $(APP)_BUILD_DIR ?= $(PB_SNITCH_SW_DIR)/apps/$(APP)/build
 $(APP)_DATA_CFG  := $(PB_SNITCH_SW_DIR)/apps/$(APP)/data/params.json
-SRC_DIR          := $(SN_ROOT)/sw/dnn/$(APP)/src
+SRC_DIR          := $(SN_ROOT)/sw/kernels/dnn/$(APP)/src
 SRCS             := $(SRC_DIR)/main.c
-$(APP)_INCDIRS   := $(SN_ROOT)/sw/dnn/src $(SN_ROOT)/sw/blas
 
-include $(SN_ROOT)/sw/apps/common.mk
-include $(SN_ROOT)/target/snitch_cluster/sw/apps/common.mk
+include $(SN_ROOT)/sw/kernels/dnn/common.mk

sw/snitch/runtime/src/.gitignore

@@ -0,0 +1,6 @@
+snitch_cluster_cfg.h
+snitch_cluster_addrmap.h
+snitch_cluster_raw_addrmap.h
+snitch_cluster_peripheral.h
+snitch_cluster_peripheral_addrmap.h
+snitch_cluster_addrmap.rdl

sw/snitch/runtime/src/snitch_cluster_addrmap.rdl.tpl

@@ -0,0 +1 @@
+../../../../.deps/snitch_cluster/sw/runtime/impl/snitch_cluster_addrmap.rdl.tpl

sw/snitch/runtime/src/snitch_cluster_cfg.h.tpl

@@ -0,0 +1 @@
+../../../../.deps/snitch_cluster/sw/runtime/impl/snitch_cluster_cfg.h.tpl

sw/snitch/runtime/src/snitch_cluster_memory.c

@@ -1 +1 @@
-../../../../.deps/snitch_cluster/target/snitch_cluster/sw/runtime/common/snitch_cluster_memory.c
+../../../../.deps/snitch_cluster/sw/runtime/impl/snitch_cluster_memory.c

sw/snitch/runtime/src/snitch_cluster_start.c

@@ -1 +1 @@
-../../../../.deps/snitch_cluster/target/snitch_cluster/sw/runtime/rtl/src/snitch_cluster_start.c
+../../../../.deps/snitch_cluster/sw/runtime/impl/snitch_cluster_start.c

sw/snitch/runtime/src/snrt.cc

@@ -1 +1 @@
-../../../../.deps/snitch_cluster/target/snitch_cluster/sw/runtime/rtl/src/snrt.cc
+../../../../.deps/snitch_cluster/sw/runtime/impl/snrt.cc

sw/sw.mk

@@ -22,54 +22,52 @@ PB_GEN_DIR = $(PB_ROOT)/.generated
 ## Snitch Cluster ##
 ####################
 
-SNRT_TARGET_DIR     = $(PB_SNITCH_SW_DIR)/runtime
-SNRT_TESTS_BUILDDIR = $(PB_SNITCH_SW_DIR)/tests/build
-SN_RVTESTS_BUILDDIR = $(PB_SNITCH_SW_DIR)/riscv-tests/build
-SNRT_INCDIRS        = $(PB_INCDIR) $(PB_GEN_DIR)
-SNRT_BUILD_APPS     = OFF
-SNRT_MEMORY_LD      = $(PB_SNITCH_SW_DIR)/memory.ld
-SNRT_HAL_BUILD_DIR  = $(PB_SNITCH_SW_DIR)/runtime/build
-SNRT_HAL_HDRS       = $(PB_GEN_DIR)/pb_addrmap.h
-SNRT_HAL_HDRS      += $(PB_GEN_DIR)/pb_raw_addrmap.h
-
-SNRT_APPS  = $(PB_SNITCH_SW_DIR)/apps/gemm_2d
-SNRT_APPS += $(PB_SNITCH_SW_DIR)/apps/gemm
-SNRT_APPS += $(PB_SNITCH_SW_DIR)/apps/axpy
-SNRT_APPS += $(SN_ROOT)/target/snitch_cluster/sw/apps/dnn/flashattention_2
-SNRT_APPS += $(PB_SNITCH_SW_DIR)/apps/fused_concat_linear
-SNRT_APPS += $(PB_SNITCH_SW_DIR)/apps/mha
-
-ifneq (,$(filter $(PB_SNITCH_SW_DIR)% chs-bootrom% chs-sw% sn% pb-sn-tests% sw%,$(MAKECMDGOALS)))
-include $(SN_ROOT)/target/snitch_cluster/sw.mk
-endif
+SN_RUNTIME_SRCDIR    = $(PB_SNITCH_SW_DIR)/runtime/src
+SN_RUNTIME_BUILDDIR  = $(PB_SNITCH_SW_DIR)/runtime/build
+SN_TESTS_BUILDDIR    = $(PB_SNITCH_SW_DIR)/tests/build
+SN_RVTESTS_BUILDDIR  = $(PB_SNITCH_SW_DIR)/riscv-tests/build
+SN_RUNTIME_INCDIRS   = $(PB_INCDIR)
+SN_RUNTIME_INCDIRS  += $(PB_GEN_DIR)
+SN_RUNTIME_HAL_HDRS  = $(PB_GEN_DIR)/pb_addrmap.h
+SN_RUNTIME_HAL_HDRS += $(PB_GEN_DIR)/pb_raw_addrmap.h
+SN_BUILD_APPS        = OFF
+
+SN_APPS  = $(PB_SNITCH_SW_DIR)/apps/gemm_2d
+SN_APPS += $(PB_SNITCH_SW_DIR)/apps/gemm
+SN_APPS += $(PB_SNITCH_SW_DIR)/apps/axpy
+SN_APPS += $(SN_ROOT)/sw/kernels/dnn/flashattention_2
+SN_APPS += $(PB_SNITCH_SW_DIR)/apps/fused_concat_linear
+SN_APPS += $(PB_SNITCH_SW_DIR)/apps/mha
+
+include $(SN_ROOT)/make/sw.mk
 
 $(PB_GEN_DIR)/pb_raw_addrmap.h: $(PB_RDL_ALL)
 	$(PEAKRDL) raw-header $< -o $@ $(PEAKRDL_INCLUDES) $(PEAKRDL_DEFINES) --base_name $(notdir $(basename $@)) --format c
 
 # Collect Snitch tests which should be built
-PB_SNRT_TESTS_DIR      = $(PB_SNITCH_SW_DIR)/tests
-PB_SNRT_TESTS_BUILDDIR = $(PB_SNITCH_SW_DIR)/tests/build
-PB_SNRT_TEST_NAMES = $(basename $(notdir $(wildcard $(PB_SNRT_TESTS_DIR)/*.c)))
-PB_SNRT_TEST_ELFS = $(abspath $(addprefix $(PB_SNRT_TESTS_BUILDDIR)/,$(addsuffix .elf,$(PB_SNRT_TEST_NAMES))))
-PB_SNRT_TEST_DUMP = $(abspath $(addprefix $(PB_SNRT_TESTS_BUILDDIR)/,$(addsuffix .dump,$(PB_SNRT_TEST_NAMES))))
+PB_SN_TESTS_DIR      = $(PB_SNITCH_SW_DIR)/tests
+PB_SN_TESTS_BUILDDIR = $(PB_SNITCH_SW_DIR)/tests/build
+PB_SN_TEST_NAMES = $(basename $(notdir $(wildcard $(PB_SN_TESTS_DIR)/*.c)))
+PB_SN_TEST_ELFS = $(abspath $(addprefix $(PB_SN_TESTS_BUILDDIR)/,$(addsuffix .elf,$(PB_SN_TEST_NAMES))))
+PB_SN_TEST_DUMP = $(abspath $(addprefix $(PB_SN_TESTS_BUILDDIR)/,$(addsuffix .dump,$(PB_SN_TEST_NAMES))))
 
-.PHONY: pb-snrt-tests clean-pb-snrt-tests
+.PHONY: pb-sn-tests clean-pb-sn-tests
 
-pb-sn-tests: $(PB_SNRT_TEST_ELFS) $(PB_SNRT_TEST_DUMP)
+pb-sn-tests: $(PB_SN_TEST_ELFS) $(PB_SN_TEST_DUMP)
 
 clean-pb-sn-tests:
-	rm -rf $(PB_SNRT_TEST_ELFS)
+	rm -rf $(PB_SN_TEST_ELFS)
 
-$(PB_SNRT_TEST_ELFS): $(PB_GEN_DIR)/pb_addrmap.h
+$(PB_SN_TEST_ELFS): $(PB_GEN_DIR)/pb_addrmap.h
 
-$(PB_SNRT_TESTS_BUILDDIR)/%.d: $(PB_SNRT_TESTS_DIR)/%.c | $(PB_SNRT_TESTS_BUILDDIR)
-	$(RISCV_CXX) $(SNRT_TESTS_RISCV_CFLAGS) -MM -MT '$(@:.d=.elf)' -x c++ $< > $@
+$(PB_SN_TESTS_BUILDDIR)/%.d: $(PB_SN_TESTS_DIR)/%.c | $(PB_SN_TESTS_BUILDDIR)
+	$(SN_RISCV_CXX) $(SN_TESTS_RISCV_CFLAGS) -MM -MT '$(@:.d=.elf)' -x c++ $< > $@
 
-$(PB_SNRT_TESTS_BUILDDIR)/%.elf: $(PB_SNRT_TESTS_DIR)/%.c $(SNRT_LIB) | $(PB_SNRT_TESTS_BUILDDIR)
-	$(RISCV_CXX) $(SNRT_TESTS_RISCV_CFLAGS) $(SNRT_TESTS_RISCV_LDFLAGS) -x c++ $< -o $@
+$(PB_SN_TESTS_BUILDDIR)/%.elf: $(PB_SN_TESTS_DIR)/%.c $(SN_RUNTIME_LIB) | $(PB_SN_TESTS_BUILDDIR)
+	$(SN_RISCV_CXX) $(SN_TESTS_RISCV_CFLAGS) $(SN_TESTS_RISCV_LDFLAGS) -x c++ $< -o $@
 
-$(PB_SNRT_TESTS_BUILDDIR)/%.dump: $(PB_SNRT_TESTS_BUILDDIR)/%.elf | $(PB_SNRT_TESTS_BUILDDIR)
-	$(RISCV_OBJDUMP) $(RISCV_OBJDUMP_FLAGS) $< > $@
+$(PB_SN_TESTS_BUILDDIR)/%.dump: $(PB_SN_TESTS_BUILDDIR)/%.elf | $(PB_SN_TESTS_BUILDDIR)
+	$(SN_RISCV_OBJDUMP) $(SN_RISCV_OBJDUMP_FLAGS) $< > $@
 
 ##############
 ## Cheshire ##
@@ -79,7 +77,7 @@ PB_LINK_MODE ?= spm
 
 # We need to include the address map and snitch cluster includes
 CHS_SW_INCLUDES += -I$(PB_INCDIR)
-CHS_SW_INCLUDES += -I$(SNRT_HAL_BUILD_DIR)
+CHS_SW_INCLUDES += -I$(SN_RUNTIME_SRCDIR)
 CHS_SW_INCLUDES += -I$(PB_GEN_DIR)
 
 # Collect tests, which should be build for all modes, and their .dump targets
@@ -89,8 +87,8 @@ PB_CHS_SW_TEST_ELF += $(PB_CHS_SW_TEST_SRC:.c=.$(PB_LINK_MODE).elf)
 
 PB_CHS_SW_TEST = $(PB_CHS_SW_TEST_DUMP)
 
-$(PB_CHS_SW_TEST_SRC): $(PB_GEN_DIR)/pb_addrmap.h
 $(PB_CHS_SW_TEST_DUMP): $(PB_CHS_SW_TEST_ELF)
+$(PB_CHS_SW_TEST_ELF): $(PB_GEN_DIR)/pb_addrmap.h $(SN_RUNTIME_HAL_HDRS)
 
 .PHONY: chs-sw-tests chs-sw-tests-clean
 

target/sim/traces.mk

@@ -4,6 +4,11 @@
 #
 # Author: Luca Colagrande <colluca@iis.ee.ethz.ch>
 
+SIM_DIR = $(PB_ROOT)
+LOGS_DIR = $(SIM_DIR)/logs
+SN_SIM_DIR = $(SIM_DIR)
+include $(SN_ROOT)/make/traces.mk
+
 CHS_ADDR2LINE      ?= $(CHS_SW_GCC_BINROOT)/riscv64-unknown-elf-addr2line
 CHS_TXT_TRACE       = $(LOGS_DIR)/trace_hart_00000.txt
 CHS_ANNOTATED_TRACE = $(LOGS_DIR)/trace_hart_00000.s
@@ -12,11 +17,13 @@ CHS_BINARY         ?= $(shell cat $(SIM_DIR)/.chsbinary)
 # Cheshire trace generation
 $(CHS_TXT_TRACE): $(SIM_DIR)/trace_hart_0.log
 	cp $< $@
-$(CHS_ANNOTATED_TRACE): $(CHS_TXT_TRACE) $(ANNOTATE_PY)
-	$(PYTHON) $(ANNOTATE_PY) -f cva6 -q --keep-time --addr2line=$(CHS_ADDR2LINE) -o $@ $(CHS_BINARY) $<
+$(CHS_ANNOTATED_TRACE): $(CHS_TXT_TRACE) $(SN_ANNOTATE_PY)
+	$(PYTHON) $(SN_ANNOTATE_PY) -f cva6 -q --keep-time --addr2line=$(CHS_ADDR2LINE) -o $@ $(CHS_BINARY) $<
 
-traces: chs-trace
-annotate: chs-annotate
+traces: sn-traces chs-trace
+annotate: sn-annotate chs-annotate
+traces-clean: sn-clean-traces chs-trace-clean
+annotate-clean: sn-clean-annotate chs-annotate-clean
 
 chs-trace: $(CHS_TXT_TRACE)
 chs-annotate: $(CHS_ANNOTATED_TRACE)