Refactor build system by reducing some unreadable macros (#49)

* refactor macro used in schemes.mk

Signed-off-by: Thing-han, Lim <[email protected]>

* add nistkat to all targets

Signed-off-by: Thing-han, Lim <[email protected]>

* tmp workaround for linked to obsolete files

Signed-off-by: Thing-han, Lim <[email protected]>

* move hal related artifacts to obj/hal

Signed-off-by: Thing-han, Lim <[email protected]>

* separate target specific artifacts

Signed-off-by: Thing-han, Lim <[email protected]>

---------

Signed-off-by: Thing-han, Lim <[email protected]>

Author: potsrevennil

.gitignore

@@ -4,6 +4,4 @@
 .vscode
 .idea
 .direnv
-obj
-elf
-bin
+build

Makefile

@@ -1,27 +1,29 @@
 # SPDX-License-Identifier: Apache-2.0
 
-.PHONY: all
-
-all: test speed stack
+.PHONY: all test speed stack nistkat
 
+all: test speed stack nistkat
 
 include mk/config.mk
 include mk/schemes.mk
 include mk/rules.mk
 
 Q ?= @
 
-test: $(foreach scheme,$(KEM_SCHEMES),$(scheme)-test)
-speed: $(foreach scheme,$(KEM_SCHEMES),$(scheme)-speed)
-stack: $(foreach scheme,$(KEM_SCHEMES),$(scheme)-stack)
-nistkat: $(foreach scheme,$(KEM_SCHEMES),$(scheme)-nistkat)
+nistkat:
+	$(Q)$(MAKE) RNG=NISTKAT PLATFORM=$(PLATFORM) mlkem512-$@
+	$(Q)$(MAKE) RNG=NISTKAT PLATFORM=$(PLATFORM) mlkem768-$@
+	$(Q)$(MAKE) RNG=NISTKAT PLATFORM=$(PLATFORM) mlkem1024-$@
+
+test speed stack:
+	$(Q)$(MAKE) PLATFORM=$(PLATFORM) mlkem512-$@
+	$(Q)$(MAKE) PLATFORM=$(PLATFORM) mlkem768-$@
+	$(Q)$(MAKE) PLATFORM=$(PLATFORM) mlkem1024-$@
 
 .PHONY: emulate clean
 
 emulate:
 	$(Q)qemu-system-arm -machine $(PLATFORM) -nographic -semihosting -kernel $(ELF_FILE)
 
 clean:
-	$(Q)rm -rf elf/
-	$(Q)rm -rf bin/
-	$(Q)rm -rf obj/
+	$(Q)rm -rf build/

mk/config.mk

@@ -16,10 +16,16 @@ SIZE := $(CROSS_PREFIX)-size
 ##############################
 # Include retained variables #
 ##############################
+PLATFORM ?= stm32f4discovery
+RNG ?= HAL
+RETAINED_VARS := PLATFORM RNG
 
-RETAINED_VARS :=
+BUILD_DIR := build
+BUILD_PLATFORM_DIR := $(BUILD_DIR)/$(PLATFORM)
+OBJ_DIR := $(BUILD_PLATFORM_DIR)/obj
+BIN_DIR := $(BUILD_PLATFORM_DIR)/bin
 
-CONFIG := obj/.config.mk
+CONFIG := $(BUILD_PLATFORM_DIR)/config.mk
 
 -include $(CONFIG)
 
@@ -32,31 +38,13 @@ $(CONFIG):
 ###############
 # Some Macros #
 ###############
-objs = $(addprefix obj/,$(addsuffix .o,$(1)))
-
-PLATFORM ?= stm32f4discovery
-RNG ?= HAL
 
-RETAINED_VARS += PLATFORM RNG
-
-# RNG config
-ifeq ($(RNG),HAL)
-	LIBHAL_SRC = hal/randombytes.c
-else ifeq ($(RNG),NOTRAND)
-	LIBHAL_SRC = hal/notrandombytes.c
-else ifeq ($(RNG),NISTKAT)
-	LIBHAL_SRC = \
-		test/common/nistkatrng.c \
-		test/common/aes.c
-	CPPFLAGS += -Itest/common
-endif
-
-LDLIBS += -lhal
-LIBDEPS += obj/libhal.a
-
-# HAL config
+LDLIBS += -lhal -L$(OBJ_DIR)/hal
+LIBDEPS += $(OBJ_DIR)/hal/libhal.a
 
 # Common config
+objs = $(addprefix $(OBJ_DIR)/,$(addsuffix .o,$(1)))
+
 include mk/$(PLATFORM).mk
 
 CFLAGS += \
@@ -82,12 +70,13 @@ LDFLAGS += \
 	-Wl,--wrap=_fstat \
 	-Wl,--wrap=_getpid \
 	-ffreestanding \
-	-Lobj \
 	-Wl,--gc-sections
 
 NTESTS ?= 1
 RETAINED_VARS += NTESTS
 
+TESTS = test speed stack nistkat
+
 KEM_SCHEMES=mlkem512 mlkem768 mlkem1024
 # mlkem k
 KEM_PARAMS=2 3 4

mk/mbed-os.mk

@@ -1,5 +1,15 @@
 # SPDX-License-Identifier: Apache-2.0
-LDSCRIPT = obj/generated.$(PLATFORM).ld
+ifeq ($(RNG),NISTKAT)
+	LIBHAL_SRC = \
+		test/common/nistkatrng.c \
+		test/common/aes.c
+	CPPFLAGS += -Itest/common
+else
+	RNG := NOTRAND
+	LIBHAL_SRC = hal/notrandombytes.c
+endif
+
+LDSCRIPT = $(OBJ_DIR)/generated.$(PLATFORM).ld
 
 CFLAGS += $(ARCH_FLAGS)
 
@@ -8,12 +18,19 @@ LDFLAGS += \
 	-T$(LDSCRIPT) \
 	$(ARCH_FLAGS)
 
-LIBHAL_SRC += hal/hal-mps2.c $(MBED_OS_TARGET_DIR)/TOOLCHAIN_GCC_ARM/startup_MPS2.S
+LIBHAL_SRC += hal/hal-mps2.c
+STARTUP_SRC = $(MBED_OS_TARGET_DIR)/TOOLCHAIN_GCC_ARM/startup_MPS2.S
+STARTUP_OBJ = $(shell echo "$(STARTUP_SRC)" | sed -E 's~(.*/)(TARGET.*)~$(OBJ_DIR)/hal/\2.o~g')
 
 MPS2_DEPS += -I$(MBED_OS_DIR)/Include -I$(MBED_OS_TARGET_DIR)
 
-obj/libhal.a: $(call objs,$(LIBHAL_SRC))
-obj/libhal.a: CPPFLAGS += $(MPS2_DEPS) $(if $(RNG)==NISTKAT,-Itest/common)
+$(OBJ_DIR)/hal/libhal.a: $(call objs,$(LIBHAL_SRC)) $(STARTUP_OBJ)
+$(OBJ_DIR)/hal/libhal.a: CPPFLAGS += $(MPS2_DEPS) $(if $(RNG)==NISTKAT,-Itest/common)
+
+$(STARTUP_OBJ): $(STARTUP_SRC)
+	@echo "  AS      $@"
+	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
+	$(Q)$(CC) -c -o $@ $(CFLAGS) $<
 
 $(LDSCRIPT): 	$(MBED_OS_TARGET_DIR)/TOOLCHAIN_GCC_ARM/MPS2.ld
 	@printf "  GENLNK  $@\n"; \
@@ -23,4 +40,3 @@ $(LDSCRIPT): 	$(MBED_OS_TARGET_DIR)/TOOLCHAIN_GCC_ARM/MPS2.ld
 $(LDSCRIPT): CPPFLAGS += $(MPS2_DEPS) $(if $(RNG)==NISTKAT,-Itest/common)
 
 LINKDEPS += $(LDSCRIPT) $(LIBDEPS)
-

mk/opencm3.mk

@@ -1,14 +1,24 @@
 # SPDX-License-Identifier: Apache-2.0
+ifeq ($(RNG),NISTKAT)
+	LIBHAL_SRC = \
+		test/common/nistkatrng.c \
+		test/common/aes.c
+	CPPFLAGS += -Itest/common
+else
+	RNG := HAL
+	LIBHAL_SRC = hal/randombytes.c
+endif
+
 OPENCM3_DIR ?=
-override LDSCRIPT := obj/generated.$(DEVICE).ld
+override LDSCRIPT := $(OBJ_DIR)/generated.$(DEVICE).ld
 
 include $(OPENCM3_DIR)/mk/genlink-config.mk
 include $(OPENCM3_DIR)/mk/genlink-rules.mk
 include $(OPENCM3_DIR)/mk/gcc-rules.mk
 
 LIBHAL_SRC += hal/hal-opencm3.c
 
-obj/libhal.a: $(call objs,$(LIBHAL_SRC))
+$(OBJ_DIR)/hal/libhal.a: $(call objs,$(LIBHAL_SRC))
 
 CFLAGS += $(ARCH_FLAGS)
 

mk/rules.mk

@@ -1,36 +1,36 @@
 # SPDX-License-Identifier: Apache-2.0
-elf/%.elf: $(CONFIG)
+$(BIN_DIR)/%.elf: $(CONFIG)
 	@echo "  LD      $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)$(LD) $(LDFLAGS) -o $@ $(filter %.o,$^) -Wl,--start-group $(LDLIBS) -Wl,--end-group
 
-obj/%.a: $(CONFIG)
+$(OBJ_DIR)/%.a: $(CONFIG)
 	@echo "  AR      $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)rm -f $@
 	$(Q)$(AR) rcs $@ $(filter %.o,$^)
 
-bin/%.bin: elf/%.elf $(CONFIG)
+$(BIN_DIR)/%.bin: $(BIN_DIR)/%.elf $(CONFIG)
 	@echo "  OBJCOPY $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)$(OBJCOPY) -Obinary $< $@
 
-bin/%.hex: elf/%.elf $(CONFIG)
+$(BIN_DIR)/%.hex: $(BIN_DIR)/%.elf $(CONFIG)
 	@echo "  OBJCOPY $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)$(OBJCOPY) -Oihex $< $@
 
-obj/%.o: %.c $(CONFIG)
+$(OBJ_DIR)/%.o: %.c $(CONFIG)
 	@echo "  CC      $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)$(CC) -c -o $@ $(CFLAGS) $<
 
-obj/%.c.o: %.c $(CONFIG)
+$(OBJ_DIR)/%.c.o: %.c $(CONFIG)
 	@echo "  CC      $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)$(CC) -c -o $@ $(CFLAGS) $<
 
-obj/%.S.o: %.S $(CONFIG)
+$(OBJ_DIR)/%.S.o: %.S $(CONFIG)
 	@echo "  AS      $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)$(CC) -c -o $@ $(CFLAGS) $<

mk/schemes.mk

@@ -4,63 +4,46 @@ FIPS202_SOURCES = $(wildcard fips202/*.c)
 
 OBJS = $(call objs,$(addprefix $(1)/,$(notdir $(basename $(MLKEM_SOURCES)))) $(basename $(FIPS202_SOURCES)))
 
-define scheme-test
-$(1)-test: CPPFLAGS += -DMLKEM_K=$(2) -DNTESTS=$$(NTESTS)
-$(1)-test: bin/$(1)-test.hex $(CONFIG)
+# all tests x all schemes
+SCHEMES_TESTS = $(foreach t,$(TESTS),$(KEM_SCHEMES:%=%-$(t)))
 
-$(1)-speed: CPPFLAGS += -DMLKEM_K=$(2) -DNTESTS=$$(NTESTS)
-$(1)-speed: bin/$(1)-speed.hex $(CONFIG)
-
-$(1)-stack: CPPFLAGS += -DMLKEM_K=$(2)
-$(1)-stack: bin/$(1)-stack.hex $(CONFIG)
-
-$(1)-nistkat: CPPFLAGS += -DMLKEM_K=$(2)
-$(1)-nistkat: bin/$(1)-nistkat.hex $(CONFIG)
+# set up tests dependencies for each scheme
+define SCHEME_DEPS
+$(TESTS:%=$(1)-%): CPPFLAGS += -DMLKEM_K=$(2) -DNTESTS=$(NTESTS)
 endef
 
-define scheme-elf
-elf/$(1)-test.elf: obj/test/$(1)-test.o $(call OBJS,$(1)) $(LINKDEPS) $(CONFIG)
-elf/$(1)-speed.elf: obj/test/$(1)-speed.o $(call OBJS,$(1)) $(LINKDEPS) $(CONFIG)
-elf/$(1)-stack.elf: obj/test/$(1)-stack.o $(call OBJS,$(1)) $(LINKDEPS) $(CONFIG)
-elf/$(1)-nistkat.elf: obj/test/$(1)-nistkat.o $(call OBJS,$(1)) $(LINKDEPS) $(CONFIG)
+define SCHEME_ELF_DEPS
+$(TESTS:%=$(BIN_DIR)/$(1)-%.elf): $(BIN_DIR)/%.elf: $(OBJ_DIR)/test/%.o $(call OBJS,$(1)) $(LINKDEPS)
 endef
 
-define compile-obj
+define COMPILE_OBJ
 	@echo "  CC      $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)$(CC) -c -o $@ $(CFLAGS) $<
 endef
 
-define scheme-obj
-obj/$(1)/%.o: mlkem/%.c $(CONFIG)
-	$$(compile-obj)
+# compile object file for each scheme
+define SCHEME_OBJ
+$(OBJ_DIR)/$(1)/%.o: mlkem/%.c $(CONFIG)
+	$$(COMPILE_OBJ)
 endef
 
-define scheme-test-obj
-obj/test/$(1)-test.o: test/test.c $(CONFIG)
-	$$(compile-obj)
-
-obj/test/$(1)-speed.o: test/speed.c $(CONFIG)
-	$$(compile-obj)
-
-obj/test/$(1)-stack.o: test/stack.c $(CONFIG)
-	$$(compile-obj)
-
-obj/test/$(1)-nistkat.o: test/nistkat.c $(CONFIG)
-	$$(compile-obj)
+define SCHEME_TEST_OBJS
+$(OBJ_DIR)/test/$(1)-%.o: test/%.c $(CONFIG)
+	$$(COMPILE_OBJ)
 endef
 
-# iterate over schemes and params to generate targets for functional test and speed benchmark
-iter=$(let scheme k rest,$(1), \
-	$(eval $(call scheme-test,$(scheme),$(k))) \
-	$(if $(rest),$(call iter,$(rest)) ))
-$(call iter,$(subst -, ,$(join $(addsuffix -,$(KEM_SCHEMES)),$(KEM_PARAMS))))
+$(SCHEMES_TESTS): % : $(BIN_DIR)/%.bin $(BIN_DIR)/%.hex
+
+$(eval $(call SCHEME_DEPS,mlkem512,2))
+$(eval $(call SCHEME_DEPS,mlkem768,3))
+$(eval $(call SCHEME_DEPS,mlkem1024,4))
 
 $(foreach scheme,$(KEM_SCHEMES), \
-	$(eval $(call scheme-elf,$(scheme))))
+	$(eval $(call SCHEME_ELF_DEPS,$(scheme))))
 
 $(foreach scheme,$(KEM_SCHEMES), \
-	$(eval $(call scheme-obj,$(scheme))))
+	$(eval $(call SCHEME_TEST_OBJS,$(scheme))))
 
 $(foreach scheme,$(KEM_SCHEMES), \
-	$(eval $(call scheme-test-obj,$(scheme))))
+	$(eval $(call SCHEME_OBJ,$(scheme))))

scripts/tests

@@ -219,7 +219,7 @@ def base_test(
         dev = usbdev(uart)
 
     for scheme in ["mlkem512", "mlkem768", "mlkem1024"]:
-        file = f"elf/{scheme}-{test_type}.elf"
+        file = f"build/{platform}/bin/{scheme}-{test_type}.elf"
         expect = expect_proc(scheme)
         if platform_cfg is None:
             raw = subprocess.run(