Multi-tile configuration of CachePool

.github/workflows/gitlab-ci.yml

@@ -29,6 +29,8 @@ jobs:
           install/*
           config/*
           hardware/tb/dpi/*
+          hardware/tb/dram_config/*
+          hardware/bootrom/*
           Bender.lock
           .gitignore
 

.gitlab-ci.yml

@@ -30,6 +30,7 @@ stages:
 build-vsim:
   extends: .base
   stage: build
+  timeout: 5h
   script:
     - echo "Using CC=$CC"
     - echo "Using CXX=$CXX"
@@ -44,6 +45,4 @@ build-vsim:
 
   artifacts:
     paths:
-      - software/build
-      - sim/bin
       - util/auto-benchmark/logs

Bender.lock

@@ -71,7 +71,7 @@ packages:
     - common_verification
     - register_interface
   insitu-cache:
-    revision: 04f72a7ac7e9091f1820f0dac59bb778b134d7f7
+    revision: fa761ddebc946f9b46509d84945bf41ee1a9ec49
     version: null
     source:
       Git: https://github.com/pulp-platform/Insitu-Cache.git
@@ -97,7 +97,7 @@ packages:
     - common_cells
     - tech_cells_generic
   spatz:
-    revision: b60f7aee5e3be31facf566955aadbd5b6eac5da3
+    revision: 94ff5f6ca70e4dfef6168e0ac25b0bdd88e40132
     version: null
     source:
       Git: https://github.com/pulp-platform/spatz.git

Bender.yml

@@ -29,12 +29,23 @@ sources:
   - hardware/src/tcdm_cache_interco.sv
   - hardware/src/tcdm_id_remapper.sv
   - hardware/src/spatz_cache_amo.sv
+  # Memory-mapped register
+  - hardware/cachepool_peripheral/cachepool_peripheral_reg_pkg.sv
+  - hardware/cachepool_peripheral/cachepool_peripheral_reg_top.sv
+  - hardware/cachepool_peripheral/cachepool_peripheral.sv
+  # Bootrom
+  - hardware/bootrom/bootrom.sv
+  # Barrier
+  - hardware/src/cachepool_tile_barrier.sv
+  - hardware/src/cachepool_cluster_barrier.sv
   # Level 1
   - hardware/src/cachepool_pkg.sv
   - hardware/src/cachepool_cc.sv
   # Level 2
   - hardware/src/cachepool_tile.sv
   # Level 3
+  - hardware/src/cachepool_group.sv
+
   - hardware/src/cachepool_cluster.sv
   # Level 4
   - hardware/tb/cachepool_cluster_wrapper.sv

Makefile

@@ -41,38 +41,40 @@ SPATZ_DIR             ?= ${DEP_DIR}/spatz
 SPZ_CLS_DIR           ?= ${SPATZ_DIR}/hw/system/spatz_cluster
 ### DramSys
 DRAMSYS_DIR           ?= ${DEP_DIR}/dram_rtl_sim
-DRAMSYS_LIB_PATH      ?= ${DRAMSYS_DIR}/dramsys_lib/DRAMSys/build/lib
-DRAMSYS_RES_PATH      ?= ${DRAMSYS_DIR}/dramsys_lib/DRAMSys/configs
+DRAMSYS_PATH          := ${DRAMSYS_DIR}/dramsys_lib/DRAMSys
+DRAMSYS_LIB_PATH      := ${DRAMSYS_PATH}/build/lib
+DRAMSYS_RES_PATH      := ${DRAMSYS_PATH}/configs
 
 ## Software subpaths
 SPATZ_SW_DIR          ?= ${SPATZ_DIR}/sw
 
 ## Simulation related
 SIM_DIR               ?= ${CACHEPOOL_DIR}/sim
-SIMLIB_DIR            ?= ${SIM_DIR}/simlib              # local c lib for simulation
+SIMLIB_DIR            ?= ${SIM_DIR}/simlib
 SNLIB_DIR             ?= ${SPATZ_DIR}/hw/ip/snitch_test/src
 BOOTLIB_DIR           := ${SPZ_CLS_DIR}/test
 WORK_DIR              := ${SIM_DIR}/work
 SIMBIN_DIR            := ${SIM_DIR}/bin
-DPI_PATH              := ${HARDWARE_DIR}/tb/dpi
+TB_DIR                := ${HARDWARE_DIR}/tb
+DPI_PATH              := ${TB_DIR}/dpi
+DRAM_CFG_PATH         := ${TB_DIR}/dram_config
 DPI_LIB               ?= work-dpi
+DEBUG                 ?= 1
 
 ## Bender usage (binary comes from toolchain.mk install)
 BENDER                ?= ${BENDER_INSTALL_DIR}/bender
-# Guarded to avoid failing before bender is installed
 CACHE_PATH            := $(shell [ -x "$(BENDER)" ] && $(BENDER) path insitu-cache || true)
 
 # Configurations
 CFG_DIR               ?= ${CACHEPOOL_DIR}/config
-config             		?= cachepool_512
+config                ?= cachepool_512
 
 # Compiler choice for SW cmake
 COMPILER              ?= llvm
 
 ############
 #  Bender  #
 ############
-# Optional sanity target to ensure the right bender version is installed
 BENDER_VERSION ?= 0.28.1
 .PHONY: bender check-bender
 bender: $(BENDER_INSTALL_DIR)/bender
@@ -89,7 +91,6 @@ check-bender:
 	  $(MAKE) bender; \
 	fi
 
-# Project-level Bender op
 .PHONY: checkout
 checkout: bender
 	${BENDER} checkout
@@ -98,7 +99,6 @@ checkout: bender
 # Prerequisites #
 #################
 
-# Export make vars so Python sees them (simple and robust)
 .EXPORT_ALL_VARIABLES:
 
 config_mk       := $(abspath $(CACHEPOOL_DIR)/config/config.mk)
@@ -112,23 +112,18 @@ gen-spatz-cfg: $(config_mk) $(HJSON_TEMPLATE) ${CACHEPOOL_DIR}/util/scripts/gen_
 	@mkdir -p $(CFG_DIR)
 	@python3 ${CACHEPOOL_DIR}/util/scripts/gen_spatz_cfg.py --template $(HJSON_TEMPLATE) --out $(HJSON_OUT)
 
-
-# Initialize submodules
 .PHONY: init
 init:
 	git submodule update --init --recursive --jobs=8
 	${BENDER} checkout
 
-# ETH-only quick tool switch (softlink prebuilt toolchains)
 .PHONY: quick-tool
 quick-tool:
 	ln -sf /home/dishen/cachepool-32b/install $(CACHEPOOL_DIR)/install
-# 	ln -sf /usr/scratch2/calanda/diyou/toolchain/cachepool-32b/install $(CACHEPOOL_DIR)/install
 
-# Build bootrom and spatz (depends on opcodes repo being present)
 .PHONY: generate
 generate: update_opcodes gen-spatz-cfg
-	$(MAKE) -C $(SPZ_CLS_DIR) generate bootrom SPATZ_CLUSTER_CFG=${CFG_DIR}/cachepool.hjson
+	$(MAKE) -C $(SPZ_CLS_DIR) generate SPATZ_CLUSTER_CFG=${CFG_DIR}/cachepool.hjson
 
 .PHONY: cache-init
 cache-init:
@@ -138,17 +133,79 @@ else
 	@echo "insitu-cache path unavailable (bender not installed yet?)"
 endif
 
+BOOTROM_DIR := $(HARDWARE_DIR)/bootrom
+SCRIPTS_DIR := $(CACHEPOOL_DIR)/util/scripts
+TPL_DIR     := $(SPATZ_DIR)/hw/system/spatz_cluster/test
+
+.PHONY: bootrom
+bootrom: $(BOOTROM_DIR)/bootrom.sv
+
+$(BOOTROM_DIR)/bootdata_bootrom.cc: $(SCRIPTS_DIR)/generate_bootdata.py $(HJSON_OUT)
+	${PYTHON} $< -c $(HJSON_OUT) -d $(BOOTROM_DIR) -t bootdata_bootrom.cc.tpl -o $@
+
+$(BOOTROM_DIR)/bootdata.cc: $(SCRIPTS_DIR)/generate_bootdata.py $(HJSON_OUT)
+	${PYTHON} $< -c $(HJSON_OUT) -d $(BOOTROM_DIR) -t bootdata.cc.tpl -o $@
+
+$(BOOTROM_DIR)/bootrom.elf $(BOOTROM_DIR)/bootrom.dump $(BOOTROM_DIR)/bootrom.bin: \
+  $(BOOTROM_DIR)/bootrom.S $(BOOTROM_DIR)/bootdata_bootrom.cc $(BOOTROM_DIR)/bootrom.ld Makefile
+	riscv -riscv64-gcc-9.5.0 riscv64-unknown-elf-gcc \
+		-mabi=ilp32 -march=rv32imaf -static -nostartfiles \
+		-T$(BOOTROM_DIR)/bootrom.ld \
+		$(BOOTROM_DIR)/bootrom.S \
+		$(BOOTROM_DIR)/bootdata_bootrom.cc \
+		-I$(SPATZ_DIR)/hw/ip/snitch_test/src \
+		-I$(SOFTWARE_DIR)/snRuntime/include \
+		-o $(BOOTROM_DIR)/bootrom.elf
+	riscv -riscv64-gcc-9.5.0 riscv64-unknown-elf-objdump -D $(BOOTROM_DIR)/bootrom.elf > $(BOOTROM_DIR)/bootrom.dump
+	riscv -riscv64-gcc-9.5.0 riscv64-unknown-elf-objcopy -O binary $(BOOTROM_DIR)/bootrom.elf $(BOOTROM_DIR)/bootrom.bin
+
+$(BOOTROM_DIR)/bootrom.sv: $(BOOTROM_DIR)/bootrom.bin $(BOOTROM_DIR)/bootdata.cc
+	${PYTHON} $(SCRIPTS_DIR)/generate_bootrom.py \
+		$< -c $(HJSON_OUT) --output $@
+
 ###########
 # DramSys #
 ###########
 USE_DRAMSYS ?= 1
 VSIM_FLAGS :=
 VSIM_BENDER =
 
-## Build DramSys
 .PHONY: dram-build
-dram-build:
-	$(MAKE) -BC ${DRAMSYS_DIR} -j8 dramsys CXX=$(CXX) CC=$(CC)
+dram-build: $(DRAMSYS_PATH)/README.md dram-clean dram-config
+	cd $(DRAMSYS_PATH) && \
+	if [ ! -d "build" ]; then \
+		mkdir build && cd build; \
+		$(CMAKE) -DCMAKE_CXX_FLAGS=-fPIC -DCMAKE_C_FLAGS=-fPIC -D DRAMSYS_WITH_DRAMPOWER=ON .. ; \
+		make -j; \
+	fi
+
+$(DRAMSYS_PATH)/README.md: dram-init
+
+dram-config:
+	cp $(DRAM_CFG_PATH)/am_hbm2e_16Gb_pc_brc.json $(DRAMSYS_PATH)/configs/addressmapping/.
+	cp $(DRAM_CFG_PATH)/mc_hbm2e_fr_fcfs_grp.json $(DRAMSYS_PATH)/configs/mcconfig/.
+	cp $(DRAM_CFG_PATH)/ms_hbm2e_16Gb_3600.json $(DRAMSYS_PATH)/configs/memspec/.
+	cp $(DRAM_CFG_PATH)/simconfig_hbm2e.json $(DRAMSYS_PATH)/configs/simconfig/.
+	@if [ -f $(DRAMSYS_PATH)/configs/hbm2-example.json ]; then \
+		mv $(DRAMSYS_PATH)/configs/hbm2-example.json \
+		   $(DRAMSYS_PATH)/configs/hbm2-example.json.ori; \
+	fi
+	cp $(DRAM_CFG_PATH)/HBM2E-3600.json $(DRAMSYS_PATH)/configs/hbm2-example.json
+	cp $(DRAM_CFG_PATH)/am_ddr4.json $(DRAMSYS_PATH)/configs/addressmapping/.
+	cp $(DRAM_CFG_PATH)/simconfig_ddr4.json $(DRAMSYS_PATH)/configs/simconfig/.
+	@if [ -f $(DRAMSYS_PATH)/configs/ddr4-example.json ]; then \
+		mv $(DRAMSYS_PATH)/configs/ddr4-example.json \
+		   $(DRAMSYS_PATH)/configs/ddr4-example.json.ori; \
+	fi
+	cp $(DRAM_CFG_PATH)/DDR4-1866.json $(DRAMSYS_PATH)/configs/ddr4-example.json
+
+dram-clean:
+	if [ -d "$(DRAMSYS_PATH)/build" ]; then \
+		rm -rf $(DRAMSYS_PATH)/build; \
+	fi
+
+dram-init:
+	make -C ${DRAMSYS_DIR} -j8 dramsys CXX=$(CXX) CC=$(CC)
 
 ############
 # Modelsim #
@@ -159,14 +216,8 @@ VSIM        = ${QUESTA_VER} vsim
 VLOG        = ${QUESTA_VER} vlog
 VSIM_HOME   = /usr/pack/${QUESTA_VER}/questasim
 
-# fesvr is built locally into work dir; needs dtc/spike path
-FESVR          ?= ${SIM_DIR}/work
-FESVR_VERSION  ?= c663ea20a53f4316db8cb4d591b1c8e437f4a0c4
-
-VSIM_FLAGS += -sv_lib $(SIM_DIR)/${DPI_LIB}/cachepool_dpi
-VSIM_FLAGS += -t 1ps
-VSIM_FLAGS += -voptargs=+acc
-VSIM_FLAGS += -suppress vsim-3999
+FESVR         ?= ${SIM_DIR}/work
+FESVR_VERSION ?= c663ea20a53f4316db8cb4d591b1c8e437f4a0c4
 
 VLOG_FLAGS += -svinputport=compat
 VLOG_FLAGS += -override_timescale 1ns/1ps
@@ -182,6 +233,7 @@ VLOG_DEFS = -DCACHEPOOL
 
 # Cluster configuration
 VLOG_DEFS += -DNUM_TILES=$(num_tiles)
+VLOG_DEFS += -DNumRemotePortTile=$(num_remote_ports_per_tile)
 VLOG_DEFS += -DNUM_CORES=$(num_cores)
 VLOG_DEFS += -DDATA_WIDTH=$(data_width)
 VLOG_DEFS += -DADDR_WIDTH=$(addr_width)
@@ -198,7 +250,6 @@ VLOG_DEFS += -DL1D_COAL_WINDOW=$(l1d_coal_window)
 VLOG_DEFS += -DL1D_NUM_WAY=$(l1d_num_way)
 VLOG_DEFS += -DL1D_TILE_SIZE=$(l1d_tile_size)
 VLOG_DEFS += -DL1D_TAG_DATA_WIDTH=$(l1d_tag_data_width)
-# derived (no spaces so vlog gets one arg)
 VLOG_DEFS += -DL1D_NUM_BANKS=$(l1d_num_banks)
 VLOG_DEFS += -DL1D_DEPTH=$(l1d_depth)
 
@@ -226,13 +277,11 @@ VLOG_DEFS += -DPERIPH_START_ADDR=$(periph_start_addr)
 VLOG_DEFS += -DBOOT_ADDR=$(boot_addr)
 VLOG_DEFS += -DUART_ADDR=$(uart_addr)
 
-
 ENABLE_CACHEPOOL_TESTS ?= 1
 
 # Bender targets
-VSIM_BENDER   += -t test -t rtl -t simulation -t spatz -t cachepool_test -t cachepool
+VSIM_BENDER += -t test -t rtl -t simulation -t spatz -t cachepool_test -t cachepool
 
-# Include the simulation makefile
 include sim/sim.mk
 
 ######
@@ -244,7 +293,7 @@ clean.sw:
 	rm -rf ${SOFTWARE_DIR}/build
 
 .PHONY: sw
-sw: clean.sw
+sw: generate bootrom clean.sw
 	echo ${SOFTWARE_DIR}
 	mkdir -p ${SOFTWARE_DIR}/build
 	cd ${SOFTWARE_DIR}/build && ${CMAKE} \
@@ -258,7 +307,7 @@ sw: clean.sw
 	  -DBUILD_TESTS=ON .. && $(MAKE)
 
 .PHONY: vsim
-vsim: dpi ${SIMBIN_DIR}/cachepool_cluster.vsim
+vsim: generate bootrom dpi ${SIMBIN_DIR}/cachepool_cluster.vsim
 	echo ${SOFTWARE_DIR}
 	mkdir -p ${SOFTWARE_DIR}/build
 	cd ${SOFTWARE_DIR}/build && ${CMAKE} \
@@ -274,6 +323,11 @@ vsim: dpi ${SIMBIN_DIR}/cachepool_cluster.vsim
 
 .PHONY: clean
 clean: clean.sw clean.vsim
+	rm -rf $(HJSON_OUT) $(BOOTROM_DIR)/bootdata.cc \
+	                    $(BOOTROM_DIR)/bootdata_bootrom.cc \
+	                    $(BOOTROM_DIR)/bootrom.sv \
+	                    $(BOOTROM_DIR)/bootrom.dump \
+	                    $(BOOTROM_DIR)/bootrom.elf
 
 ########
 # Lint #
@@ -289,8 +343,7 @@ lint: ${LINT_PATH}/tmp/files ${LINT_PATH}/sdc/func.sdc ${LINT_PATH}/script/lint.
 ${LINT_PATH}/tmp/files:
 	mkdir -p ${LINT_PATH}/tmp
 	@if [ ! -x "$(BENDER)" ]; then echo "bender not installed; run 'make bender'"; exit 1; fi
-	${BENDER} script verilator  $(VLOG_DEFS) -t rtl -t spatz -t cachepool -t dramsys --define COMMON_CELLS_ASSERTS_OFF > ${LINT_PATH}/tmp/files
-
+	${BENDER} script verilator $(VLOG_DEFS) -t rtl -t spatz -t cachepool -t dramsys --define COMMON_CELLS_ASSERTS_OFF > ${LINT_PATH}/tmp/files
 
 ########
 # Help #
@@ -303,27 +356,39 @@ help:
 	@echo "--------------------------------------------------------------------------------------------------------"
 	@echo "Initialization:"
 	@echo ""
-	@echo "*init*:       clone the git submodules"
-	@echo "*toolchain*:  build the necessary toolchains (LLVM/GCC/Spike)"
-	@echo "*quick-tool*: *ETH Member Only* soft link to prebuilt toolchains"
-	@echo "*generate*:   generate the Spatz package, bootrom and opcodes"
-	@echo "*dram-build*: build DRAMSys for simulation"
+	@echo "*init*:           clone git submodules and run bender checkout"
+	@echo "*bender*:         install the bender dependency manager"
+	@echo "*check-bender*:   verify bender version (>= $(BENDER_VERSION)), reinstall if outdated"
+	@echo "*checkout*:       run bender checkout to fetch hardware dependencies"
+	@echo "*toolchain*:      build the necessary toolchains (LLVM/GCC/Spike) [from toolchain.mk]"
+	@echo "*quick-tool*:     *ETH Member Only* soft link to prebuilt toolchains"
+	@echo "*generate*:       generate the Spatz package and opcodes, and the cluster config HJSON"
+	@echo "*cache-init*:     source the insitu-cache environment (requires bender checkout)"
+	@echo "*bootrom*:        compile and generate the bootrom SystemVerilog module"
+	@echo ""
+	@echo "DRAMSys:"
+	@echo ""
+	@echo "*dram-build*:     build the DRAMSys simulation library"
+	@echo "*dram-clean*:     remove the DRAMSys build directory"
 	@echo ""
 	@echo "SW Build:"
 	@echo ""
-	@echo "*clean.sw*:   remove the current software build"
-	@echo "*sw*:         generate the latest kernel build (will overwrite the previous build)"
+	@echo "*sw*:             build software (generate + bootrom + cmake); overwrites previous build"
+	@echo "*clean.sw*:       remove the software build directory"
 	@echo ""
 	@echo "Simulation:"
 	@echo ""
-	@echo "*clean.vsim*: remove the current hardware build"
-	@echo "*vsim*:       build both the software and hardware (not overwriting prev build by default)"
-	@echo "              USE_DRAMSYS=1 to use DRAMSys (default 1)"
+	@echo "*vsim*:           build hardware and software for QuestaSim simulation"
+	@echo "*clean.vsim*:     remove the hardware simulation build [from sim/sim.mk]"
+	@echo "*clean*:          remove SW build, vsim build, and all generated HW files"
 	@echo ""
 	@echo "Lint:"
-	@echo "*lint*:       run SpyGlass lint (requires bender + SpyGlass in PATH)"
+	@echo ""
+	@echo "*lint*:           run SpyGlass lint (requires bender + SpyGlass in PATH)"
 	@echo ""
 	@echo "--------------------------------------------------------------------------------------------------------"
-	@echo "Settings"
-	@echo "*CMAKE*:      CMake version needs to be >= 3.28 for DRAMSys"
+	@echo "Settings:"
+	@echo "*config*:         cluster configuration name (default: $(config))"
+	@echo "*CMAKE*:          CMake binary (default: $(CMAKE)); must be >= 3.28 for DRAMSys"
+	@echo "*DEBUG*:          enable +acc for waveform visibility in vsim (default: $(DEBUG))"
 	@echo ""