Skip to content

Commit

Permalink
Remove dynamic tlb config from constructor
Browse files Browse the repository at this point in the history
  • Loading branch information
@pjanevskiTT
pjanevskiTT committed Oct 8, 2024
1 parent 87d280d commit 14a5cf5
Show file tree
Hide file tree
Showing 14 changed files with 52 additions and 109 deletions.
1 change: 1 addition & 0 deletions device/architecture_implementation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class architecture_implementation {
virtual uint32_t get_grid_size_x() const = 0;
virtual uint32_t get_grid_size_y() const = 0;
virtual uint32_t get_tlb_cfg_reg_size_bytes() const = 0;
virtual uint32_t get_small_read_write_tlb() const = 0;
// Replace with std::span once we enable C++20
virtual const std::vector<uint32_t>& get_harvesting_noc_locations() const = 0;
virtual const std::vector<uint32_t>& get_t6_x_locations() const = 0;
Expand Down
1 change: 1 addition & 0 deletions device/blackhole/blackhole_implementation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,7 @@ class blackhole_implementation : public architecture_implementation {
uint32_t get_grid_size_x() const override { return blackhole::GRID_SIZE_X; }
uint32_t get_grid_size_y() const override { return blackhole::GRID_SIZE_Y; }
uint32_t get_tlb_cfg_reg_size_bytes() const override { return blackhole::TLB_CFG_REG_SIZE_BYTES; }
uint32_t get_small_read_write_tlb() const override { return blackhole::MEM_SMALL_READ_WRITE_TLB; }
const std::vector<uint32_t>& get_harvesting_noc_locations() const override {
return blackhole::HARVESTING_NOC_LOCATIONS;
}
Expand Down
1 change: 1 addition & 0 deletions device/grayskull/grayskull_implementation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ class grayskull_implementation : public architecture_implementation {
uint32_t get_grid_size_x() const override { return grayskull::GRID_SIZE_X; }
uint32_t get_grid_size_y() const override { return grayskull::GRID_SIZE_Y; }
uint32_t get_tlb_cfg_reg_size_bytes() const override { return grayskull::TLB_CFG_REG_SIZE_BYTES; }
uint32_t get_small_read_write_tlb() const override { return grayskull::MEM_SMALL_READ_WRITE_TLB; }
const std::vector<uint32_t>& get_harvesting_noc_locations() const override {
return grayskull::HARVESTING_NOC_LOCATIONS;
}
Expand Down
4 changes: 2 additions & 2 deletions device/tt_device.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -609,8 +609,8 @@ class tt_SiliconDevice: public tt_device
* @param simulated_harvesting_masks
*/
tt_SiliconDevice(const std::string &sdesc_path, const std::string &ndesc_path = "", const std::set<chip_id_t> &target_devices = {},
const uint32_t &num_host_mem_ch_per_mmio_device = 1, const std::unordered_map<std::string, std::int32_t>& dynamic_tlb_config_ = {},
const bool skip_driver_allocs = false, const bool clean_system_resources = false, bool perform_harvesting = true, std::unordered_map<chip_id_t, uint32_t> simulated_harvesting_masks = {});
const uint32_t &num_host_mem_ch_per_mmio_device = 1, const bool skip_driver_allocs = false,
const bool clean_system_resources = false, bool perform_harvesting = true, std::unordered_map<chip_id_t, uint32_t> simulated_harvesting_masks = {});

//Setup/Teardown Functions
virtual std::unordered_map<chip_id_t, tt_SocDescriptor>& get_virtual_soc_descriptors();
Expand Down
10 changes: 6 additions & 4 deletions device/tt_silicon_driver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -580,8 +580,8 @@ std::unordered_map<chip_id_t, uint32_t> tt_SiliconDevice::get_harvesting_masks_f
}

tt_SiliconDevice::tt_SiliconDevice(const std::string &sdesc_path, const std::string &ndesc_path, const std::set<chip_id_t> &target_devices,
const uint32_t &num_host_mem_ch_per_mmio_device, const std::unordered_map<std::string, std::int32_t>& dynamic_tlb_config_,
const bool skip_driver_allocs, const bool clean_system_resources, bool perform_harvesting, std::unordered_map<chip_id_t, uint32_t> simulated_harvesting_masks) : tt_device(sdesc_path) {
const uint32_t &num_host_mem_ch_per_mmio_device, const bool skip_driver_allocs,
const bool clean_system_resources, bool perform_harvesting, std::unordered_map<chip_id_t, uint32_t> simulated_harvesting_masks) : tt_device(sdesc_path) {
std::unordered_set<chip_id_t> target_mmio_device_ids;
target_devices_in_cluster = target_devices;
arch_name = tt_SocDescriptor(sdesc_path).arch;
Expand All @@ -590,8 +590,9 @@ tt_SiliconDevice::tt_SiliconDevice(const std::string &sdesc_path, const std::str
auto available_device_ids = detect_available_device_ids();
m_num_pci_devices = available_device_ids.size();

if (!skip_driver_allocs)
if (!skip_driver_allocs) {
log_info(LogSiliconDriver, "Detected {} PCI device{} : {}", m_num_pci_devices, (m_num_pci_devices > 1) ? "s":"", available_device_ids);
}

if (ndesc_path == "") {
ndesc = tt_ClusterDescriptor::create_for_grayskull_cluster(target_devices, available_device_ids);
Expand All @@ -608,12 +609,13 @@ tt_SiliconDevice::tt_SiliconDevice(const std::string &sdesc_path, const std::str
target_remote_chips.insert(d);
}
}
dynamic_tlb_config = dynamic_tlb_config_;

// It is mandatory for all devices to have these TLBs set aside, as the driver needs them to issue remote reads and writes.
auto architecture_implementation = tt::umd::architecture_implementation::create(static_cast<tt::umd::architecture>(arch_name));
dynamic_tlb_config["LARGE_READ_TLB"] = architecture_implementation->get_mem_large_read_tlb();
dynamic_tlb_config["LARGE_WRITE_TLB"] = architecture_implementation->get_mem_large_write_tlb();
dynamic_tlb_config["REG_TLB"] = architecture_implementation->get_reg_tlb();
dynamic_tlb_config["SMALL_READ_WRITE_TLB"] = architecture_implementation->get_small_read_write_tlb();

for(const auto& tlb : dynamic_tlb_config) {
dynamic_tlb_ordering_modes.insert({tlb.first, TLB_DATA::Relaxed}); // All dynamic TLBs use Relaxed Ordering by default; MT: Good for BH
Expand Down
1 change: 1 addition & 0 deletions device/wormhole/wormhole_implementation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ class wormhole_implementation : public architecture_implementation {
uint32_t get_grid_size_x() const override { return wormhole::GRID_SIZE_X; }
uint32_t get_grid_size_y() const override { return wormhole::GRID_SIZE_Y; }
uint32_t get_tlb_cfg_reg_size_bytes() const override { return wormhole::TLB_CFG_REG_SIZE_BYTES; }
uint32_t get_small_read_write_tlb() const override { return wormhole::MEM_SMALL_READ_WRITE_TLB; }
const std::vector<uint32_t>& get_harvesting_noc_locations() const override {
return wormhole::HARVESTING_NOC_LOCATIONS;
}
Expand Down
3 changes: 1 addition & 2 deletions tests/blackhole/test_bh_common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,7 @@ class BlackholeTestFixture : public ::testing::Test {
std::iota(devices.begin(), devices.end(), 0);
std::set<chip_id_t> target_devices = {devices.begin(), devices.end()};
uint32_t num_host_mem_ch_per_mmio_device = 1;
std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {}; // Don't set any dynamic TLBs in this test
device = std::make_unique<tt_SiliconDevice>(test_utils::GetAbsPath(SOC_DESC_PATH), test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true);
device = std::make_unique<tt_SiliconDevice>(test_utils::GetAbsPath(SOC_DESC_PATH), test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);
assert(device != nullptr);
assert(device->get_cluster_description()->get_number_of_chips() == get_detected_num_chips());

Expand Down
37 changes: 11 additions & 26 deletions tests/blackhole/test_silicon_driver_bh.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,10 @@ std::set<chip_id_t> get_target_devices() {
TEST(SiliconDriverBH, CreateDestroy) {
std::set<chip_id_t> target_devices = get_target_devices();
uint32_t num_host_mem_ch_per_mmio_device = 1;
std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {}; // Don't set any dynamic TLBs in this test
tt_device_params default_params;
// Initialize the driver with a 1x1 descriptor and explictly do not perform harvesting
for(int i = 0; i < 50; i++) {
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, false);
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, false, true, false);
set_params_for_remote_txn(device);
device.start_device(default_params);
device.deassert_risc_reset();
Expand All @@ -104,9 +103,8 @@ TEST(SiliconDriverBH, CreateDestroy) {
// }
// }

// std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {}; // Don't set any dynamic TLBs in this test
// uint32_t num_host_mem_ch_per_mmio_device = 1;
// tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_8x10.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true, simulated_harvesting_masks);
// tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_8x10.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true, simulated_harvesting_masks);
// auto sdesc_per_chip = device.get_virtual_soc_descriptors();

// ASSERT_EQ(device.using_harvested_soc_descriptors(), true) << "Expected Driver to have performed harvesting";
Expand All @@ -128,10 +126,9 @@ TEST(SiliconDriverBH, CreateDestroy) {
// }
// }

// std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {}; // Don't set any dynamic TLBs in this test
// uint32_t num_host_mem_ch_per_mmio_device = 1;
// // Initialize the driver with a 1x1 descriptor and explictly do not perform harvesting
// tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_1x1.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, false, simulated_harvesting_masks);
// tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_1x1.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, false, true, false, simulated_harvesting_masks);
// auto sdesc_per_chip = device.get_virtual_soc_descriptors();

// ASSERT_EQ(device.using_harvested_soc_descriptors(), false) << "SOC descriptors should not be modified when harvesting is disabled";
Expand All @@ -156,9 +153,8 @@ TEST(SiliconDriverBH, CreateDestroy) {
// }

// uint32_t num_host_mem_ch_per_mmio_device = 1;
// std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {{"SMALL_READ_WRITE_TLB", 157}}; // Use both static and dynamic TLBs here

// tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_8x10.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true, simulated_harvesting_masks);
// tt_SiliconDevice device = tt_SiliconDevice("./tests/soc_descs/wormhole_b0_8x10.yaml", test_utils::GetClusterDescYAML(), target_devices, num_host_mem_ch_per_mmio_device, false, true, true, simulated_harvesting_masks);
// set_params_for_remote_txn(device);
// auto mmio_devices = device.get_target_mmio_device_ids();

Expand Down Expand Up @@ -219,11 +215,9 @@ TEST(SiliconDriverBH, UnalignedStaticTLB_RW) {

std::set<chip_id_t> target_devices = get_target_devices();

std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {}; // Don't set any dynamic TLBs in this test
dynamic_tlb_config["REG_TLB"] = 184;
uint32_t num_host_mem_ch_per_mmio_device = 1;

tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true);
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);
set_params_for_remote_txn(device);
auto mmio_devices = device.get_target_mmio_device_ids();

Expand Down Expand Up @@ -281,10 +275,9 @@ TEST(SiliconDriverBH, StaticTLB_RW) {

std::set<chip_id_t> target_devices = get_target_devices();

std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {}; // Don't set any dynamic TLBs in this test
uint32_t num_host_mem_ch_per_mmio_device = 1;

tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true);
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);
set_params_for_remote_txn(device);
auto mmio_devices = device.get_target_mmio_device_ids();

Expand Down Expand Up @@ -334,10 +327,8 @@ TEST(SiliconDriverBH, DynamicTLB_RW) {
// Don't use any static TLBs in this test. All writes go through a dynamic TLB that needs to be reconfigured for each transaction
std::set<chip_id_t> target_devices = get_target_devices();

std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {};
uint32_t num_host_mem_ch_per_mmio_device = 1;
dynamic_tlb_config.insert({"DYNAMIC_TLB_BASE_INDEX", 190}); // Use this for all reads and writes to worker cores
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true);
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);

set_params_for_remote_txn(device);

Expand Down Expand Up @@ -400,10 +391,8 @@ TEST(SiliconDriverBH, MultiThreadedDevice) {

std::set<chip_id_t> target_devices = get_target_devices();

std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {};
uint32_t num_host_mem_ch_per_mmio_device = 1;
dynamic_tlb_config.insert({"SMALL_READ_WRITE_TLB", 157}); // Use this for all reads and writes to worker cores
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true);
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);

set_params_for_remote_txn(device);

Expand Down Expand Up @@ -460,11 +449,9 @@ TEST(SiliconDriverBH, MultiThreadedMemBar) {

std::set<chip_id_t> target_devices = get_target_devices();
uint32_t base_addr = l1_mem::address_map::DATA_BUFFER_SPACE_BASE;
std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {};
dynamic_tlb_config.insert({"SMALL_READ_WRITE_TLB", 157}); // Use this for reading back membar values
uint32_t num_host_mem_ch_per_mmio_device = 1;

tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true);
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);
set_params_for_remote_txn(device);
for(int i = 0; i < target_devices.size(); i++) {
// Iterate over devices and only setup static TLBs for functional worker cores
Expand Down Expand Up @@ -564,10 +551,9 @@ TEST(SiliconDriverBH, DISABLED_BroadcastWrite) { // Cannot broadcast to tensix/e
// Broadcast multiple vectors to tensix and dram grid. Verify broadcasted data is read back correctly
std::set<chip_id_t> target_devices = get_target_devices();

std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {}; // Don't set any dynamic TLBs in this test
uint32_t num_host_mem_ch_per_mmio_device = 1;

tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true);
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);
set_params_for_remote_txn(device);
auto mmio_devices = device.get_target_mmio_device_ids();

Expand Down Expand Up @@ -622,10 +608,9 @@ TEST(SiliconDriverBH, DISABLED_VirtualCoordinateBroadcast) { // same problem as
// Broadcast multiple vectors to tensix and dram grid. Verify broadcasted data is read back correctly
std::set<chip_id_t> target_devices = get_target_devices();

std::unordered_map<std::string, std::int32_t> dynamic_tlb_config = {}; // Don't set any dynamic TLBs in this test
uint32_t num_host_mem_ch_per_mmio_device = 1;

tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, dynamic_tlb_config, false, true, true);
tt_SiliconDevice device = tt_SiliconDevice(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"), test_utils::GetAbsPath("blackhole_1chip_cluster.yaml"), target_devices, num_host_mem_ch_per_mmio_device, false, true, true);
set_params_for_remote_txn(device);
auto mmio_devices = device.get_target_mmio_device_ids();

Expand Down
Loading

0 comments on commit 14a5cf5

Please sign in to comment.