From 9f5ca2349c14e7a801a599baaf39c24483359ad4 Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Tue, 13 May 2025 00:46:58 -0700 Subject: [PATCH 1/8] aie debugger support --- src/shim/umq/aiedbg.h | 11 ++ src/shim/umq/dbg_cmd.h | 34 ++++++ src/shim/umq/dbg_hwq.cpp | 95 ++++++++++++++++ src/shim/umq/dbg_hwq.h | 35 ++++++ src/shim/umq/hwctx.cpp | 42 +++++-- src/shim/umq/hwctx.h | 44 +++++--- src/shim/umq/tcp_server.cpp | 217 ++++++++++++++++++++++++++++++++++++ src/shim/umq/tcp_server.h | 28 +++++ 8 files changed, 478 insertions(+), 28 deletions(-) create mode 100644 src/shim/umq/aiedbg.h create mode 100644 src/shim/umq/dbg_cmd.h create mode 100644 src/shim/umq/dbg_hwq.cpp create mode 100644 src/shim/umq/dbg_hwq.h create mode 100644 src/shim/umq/tcp_server.cpp create mode 100644 src/shim/umq/tcp_server.h diff --git a/src/shim/umq/aiedbg.h b/src/shim/umq/aiedbg.h new file mode 100644 index 00000000..853155a0 --- /dev/null +++ b/src/shim/umq/aiedbg.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. + +//definitions for aie debugger backend and frontend communication +#define ATTACH_CMD 1 +#define READ_MEM_CMD 2 +#define WRITE_MEM_CMD 3 +#define DETACH_CMD 0xffff + +#define AIE_DBG_SUCCESS 0 +#define AIE_DBG_NOT_ATTACHED 0xffff diff --git a/src/shim/umq/dbg_cmd.h b/src/shim/umq/dbg_cmd.h new file mode 100644 index 00000000..ff89e530 --- /dev/null +++ b/src/shim/umq/dbg_cmd.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#ifndef _DBG_CMD_H_ +#define _DBG_CMD_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define DBG_PKT_SUCCESS (1) +#define DBG_PKT_EXIT (2) +#define DBG_PKT_INVALID (2) + +enum dbg_packet_opcode; +{ + DBG_CMD_TEST = 10, + DBG_CMD_EXIT = 11, + DBG_CMD_READ = 12, + DBG_CMD_WRITE = 13, +}; + +struct rw_mem +{ + uint32_t aie_addr; + uint32_t length; + uint32_t host_addr_high; + uint32_t host_addr_low; +}; +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/shim/umq/dbg_hwq.cpp b/src/shim/umq/dbg_hwq.cpp new file mode 100644 index 00000000..c8cf939a --- /dev/null +++ b/src/shim/umq/dbg_hwq.cpp @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include "dbg_hwq.h" + +namespace shim_xdna { + +dbg_hwq_umq:: +dbg_hwq_umq(const device& dev) + : m_pdev(device.get_pdev()) +{ + const size_t header_sz = sizeof(struct host_queue_header); + const size_t queue_sz = sizeof(struct host_queue_packet); + const size_t comp_sz = sizeof(uint32_t); + + const size_t umq_sz = header_sz + queue_sz + comp_sz; + + shim_debug("dbg umq sz %ld", umq_sz); + + m_dbg_umq_bo = std::make_unique(m_pdev, umq_sz, AMDXDNA_BO_CMD); + m_dbg_umq_bo_buf = m_dbg_umq_bo->vaddr(); + m_dbg_umq_addr = m_dbg_umq_bo->paddr(); + m_dbg_umq_hdr = reinterpret_cast(m_dbg_umq_bo_buf); + m_dbg_umq_pkt = reinterpret_cast + ((char *)m_dbg_umq_bo_buf + header_sz); + m_dbg_umq_comp = m_dbg_umq_bo->paddr() + header_sz + queue_sz; + m_dbg_umq_comp_ptr = reinterpret_cast(m_dbg_umq_comp); + + // set all mapped memory to 0 + std::memset(m_dbg_umq_bo_buf, 0, umq_sz); + m_dbg_umq_pkt->completion_signal = m_dbg_umq_comp; + + m_dbg_umq_pkt->xrt_header.common_header.type = HOST_QUEUE_PACKET_TYPE_INVALID; + m_dbg_umq_hdr->capacity = 1; + + shim_debug("Created DBG UMQ HW queue"); +} + +dbg_hwq_umq:: +~dbg_hwq_umq() +{ + shim_debug("Destroying DBG UMA HW queue"); +} + +uint32_t +dbg_hwq_umq:: +issue_exit_cmd() +{ + auto hdr = m_dbg_umq_pkt->xrt_header; + // always case 1 + m_dbg_umq_hdr->write_index++; + struct xrt_packet_header *ehp = &m_dbg_umq_pkt->xrt_header; + ehp->common_header.opcode = DBG_CMD_EXIT; + ehp->common_header.count = 0; + + submit(); +} + +uint32_t +dbg_hwq_umq:: +issue_rw_cmd(struct rw_mem &data, uint16_t opcode) +{ + auto hdr = m_dbg_umq_pkt->xrt_header; + // always case 1 + m_dbg_umq_hdr->write_index++; + struct xrt_packet_header *ehp = &m_dbg_umq_pkt->xrt_header; + ehp->common_header.opcode = opcode; + ehp->common_header.count = sizeof (struct rw_mem); + + struct rw_mem *rwp = reinterpret_cast(m_dbg_umq_pkt->data); + std::memcpy(rwp, &data, sizeof(struct rw_mem)); + + submit(); +} + +uint32_t +dbg_hwq_umq:: +submit() +{ + *m_dbg_umq_comp_ptr = 0; + + /* Issue mfence instruction to make sure all writes to the slot before is done */ + std::atomic_thread_fence(std::memory_order::memory_order_seq_cst); + m_dbg_umq_pkt->xrt_header.common_header.type = HOST_QUEUE_PACKET_TYPE_VENDOR_SPECIFIC; + + while (1) + { + if (*m_dbg_umq_comp_ptr) + { + return (*m_dbg_umq_comp_ptr); + } + } +} + +} // shim_xdna diff --git a/src/shim/umq/dbg_hwq.h b/src/shim/umq/dbg_hwq.h new file mode 100644 index 00000000..d00c7211 --- /dev/null +++ b/src/shim/umq/dbg_hwq.h @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#ifndef DBG_HWQ_UMQ_H +#define DBG_HWQ_UMQ_H + +#include "host_queue.h" +#include "dbg_cmd.h" +#include "aiedbg.h" + +namespace shim_xdna { + +class dbg_hwq_umq +{ +public: + dbg_hwq_umq(const device& device); + ~dbg_hwq_umq(); + +private: + const pdev& m_pdev; + void *m_dbg_umq_bo_buf; + uint64_t m_dbg_umq_addr; + volatile struct host_queue_header *m_dbg_umq_hdr = nullptr; + volatile struct host_queue_packet *m_dbg_umq_pkt = nullptr; + uint64_t m_dbg_umq_comp; + volatile uint32_t *m_dbg_umq_comp_ptr = nullptr; + + uint32_t issue_exit_cmd(); + uint32_t issue_rw_cmd(struct rw_mem &data, uint16_t opcode); + uint32_t submit(); +}; + +} + +#endif diff --git a/src/shim/umq/hwctx.cpp b/src/shim/umq/hwctx.cpp index ee263287..abac6905 100644 --- a/src/shim/umq/hwctx.cpp +++ b/src/shim/umq/hwctx.cpp @@ -9,11 +9,12 @@ namespace shim_xdna { hwctx_umq:: hwctx_umq(const device& device, const xrt::xclbin& xclbin, const qos_type& qos) : hwctx(device, qos, xclbin, std::make_unique(device, 8)) - , m_metadata() + , m_log_metadata() { xclbin_parser xp(xclbin); m_col_cnt = xp.get_column_cnt(); + init_tcp_server(device); init_log_buf(); // TODO: configure log BO on the hwctx shim_debug("Created UMQ HW context (%d)", get_slotidx()); @@ -23,6 +24,7 @@ hwctx_umq:: ~hwctx_umq() { shim_debug("Destroying UMQ HW context (%d)...", get_slotidx()); + fini_tcp_server(); // TODO: unconfigure log BO on the hwctx fini_log_buf(); } @@ -32,14 +34,14 @@ hwctx_umq:: init_log_buf() { size_t column_size = 1024; - auto log_buf_size = m_col_cnt * column_size + sizeof(m_metadata); + auto log_buf_size = m_col_cnt * column_size + sizeof(m_log_metadata); auto log_bo = alloc_bo(log_buf_size, XCL_BO_FLAGS_EXECBUF); m_log_bo = std::unique_ptr(static_cast(log_bo.release())); m_log_buf = m_log_bo->vaddr(); uint64_t bo_paddr = m_log_bo->paddr(); set_metadata(m_col_cnt, column_size, bo_paddr, UMQ_LOG_BUFFER); std::memset(m_log_buf, 0, log_buf_size); - std::memcpy(m_log_buf, &m_metadata, sizeof(m_metadata)); + std::memcpy(m_log_buf, &m_log_metadata, sizeof(m_log_metadata)); } void @@ -54,14 +56,34 @@ hwctx_umq:: set_metadata(int num_ucs, size_t size, uint64_t bo_paddr, enum umq_log_flag flag) { const uint32_t LOG_MAGIC_NO = 0x43455254; - m_metadata.magic_no = LOG_MAGIC_NO; - m_metadata.major = 0; - m_metadata.minor = 1; - m_metadata.umq_log_flag = flag; - m_metadata.num_ucs = num_ucs; + m_log_metadata.magic_no = LOG_MAGIC_NO; + m_log_metadata.major = 0; + m_log_metadata.minor = 1; + m_log_metadata.umq_log_flag = flag; + m_log_metadata.num_ucs = num_ucs; for (int i = 0; i < num_ucs; i++) { - m_metadata.uc_paddr[i] = bo_paddr + size * i + sizeof(m_metadata); - m_metadata.uc_size[i] = size; + m_log_metadata.uc_paddr[i] = bo_paddr + size * i + sizeof(m_log_metadata); + m_log_metadata.uc_size[i] = size; + } +} + +void +hwctx_umq:: +init_tcp_server(const device& dev) +{ + //check xrt.ini to start tcp server + m_tcp_server(dev); + m_thread_(m_tcp_server::start); +} + +void +hwctx_umq:: +fini_tcp_server() +{ + if (m_thread_.joinable()) + { + pthread_kill(m_thread_.native_handle(), SIGINT); + m_thread_.join(); } } diff --git a/src/shim/umq/hwctx.h b/src/shim/umq/hwctx.h index 5674adae..25280edd 100644 --- a/src/shim/umq/hwctx.h +++ b/src/shim/umq/hwctx.h @@ -6,9 +6,27 @@ #include "../hwctx.h" #include "../buffer.h" +#include "tcp_server.h" namespace shim_xdna { +enum umq_log_flag { + UMQ_DEBUG_BUFFER = 0, + UMQ_TRACE_BUFFER, + UMQ_DBG_QUEUE, + UMQ_LOG_BUFFER +}; + +struct umq_fw_metadata { + uint32_t magic_no; + uint8_t major; + uint8_t minor; + uint8_t umq_log_flag; + uint8_t num_ucs; // how many valid ucs, up to 8 for now + uint64_t uc_paddr[8]; // device accessible address array for each valid uc + uint32_t uc_size[8]; // bo size for each valid uc +}; + class hwctx_umq : public hwctx { public: hwctx_umq(const device& device, const xrt::xclbin& xclbin, const qos_type& qos); @@ -18,26 +36,16 @@ class hwctx_umq : public hwctx { std::unique_ptr m_log_bo; uint32_t m_col_cnt = 0; - enum umq_log_flag { - UMQ_DEBUG_BUFFER = 0, - UMQ_TRACE_BUFFER, - UMQ_DBG_QUEUE, - UMQ_LOG_BUFFER - }; - - struct umq_log_metadata { - uint32_t magic_no; - uint8_t major; - uint8_t minor; - uint8_t umq_log_flag; - uint8_t num_ucs; // how many valid ucs, up to 8 for now - uint64_t uc_paddr[8]; // device accessible address array for each valid uc - uint32_t uc_size[8]; // bo size for each valid uc - }; - - struct umq_log_metadata m_metadata; + umq_fw_metadata m_debugger_metadata; + umq_fw_metadata m_log_metadata; void *m_log_buf = nullptr; + tcp_server m_tcp_server; + std::thread m_thread_; + + void init_tcp_server(const device& dev); + void fini_tcp_server(); + void init_log_buf(); void fini_log_buf(); void set_metadata(int num_cols, size_t size, uint64_t bo_paddr, enum umq_log_flag flag); diff --git a/src/shim/umq/tcp_server.cpp b/src/shim/umq/tcp_server.cpp new file mode 100644 index 00000000..23b03c9a --- /dev/null +++ b/src/shim/umq/tcp_server.cpp @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#include +#include +#include +#include + +#include "tcp_server.h" + +namespace shim_xdna { + +tcp_server:: +tcp_server(const device &dev) : +m_aie_attached(false), m_dbg_umq(dev) {}, m_def_size(16) +{ + auto def_buf_size = m_def_size * sizeof(uint32_t); + auto def_bo = alloc_bo(def_buf_size, XCL_BO_FLAGS_EXECBUF); + m_def_bo = std::unique_ptr(static_cast(def_bo.release())); + m_def_buf = m_def_bo->vaddr(); + m_def_paddr = m_def_bo->paddr(); +} + +tcp_server:: +~tcp_server() +{ +} + +void +tcp_server:: +start() +{ + int serverSocket = socket(AF_INET, SOCK_STREAM, 0); + + // specifying the address + sockaddr_in serverAddress; + serverAddress.sin_family = AF_INET; + serverAddress.sin_port = htons(6666); + serverAddress.sin_addr.s_addr = INADDR_ANY; + + // binding socket. + bind(serverSocket, (struct sockaddr*)&serverAddress, sizeof(serverAddress)); + + // listening to the assigned socket + // we allow only one debugger running + listen(serverSocket, 1); + + while (1) + { + shim_debug("Waiting for incoming connection...\n"); + + // accepting connection request + int clientSocket = accept(serverSocket, nullptr, nullptr); + if (clientSocket < 0) + { + if (errno == EINTR) + { + shim_debug("Tcp thread exit!\n"); + break; + } + } + + bool loop = true; + while (loop) + { + int length = 0; + recv(clientSocket, &length, sizeof(int), 0); + + if (!length) + { + break; + } + std::vector buffer(length >> 2); + recv(clientSocket, buffer.data(), length, 0); + switch (buffer[0]) + { + case ATTACH_CMD: + { + uint32_t status = handle_attach(); + std::vector ret; + ret.push_back(sizeof(uint32_t)); + ret.push_back(status); + send(clientSocket, ret.data(), ret[0] + sizeof(uint32_t), 0); + break; + } + case READ_MEM_CMD: + { + auto data = handle_read_mem(buffer[1], buffer[2]); + std::vector ret; + ret.push_back(sizeof(uint32_t) * (buffer[2] + 1)); + ret.insert(ret.end(), data->begin(), data->end()); + + send(clientSocket, ret.data(), ret[0] + sizeof(uint32_t), 0); + break; + } + case WRITE_MEM_CMD: + { + std::vector data = {buffer.begin() + 2, buffer.end()}; + uint32_t status = handle_write_mem(buffer[1], data); + std::vector ret; + ret.push_back(sizeof(uint32_t)); + ret.push_back(status); + send(clientSocket, ret.data(), ret[0] + sizeof(uint32_t), 0); + break; + } + case DETACH_CMD: + handle_detach(); + loop = false; + break; + default: + break; + } + } + // closing the client socket. + close(clientSocket); + } + + // closing the server socket. + close(serverSocket); +} + +std::unique_ptr> +tcp_server:: +handle_read_mem(uint32_t addr, uint32_t length) +{ + //we return one extra word to front end + //1st word returned is the status + std::vector data(length + 1); + + if (!m_aie_attached) + { + data[0] = AIE_DBG_NOT_ATTACHED; + return std::make_unique>(data); + } + + if (length > m_def_size) + { + buffer_extend(length); + } + + struct rw_mem rw; + rw.host_addr_high = m_def_paddr >> 32; + rw.host_addr_low = m_def_paddr & 0xffffffff; + rw.aie_addr = addr; + rw.length = length; + + uint32_t ret = m_dbg_umq.issue_rw_cmd(rw, DBG_CMD_READ); + if (ret != DBG_PKT_SUCCESS) + { + data[0] = ret; + } + else + { + data[0] = AIE_DBG_SUCCESS; + std::memcpy(data.data() + 1, m_def_buf, length * sizeof (uint32_t)); + } + + return std::make_unique>(data); +} + +uint32_t +tcp_server:: +handle_write_mem(uint32_t addr, std::vector &data) +{ + if (!m_aie_attached) + { + return AIE_DBG_NOT_ATTACHED; + } + + if (data.size() > m_def_size) + { + buffer_extend(data.size()); + } + + struct rw_mem rw; + rw.host_addr_high = m_def_paddr >> 32; + rw.host_addr_low = m_def_paddr & 0xffffffff; + rw.aie_addr = addr; + rw.length = data.size(); + + std::memcpy(m_def_buf, data.data(), data.size() * sizeof(uint32_t)); + uint32_t ret = m_dbg_umq.issue_rw_cmd(rw, DBG_CMD_WRITE); + + return ret != DBG_PKT_SUCCESS ? ret : AIE_DBG_SUCCESS; +} + +void +tcp_server:: +buffer_extend(size_t new_size) +{ + auto n_buf_size = new_size * sizeof(uint32_t); + auto n_bo = alloc_bo(n_buf_size, XCL_BO_FLAGS_EXECBUF); + m_def_bo = std::move(std::unique_ptr(static_cast(n_bo.release()))); + m_def_buf = m_def_bo->vaddr(); + m_def_paddr = m_def_bo->paddr(); +} + +uint32_t +tcp_server:: +handle_attach() +{ + // issue ioctl to attach the dbg hsa queue + // send a DBG_CMD_TEST opcode + m_aie_attached = true; + return AIE_DBG_SUCCESS; +} + +void +tcp_server:: +handle_detach() +{ + issue_exit_cmd(); + // issue ioctl to detach the dbg hsa queue + m_aie_attached = false; +} + +} // shim_xdna diff --git a/src/shim/umq/tcp_server.h b/src/shim/umq/tcp_server.h new file mode 100644 index 00000000..e9e4593a --- /dev/null +++ b/src/shim/umq/tcp_server.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. + +#ifndef TCP_SERVER_H +#define TCP_SERVER_H + +#include "dbg_hwq.h" +#include "../buffer.h" + +namespace shim_xdna { + +class tcp_server { +public: + tcp_server(); + ~tcp_server(); + +private: + dbg_hwq_umq m_dbg_umq; + uint32_t m_def_size; + std::unique_ptr m_def_bo; + volatile void *m_def_buf; + uint64_t m_def_addr; + bool m_aie_attached; +}; + +} + +#endif From 15d28113d1b1910484338e80c194ea08768556a3 Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Wed, 14 May 2025 15:02:15 -0700 Subject: [PATCH 2/8] some fixes --- src/shim/umq/tcp_server.cpp | 2 +- src/shim/umq/tcp_server.h | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/shim/umq/tcp_server.cpp b/src/shim/umq/tcp_server.cpp index 23b03c9a..1f50e02e 100644 --- a/src/shim/umq/tcp_server.cpp +++ b/src/shim/umq/tcp_server.cpp @@ -209,7 +209,7 @@ void tcp_server:: handle_detach() { - issue_exit_cmd(); + m_dbg_umq.issue_exit_cmd(); // issue ioctl to detach the dbg hsa queue m_aie_attached = false; } diff --git a/src/shim/umq/tcp_server.h b/src/shim/umq/tcp_server.h index e9e4593a..1827b576 100644 --- a/src/shim/umq/tcp_server.h +++ b/src/shim/umq/tcp_server.h @@ -13,6 +13,7 @@ class tcp_server { public: tcp_server(); ~tcp_server(); + void start(); private: dbg_hwq_umq m_dbg_umq; @@ -21,6 +22,13 @@ class tcp_server { volatile void *m_def_buf; uint64_t m_def_addr; bool m_aie_attached; + + std::unique_ptr> + handle_read_mem(uint32_t addr, uint32_t length); + uint32_t handle_write_mem(uint32_t addr, std::vector &data); + void buffer_extend(size_t new_size); + void handle_attach(); + void handle_detach(); }; } From d27b8653e57fbaad07881782183e25bb27b7f79f Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Thu, 15 May 2025 23:53:03 -0700 Subject: [PATCH 3/8] unit test passed --- src/shim/umq/aiedbg.h | 35 +++++++++++++++++ src/shim/umq/dbg_cmd.h | 2 +- src/shim/umq/dbg_hwq.cpp | 40 +++++++++++++------ src/shim/umq/dbg_hwq.h | 13 ++++-- src/shim/umq/fw_buf_metadata.h | 29 ++++++++++++++ src/shim/umq/hwctx.cpp | 17 ++++---- src/shim/umq/hwctx.h | 26 +++--------- src/shim/umq/tcp_server.cpp | 72 +++++++++++++++++++++------------- src/shim/umq/tcp_server.h | 17 +++++--- 9 files changed, 171 insertions(+), 80 deletions(-) create mode 100644 src/shim/umq/fw_buf_metadata.h diff --git a/src/shim/umq/aiedbg.h b/src/shim/umq/aiedbg.h index 853155a0..82b71561 100644 --- a/src/shim/umq/aiedbg.h +++ b/src/shim/umq/aiedbg.h @@ -1,6 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. +#ifndef _AIE_DBG_H_ +#define _AIE_DBG_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + //definitions for aie debugger backend and frontend communication #define ATTACH_CMD 1 #define READ_MEM_CMD 2 @@ -9,3 +17,30 @@ #define AIE_DBG_SUCCESS 0 #define AIE_DBG_NOT_ATTACHED 0xffff + +struct aie_debugger_cmd +{ + uint32_t type; + union Cmd + { + struct Attach + { + uint32_t uc_index; + } attach; + struct Read_mem + { + uint32_t aie_addr; + uint32_t length; + } read_mem; + struct Write_mem + { + uint32_t aie_addr; + uint32_t data[1]; + } write_mem; + } cmd; +}; + +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/shim/umq/dbg_cmd.h b/src/shim/umq/dbg_cmd.h index ff89e530..58a3db4d 100644 --- a/src/shim/umq/dbg_cmd.h +++ b/src/shim/umq/dbg_cmd.h @@ -13,7 +13,7 @@ extern "C" #define DBG_PKT_EXIT (2) #define DBG_PKT_INVALID (2) -enum dbg_packet_opcode; +enum dbg_packet_opcode { DBG_CMD_TEST = 10, DBG_CMD_EXIT = 11, diff --git a/src/shim/umq/dbg_hwq.cpp b/src/shim/umq/dbg_hwq.cpp index c8cf939a..31cf4df6 100644 --- a/src/shim/umq/dbg_hwq.cpp +++ b/src/shim/umq/dbg_hwq.cpp @@ -7,7 +7,7 @@ namespace shim_xdna { dbg_hwq_umq:: dbg_hwq_umq(const device& dev) - : m_pdev(device.get_pdev()) + : m_pdev(dev.get_pdev()) { const size_t header_sz = sizeof(struct host_queue_header); const size_t queue_sz = sizeof(struct host_queue_packet); @@ -19,16 +19,18 @@ dbg_hwq_umq(const device& dev) m_dbg_umq_bo = std::make_unique(m_pdev, umq_sz, AMDXDNA_BO_CMD); m_dbg_umq_bo_buf = m_dbg_umq_bo->vaddr(); - m_dbg_umq_addr = m_dbg_umq_bo->paddr(); - m_dbg_umq_hdr = reinterpret_cast(m_dbg_umq_bo_buf); + m_dbg_umq_paddr = m_dbg_umq_bo->paddr(); + m_dbg_umq_hdr = + reinterpret_cast(m_dbg_umq_bo_buf); m_dbg_umq_pkt = reinterpret_cast ((char *)m_dbg_umq_bo_buf + header_sz); m_dbg_umq_comp = m_dbg_umq_bo->paddr() + header_sz + queue_sz; - m_dbg_umq_comp_ptr = reinterpret_cast(m_dbg_umq_comp); + m_dbg_umq_comp_ptr = reinterpret_cast + ((char *)m_dbg_umq_bo_buf + header_sz + queue_sz); // set all mapped memory to 0 std::memset(m_dbg_umq_bo_buf, 0, umq_sz); - m_dbg_umq_pkt->completion_signal = m_dbg_umq_comp; + m_dbg_umq_pkt->xrt_header.completion_signal = m_dbg_umq_comp; m_dbg_umq_pkt->xrt_header.common_header.type = HOST_QUEUE_PACKET_TYPE_INVALID; m_dbg_umq_hdr->capacity = 1; @@ -46,31 +48,41 @@ uint32_t dbg_hwq_umq:: issue_exit_cmd() { - auto hdr = m_dbg_umq_pkt->xrt_header; + auto hdr = &m_dbg_umq_pkt->xrt_header; // always case 1 m_dbg_umq_hdr->write_index++; - struct xrt_packet_header *ehp = &m_dbg_umq_pkt->xrt_header; + auto ehp = &m_dbg_umq_pkt->xrt_header; ehp->common_header.opcode = DBG_CMD_EXIT; ehp->common_header.count = 0; - submit(); + shim_debug("dbg umq: issue exit cmd"); + return submit(); } uint32_t dbg_hwq_umq:: issue_rw_cmd(struct rw_mem &data, uint16_t opcode) { - auto hdr = m_dbg_umq_pkt->xrt_header; + auto hdr = &m_dbg_umq_pkt->xrt_header; // always case 1 m_dbg_umq_hdr->write_index++; - struct xrt_packet_header *ehp = &m_dbg_umq_pkt->xrt_header; + auto ehp = &m_dbg_umq_pkt->xrt_header; ehp->common_header.opcode = opcode; ehp->common_header.count = sizeof (struct rw_mem); - struct rw_mem *rwp = reinterpret_cast(m_dbg_umq_pkt->data); + struct rw_mem *rwp = reinterpret_cast + (const_cast(m_dbg_umq_pkt->data)); std::memcpy(rwp, &data, sizeof(struct rw_mem)); - submit(); + shim_debug("dbg umq: issue rw cmd"); + return submit(); +} + +uint64_t +dbg_hwq_umq:: +get_bo_paddr() const +{ + return m_dbg_umq_paddr; } uint32_t @@ -81,8 +93,10 @@ submit() /* Issue mfence instruction to make sure all writes to the slot before is done */ std::atomic_thread_fence(std::memory_order::memory_order_seq_cst); - m_dbg_umq_pkt->xrt_header.common_header.type = HOST_QUEUE_PACKET_TYPE_VENDOR_SPECIFIC; + m_dbg_umq_pkt->xrt_header.common_header.type = + HOST_QUEUE_PACKET_TYPE_VENDOR_SPECIFIC; + shim_debug("dbg umq: submit cmd"); while (1) { if (*m_dbg_umq_comp_ptr) diff --git a/src/shim/umq/dbg_hwq.h b/src/shim/umq/dbg_hwq.h index d00c7211..ad197398 100644 --- a/src/shim/umq/dbg_hwq.h +++ b/src/shim/umq/dbg_hwq.h @@ -4,7 +4,9 @@ #ifndef DBG_HWQ_UMQ_H #define DBG_HWQ_UMQ_H +#include #include "host_queue.h" +#include "../buffer.h" #include "dbg_cmd.h" #include "aiedbg.h" @@ -13,20 +15,23 @@ namespace shim_xdna { class dbg_hwq_umq { public: - dbg_hwq_umq(const device& device); + dbg_hwq_umq(const device& dev); ~dbg_hwq_umq(); + uint32_t issue_exit_cmd(); + uint32_t issue_rw_cmd(struct rw_mem &data, uint16_t opcode); + uint64_t get_bo_paddr() const; + private: const pdev& m_pdev; + std::unique_ptr m_dbg_umq_bo; void *m_dbg_umq_bo_buf; - uint64_t m_dbg_umq_addr; + uint64_t m_dbg_umq_paddr; volatile struct host_queue_header *m_dbg_umq_hdr = nullptr; volatile struct host_queue_packet *m_dbg_umq_pkt = nullptr; uint64_t m_dbg_umq_comp; volatile uint32_t *m_dbg_umq_comp_ptr = nullptr; - uint32_t issue_exit_cmd(); - uint32_t issue_rw_cmd(struct rw_mem &data, uint16_t opcode); uint32_t submit(); }; diff --git a/src/shim/umq/fw_buf_metadata.h b/src/shim/umq/fw_buf_metadata.h new file mode 100644 index 00000000..b97a4feb --- /dev/null +++ b/src/shim/umq/fw_buf_metadata.h @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2023-2025, Advanced Micro Devices, Inc. All rights reserved. + +#ifndef FW_METADATA_H +#define FW_METADATA_H + +namespace shim_xdna { + +enum umq_fw_flag { + UMQ_DEBUG_BUFFER = 0, + UMQ_TRACE_BUFFER, + UMQ_DBG_QUEUE, + UMQ_LOG_BUFFER +}; + +struct umq_fw_metadata { + uint16_t reserved; + uint8_t umq_fw_flag; + uint8_t num_ucs; // how many valid ucs, up to 8 for now + struct { + uint64_t paddr : 57; // device accessible address array for each valid uc + uint64_t index : 7; // uc index + uint32_t size; // bo size for each valid uc in words + } uc_info[8]; +}; + +} + +#endif diff --git a/src/shim/umq/hwctx.cpp b/src/shim/umq/hwctx.cpp index abac6905..d1e1a0fb 100644 --- a/src/shim/umq/hwctx.cpp +++ b/src/shim/umq/hwctx.cpp @@ -53,17 +53,14 @@ fini_log_buf(void) void hwctx_umq:: -set_metadata(int num_ucs, size_t size, uint64_t bo_paddr, enum umq_log_flag flag) +set_metadata(int num_ucs, size_t size, uint64_t bo_paddr, enum umq_fw_flag flag) { - const uint32_t LOG_MAGIC_NO = 0x43455254; - m_log_metadata.magic_no = LOG_MAGIC_NO; - m_log_metadata.major = 0; - m_log_metadata.minor = 1; - m_log_metadata.umq_log_flag = flag; + m_log_metadata.umq_fw_flag = flag; m_log_metadata.num_ucs = num_ucs; for (int i = 0; i < num_ucs; i++) { - m_log_metadata.uc_paddr[i] = bo_paddr + size * i + sizeof(m_log_metadata); - m_log_metadata.uc_size[i] = size; + m_log_metadata.uc_info[i].paddr = bo_paddr + size * i + sizeof(m_log_metadata); + m_log_metadata.uc_info[i].size = size; + m_log_metadata.uc_info[i].index = i; } } @@ -72,8 +69,8 @@ hwctx_umq:: init_tcp_server(const device& dev) { //check xrt.ini to start tcp server - m_tcp_server(dev); - m_thread_(m_tcp_server::start); + m_tcp_server = std::make_unique(dev, this); + m_thread_ = std::thread([&] () { m_tcp_server->start(); }); } void diff --git a/src/shim/umq/hwctx.h b/src/shim/umq/hwctx.h index 25280edd..d65fa7f4 100644 --- a/src/shim/umq/hwctx.h +++ b/src/shim/umq/hwctx.h @@ -4,29 +4,16 @@ #ifndef HWCTX_UMQ_H #define HWCTX_UMQ_H +#include +#include +#include #include "../hwctx.h" #include "../buffer.h" #include "tcp_server.h" +#include "fw_buf_metadata.h" namespace shim_xdna { -enum umq_log_flag { - UMQ_DEBUG_BUFFER = 0, - UMQ_TRACE_BUFFER, - UMQ_DBG_QUEUE, - UMQ_LOG_BUFFER -}; - -struct umq_fw_metadata { - uint32_t magic_no; - uint8_t major; - uint8_t minor; - uint8_t umq_log_flag; - uint8_t num_ucs; // how many valid ucs, up to 8 for now - uint64_t uc_paddr[8]; // device accessible address array for each valid uc - uint32_t uc_size[8]; // bo size for each valid uc -}; - class hwctx_umq : public hwctx { public: hwctx_umq(const device& device, const xrt::xclbin& xclbin, const qos_type& qos); @@ -36,11 +23,10 @@ class hwctx_umq : public hwctx { std::unique_ptr m_log_bo; uint32_t m_col_cnt = 0; - umq_fw_metadata m_debugger_metadata; umq_fw_metadata m_log_metadata; void *m_log_buf = nullptr; - tcp_server m_tcp_server; + std::unique_ptr m_tcp_server; std::thread m_thread_; void init_tcp_server(const device& dev); @@ -48,7 +34,7 @@ class hwctx_umq : public hwctx { void init_log_buf(); void fini_log_buf(); - void set_metadata(int num_cols, size_t size, uint64_t bo_paddr, enum umq_log_flag flag); + void set_metadata(int num_cols, size_t size, uint64_t bo_paddr, enum umq_fw_flag flag); }; } diff --git a/src/shim/umq/tcp_server.cpp b/src/shim/umq/tcp_server.cpp index 1f50e02e..98521532 100644 --- a/src/shim/umq/tcp_server.cpp +++ b/src/shim/umq/tcp_server.cpp @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved. -#include -#include #include #include @@ -11,14 +9,14 @@ namespace shim_xdna { tcp_server:: -tcp_server(const device &dev) : -m_aie_attached(false), m_dbg_umq(dev) {}, m_def_size(16) +tcp_server(const device& dev, hwctx* hwctx) : +m_aie_attached(false), m_dbg_umq(dev), m_def_size(16), m_pdev(dev.get_pdev()) { + m_hwctx = hwctx; auto def_buf_size = m_def_size * sizeof(uint32_t); - auto def_bo = alloc_bo(def_buf_size, XCL_BO_FLAGS_EXECBUF); - m_def_bo = std::unique_ptr(static_cast(def_bo.release())); - m_def_buf = m_def_bo->vaddr(); - m_def_paddr = m_def_bo->paddr(); + m_data_bo = std::make_unique(m_pdev, def_buf_size, AMDXDNA_BO_CMD); + m_data_buf = m_data_bo->vaddr(); + m_data_paddr = m_data_bo->paddr(); } tcp_server:: @@ -72,35 +70,37 @@ start() } std::vector buffer(length >> 2); recv(clientSocket, buffer.data(), length, 0); - switch (buffer[0]) + + auto cmd = reinterpret_cast(buffer.data()); + switch (cmd->type) { case ATTACH_CMD: { - uint32_t status = handle_attach(); + uint32_t status = handle_attach(cmd->cmd.attach.uc_index); std::vector ret; ret.push_back(sizeof(uint32_t)); ret.push_back(status); - send(clientSocket, ret.data(), ret[0] + sizeof(uint32_t), 0); + send(clientSocket, ret.data(), ret.size() * sizeof(uint32_t), 0); break; } case READ_MEM_CMD: { - auto data = handle_read_mem(buffer[1], buffer[2]); + auto data = handle_read_mem(cmd->cmd.read_mem.aie_addr, cmd->cmd.read_mem.length); std::vector ret; - ret.push_back(sizeof(uint32_t) * (buffer[2] + 1)); + ret.push_back(sizeof(uint32_t) * (cmd->cmd.read_mem.length + 1)); ret.insert(ret.end(), data->begin(), data->end()); - send(clientSocket, ret.data(), ret[0] + sizeof(uint32_t), 0); + send(clientSocket, ret.data(), ret.size() + sizeof(uint32_t), 0); break; } case WRITE_MEM_CMD: { std::vector data = {buffer.begin() + 2, buffer.end()}; - uint32_t status = handle_write_mem(buffer[1], data); + uint32_t status = handle_write_mem(cmd->cmd.write_mem.aie_addr, data); std::vector ret; ret.push_back(sizeof(uint32_t)); ret.push_back(status); - send(clientSocket, ret.data(), ret[0] + sizeof(uint32_t), 0); + send(clientSocket, ret.data(), ret.size() * sizeof(uint32_t), 0); break; } case DETACH_CMD: @@ -139,8 +139,8 @@ handle_read_mem(uint32_t addr, uint32_t length) } struct rw_mem rw; - rw.host_addr_high = m_def_paddr >> 32; - rw.host_addr_low = m_def_paddr & 0xffffffff; + rw.host_addr_high = m_data_paddr >> 32; + rw.host_addr_low = m_data_paddr & 0xffffffff; rw.aie_addr = addr; rw.length = length; @@ -152,8 +152,11 @@ handle_read_mem(uint32_t addr, uint32_t length) else { data[0] = AIE_DBG_SUCCESS; - std::memcpy(data.data() + 1, m_def_buf, length * sizeof (uint32_t)); + std::memcpy(data.data() + 1, + const_cast(m_data_buf), + length * sizeof (uint32_t)); } + shim_debug("TCP server read mem: addr (0x%x) length (%dW)\n", addr, length); return std::make_unique>(data); } @@ -173,14 +176,17 @@ handle_write_mem(uint32_t addr, std::vector &data) } struct rw_mem rw; - rw.host_addr_high = m_def_paddr >> 32; - rw.host_addr_low = m_def_paddr & 0xffffffff; + rw.host_addr_high = m_data_paddr >> 32; + rw.host_addr_low = m_data_paddr & 0xffffffff; rw.aie_addr = addr; rw.length = data.size(); - std::memcpy(m_def_buf, data.data(), data.size() * sizeof(uint32_t)); + std::memcpy(const_cast(m_data_buf), + data.data(), + data.size() * sizeof(uint32_t)); uint32_t ret = m_dbg_umq.issue_rw_cmd(rw, DBG_CMD_WRITE); + shim_debug("TCP server write mem: addr (0x%x)\n", addr); return ret != DBG_PKT_SUCCESS ? ret : AIE_DBG_SUCCESS; } @@ -188,19 +194,29 @@ void tcp_server:: buffer_extend(size_t new_size) { + shim_debug("TCP server buffer extend to (%d)W\n", new_size); auto n_buf_size = new_size * sizeof(uint32_t); - auto n_bo = alloc_bo(n_buf_size, XCL_BO_FLAGS_EXECBUF); - m_def_bo = std::move(std::unique_ptr(static_cast(n_bo.release()))); - m_def_buf = m_def_bo->vaddr(); - m_def_paddr = m_def_bo->paddr(); + m_data_bo = std::make_unique(m_pdev, n_buf_size, AMDXDNA_BO_CMD); + m_data_buf = m_data_bo->vaddr(); + m_data_paddr = m_data_bo->paddr(); } uint32_t tcp_server:: -handle_attach() +handle_attach(uint32_t uc_index) { // issue ioctl to attach the dbg hsa queue // send a DBG_CMD_TEST opcode + m_ctrl_bo = std::make_unique(m_pdev, sizeof(umq_fw_metadata), AMDXDNA_BO_CMD); + auto debugger_metadata = reinterpret_cast(m_ctrl_bo->vaddr()); + debugger_metadata->umq_fw_flag = UMQ_DBG_QUEUE; + debugger_metadata->num_ucs = 1; + debugger_metadata->uc_info[0].paddr = m_dbg_umq.get_bo_paddr(); + debugger_metadata->uc_info[0].index = uc_index; + + m_ctrl_bo->bind_hwctx(*m_hwctx); + shim_debug("TCP server ioctl: debugger attach\n"); + m_aie_attached = true; return AIE_DBG_SUCCESS; } @@ -210,8 +226,10 @@ tcp_server:: handle_detach() { m_dbg_umq.issue_exit_cmd(); + m_ctrl_bo = nullptr; // issue ioctl to detach the dbg hsa queue m_aie_attached = false; + shim_debug("TCP server ioctl: debugger queue detach\n"); } } // shim_xdna diff --git a/src/shim/umq/tcp_server.h b/src/shim/umq/tcp_server.h index 1827b576..7d82b89d 100644 --- a/src/shim/umq/tcp_server.h +++ b/src/shim/umq/tcp_server.h @@ -6,28 +6,35 @@ #include "dbg_hwq.h" #include "../buffer.h" +#include "../hwctx.h" +#include "fw_buf_metadata.h" namespace shim_xdna { class tcp_server { public: - tcp_server(); + tcp_server(const device& dev, hwctx* hwctx); ~tcp_server(); void start(); private: dbg_hwq_umq m_dbg_umq; uint32_t m_def_size; - std::unique_ptr m_def_bo; - volatile void *m_def_buf; - uint64_t m_def_addr; + std::unique_ptr m_data_bo; + volatile void *m_data_buf; + uint64_t m_data_paddr; + std::unique_ptr m_ctrl_bo; + volatile void *m_ctrl_buf; + uint64_t m_ctrl_paddr; bool m_aie_attached; + hwctx* m_hwctx; + const pdev& m_pdev; std::unique_ptr> handle_read_mem(uint32_t addr, uint32_t length); uint32_t handle_write_mem(uint32_t addr, std::vector &data); void buffer_extend(size_t new_size); - void handle_attach(); + uint32_t handle_attach(uint32_t); void handle_detach(); }; From ad022a0c46ab87438349a3b7fcdcb1e0063c739e Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Tue, 20 May 2025 16:42:18 -0700 Subject: [PATCH 4/8] support config_bo/unconfig_bo --- src/shim/buffer.cpp | 92 ++++++++++++++++++++++++++++++++++ src/shim/buffer.h | 26 +++++++++- src/shim/device.cpp | 5 ++ src/shim/umq/dbg_hwq.cpp | 24 +++++++-- src/shim/umq/dbg_hwq.h | 5 +- src/shim/umq/fw_buf_metadata.h | 29 ----------- src/shim/umq/hwctx.cpp | 44 +++++++++------- src/shim/umq/hwctx.h | 7 +-- src/shim/umq/tcp_server.cpp | 25 ++++----- src/shim/umq/tcp_server.h | 1 - 10 files changed, 184 insertions(+), 74 deletions(-) delete mode 100644 src/shim/umq/fw_buf_metadata.h diff --git a/src/shim/buffer.cpp b/src/shim/buffer.cpp index 5c87806f..8427aa3e 100644 --- a/src/shim/buffer.cpp +++ b/src/shim/buffer.cpp @@ -10,11 +10,35 @@ namespace { +uint8_t +use_to_type(uint8_t use) +{ + switch (use) { + case XRT_BO_USE_DEBUG: + return AMDXDNA_FW_BUF_DEBUG; + case XRT_BO_USE_DTRACE: + return AMDXDNA_FW_BUF_TRACE; + case XRT_BO_USE_DEBUG_QUEUE: + return AMDXDNA_FW_BUF_DBG_Q; + case XRT_BO_USE_LOG: + return AMDXDNA_FW_BUF_LOG; + } + throw; +} + std::string type_to_name(int type, uint64_t flags) { switch (type) { case AMDXDNA_BO_SHARE: + if (xcl_bo_flags{flags}.use == XRT_BO_USE_DTRACE) + return std::string("AMDXDNA_BO_UC_DTRACE"); + else if (xcl_bo_flags{flags}.use == XRT_BO_USE_LOG) + return std::string("AMDXDNA_BO_UC_LOG"); + else if (xcl_bo_flags{flags}.use == XRT_BO_USE_DEBUG_QUEUE) + return std::string("AMDXDNA_BO_UC_DEBUG_QUEUE"); + //else if (xcl_bo_flags{flags}.use == XRT_BO_USE_UC_DEBUG) + // return std::string("AMDXDNA_BO_UC_DEBUG_BUF"); return std::string("AMDXDNA_BO_SHARE"); case AMDXDNA_BO_DEV_HEAP: return std::string("AMDXDNA_BO_DEV_HEAP"); @@ -354,6 +378,13 @@ bind_hwctx(const hwctx& hwctx) // Nothing to do. } +void +buffer:: +unbind_hwctx() +{ + // Nothing to do. +} + void buffer:: set_flags(uint64_t flags) @@ -361,6 +392,13 @@ set_flags(uint64_t flags) m_flags = flags; } +uint64_t +buffer:: +get_flags() +{ + return m_flags; +} + std::string buffer:: describe() const @@ -492,6 +530,20 @@ bind_hwctx(const hwctx& hwctx) shim_debug("Attached DEBUG BO %d to hwctx %d", id().handle, m_ctx_id); } +void +dbg_buffer:: +unbind_hwctx() +{ + try { + config_debug_bo(true); + m_ctx_id = AMDXDNA_INVALID_CTX_HANDLE; + } catch (const xrt_core::system_error& e) { + std::cout << "Failed to detach DEBUG BO " << std::to_string(id().handle) + << " from hwctx " << std::to_string(m_ctx_id) + << ": " << e.what() << std::endl; + } +} + void dbg_buffer:: config_debug_bo(bool is_detach) @@ -519,4 +571,44 @@ dbg_buffer:: } } +// +// Impl for class uc_dbg_buffer +// +void +uc_dbg_buffer:: +config(xrt_core::hwctx_handle* hwctx, const std::map& buf_sizes) +{ + auto meta_buf_size = offsetof(struct fw_buffer_metadata, uc_info) + + buf_sizes.size() * sizeof(struct uc_info_entry); + m_metadata_bo = std::make_unique(m_pdev, meta_buf_size, AMDXDNA_BO_SHARE); + auto metadata = reinterpret_cast(m_metadata_bo->vaddr()); + metadata->bo_handle = id().handle; + metadata->command_id = 0; //support this later + auto f = xcl_bo_flags{get_flags()}; + metadata->buf_type = use_to_type(f.use); + metadata->num_ucs = buf_sizes.size(); + int i = 0; + for (const auto& pair : buf_sizes) + { + metadata->uc_info[i].size = pair.second; + metadata->uc_info[i].index = pair.first; + i++; + } + + shim_debug("Config CMD BO %d (%s) for %d uC", id().handle, + type_to_name(AMDXDNA_BO_SHARE, get_flags()).c_str(), i); + + auto ctx = static_cast(hwctx); + m_metadata_bo->bind_hwctx(*ctx); +} + +void +uc_dbg_buffer:: +unconfig(xrt_core::hwctx_handle* hwctx) +{ + shim_debug("Unconfig CMD BO %d (%s)", id().handle, + type_to_name(AMDXDNA_BO_SHARE, get_flags()).c_str()); + m_metadata_bo->unbind_hwctx(); +} + } diff --git a/src/shim/buffer.h b/src/shim/buffer.h index c4992f47..e186f96e 100644 --- a/src/shim/buffer.h +++ b/src/shim/buffer.h @@ -11,6 +11,7 @@ #include "core/common/shim/hwctx_handle.h" #include "core/common/shim/buffer_handle.h" #include +#include "drm_local/amdxdna_accel.h" namespace shim_xdna { @@ -90,9 +91,12 @@ class buffer : public xrt_core::buffer_handle virtual void bind_hwctx(const hwctx& hwctx); + virtual void + unbind_hwctx(); + // Save flags in buffer which later returns via get_properties() - void - set_flags(uint64_t flags); + void set_flags(uint64_t flags); + uint64_t get_flags(); virtual std::set get_arg_bo_ids() const; @@ -146,6 +150,9 @@ class dbg_buffer : public buffer void bind_hwctx(const hwctx& hwctx) override; + void + unbind_hwctx() override; + private: void config_debug_bo(bool is_detach); @@ -153,6 +160,21 @@ class dbg_buffer : public buffer xrt_core::hwctx_handle::slot_id m_ctx_id = AMDXDNA_INVALID_CTX_HANDLE; }; +class uc_dbg_buffer : public buffer +{ +public: + using buffer::buffer; + + void + config(xrt_core::hwctx_handle* hwctx, const std::map& buf_sizes) override; + + void + unconfig(xrt_core::hwctx_handle* hwctx) override; + +private: + std::unique_ptr m_metadata_bo; +}; + } #endif diff --git a/src/shim/device.cpp b/src/shim/device.cpp index 604508da..3a8f6f6a 100644 --- a/src/shim/device.cpp +++ b/src/shim/device.cpp @@ -1409,6 +1409,11 @@ alloc_bo(void* userptr, size_t size, uint64_t flags) std::unique_ptr bo; if (f.use == XRT_BO_USE_DEBUG) bo = std::make_unique(get_pdev(), size, type); + else if (f.use == XRT_BO_USE_DTRACE || + f.use == XRT_BO_USE_LOG || + f.use == XRT_BO_USE_DEBUG_QUEUE + /*f.use == XRT_BO_USE_UC_DEBUG*/) //need define last one in xrt + bo = std::make_unique(get_pdev(), size, type); else if (type == AMDXDNA_BO_CMD) bo = std::make_unique(get_pdev(), size, type); else diff --git a/src/shim/umq/dbg_hwq.cpp b/src/shim/umq/dbg_hwq.cpp index 31cf4df6..7b62fc48 100644 --- a/src/shim/umq/dbg_hwq.cpp +++ b/src/shim/umq/dbg_hwq.cpp @@ -17,9 +17,8 @@ dbg_hwq_umq(const device& dev) shim_debug("dbg umq sz %ld", umq_sz); - m_dbg_umq_bo = std::make_unique(m_pdev, umq_sz, AMDXDNA_BO_CMD); + m_dbg_umq_bo = std::make_unique(m_pdev, umq_sz, AMDXDNA_BO_SHARE); m_dbg_umq_bo_buf = m_dbg_umq_bo->vaddr(); - m_dbg_umq_paddr = m_dbg_umq_bo->paddr(); m_dbg_umq_hdr = reinterpret_cast(m_dbg_umq_bo_buf); m_dbg_umq_pkt = reinterpret_cast @@ -35,6 +34,7 @@ dbg_hwq_umq(const device& dev) m_dbg_umq_pkt->xrt_header.common_header.type = HOST_QUEUE_PACKET_TYPE_INVALID; m_dbg_umq_hdr->capacity = 1; + set_use_flag(); shim_debug("Created DBG UMQ HW queue"); } @@ -78,11 +78,24 @@ issue_rw_cmd(struct rw_mem &data, uint16_t opcode) return submit(); } -uint64_t +void dbg_hwq_umq:: -get_bo_paddr() const +set_use_flag() const { - return m_dbg_umq_paddr; + auto f = xcl_bo_flags{0}; + f.use = XRT_BO_USE_DEBUG_QUEUE; + f.flags = XRT_BO_FLAGS_CACHEABLE; + f.access = XRT_BO_ACCESS_LOCAL; + f.dir = XRT_BO_ACCESS_READ_WRITE; + + m_dbg_umq_bo->set_flags(f.all); +} + +buffer* +dbg_hwq_umq:: +get_dbg_umq_bo() const +{ + return m_dbg_umq_bo.get(); } uint32_t @@ -97,6 +110,7 @@ submit() HOST_QUEUE_PACKET_TYPE_VENDOR_SPECIFIC; shim_debug("dbg umq: submit cmd"); + while (1) { if (*m_dbg_umq_comp_ptr) diff --git a/src/shim/umq/dbg_hwq.h b/src/shim/umq/dbg_hwq.h index ad197398..0737e852 100644 --- a/src/shim/umq/dbg_hwq.h +++ b/src/shim/umq/dbg_hwq.h @@ -5,6 +5,7 @@ #define DBG_HWQ_UMQ_H #include +#include #include "host_queue.h" #include "../buffer.h" #include "dbg_cmd.h" @@ -20,19 +21,19 @@ class dbg_hwq_umq uint32_t issue_exit_cmd(); uint32_t issue_rw_cmd(struct rw_mem &data, uint16_t opcode); - uint64_t get_bo_paddr() const; + buffer* get_dbg_umq_bo() const; private: const pdev& m_pdev; std::unique_ptr m_dbg_umq_bo; void *m_dbg_umq_bo_buf; - uint64_t m_dbg_umq_paddr; volatile struct host_queue_header *m_dbg_umq_hdr = nullptr; volatile struct host_queue_packet *m_dbg_umq_pkt = nullptr; uint64_t m_dbg_umq_comp; volatile uint32_t *m_dbg_umq_comp_ptr = nullptr; uint32_t submit(); + void set_use_flag() const; }; } diff --git a/src/shim/umq/fw_buf_metadata.h b/src/shim/umq/fw_buf_metadata.h deleted file mode 100644 index b97a4feb..00000000 --- a/src/shim/umq/fw_buf_metadata.h +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright (C) 2023-2025, Advanced Micro Devices, Inc. All rights reserved. - -#ifndef FW_METADATA_H -#define FW_METADATA_H - -namespace shim_xdna { - -enum umq_fw_flag { - UMQ_DEBUG_BUFFER = 0, - UMQ_TRACE_BUFFER, - UMQ_DBG_QUEUE, - UMQ_LOG_BUFFER -}; - -struct umq_fw_metadata { - uint16_t reserved; - uint8_t umq_fw_flag; - uint8_t num_ucs; // how many valid ucs, up to 8 for now - struct { - uint64_t paddr : 57; // device accessible address array for each valid uc - uint64_t index : 7; // uc index - uint32_t size; // bo size for each valid uc in words - } uc_info[8]; -}; - -} - -#endif diff --git a/src/shim/umq/hwctx.cpp b/src/shim/umq/hwctx.cpp index d1e1a0fb..c326d941 100644 --- a/src/shim/umq/hwctx.cpp +++ b/src/shim/umq/hwctx.cpp @@ -9,15 +9,14 @@ namespace shim_xdna { hwctx_umq:: hwctx_umq(const device& device, const xrt::xclbin& xclbin, const qos_type& qos) : hwctx(device, qos, xclbin, std::make_unique(device, 8)) - , m_log_metadata() + , m_pdev(device.get_pdev()) { + shim_debug("Created UMQ HW context (%d)", get_slotidx()); xclbin_parser xp(xclbin); m_col_cnt = xp.get_column_cnt(); init_tcp_server(device); init_log_buf(); - // TODO: configure log BO on the hwctx - shim_debug("Created UMQ HW context (%d)", get_slotidx()); } hwctx_umq:: @@ -34,14 +33,26 @@ hwctx_umq:: init_log_buf() { size_t column_size = 1024; - auto log_buf_size = m_col_cnt * column_size + sizeof(m_log_metadata); - auto log_bo = alloc_bo(log_buf_size, XCL_BO_FLAGS_EXECBUF); - m_log_bo = std::unique_ptr(static_cast(log_bo.release())); - m_log_buf = m_log_bo->vaddr(); - uint64_t bo_paddr = m_log_bo->paddr(); - set_metadata(m_col_cnt, column_size, bo_paddr, UMQ_LOG_BUFFER); - std::memset(m_log_buf, 0, log_buf_size); - std::memcpy(m_log_buf, &m_log_metadata, sizeof(m_log_metadata)); + auto log_buf_size = m_col_cnt * column_size; + m_log_bo = std::make_unique + (m_pdev, log_buf_size, AMDXDNA_BO_SHARE); + auto log_buf = m_log_bo->vaddr(); + std::memset(log_buf, 0, log_buf_size); + + auto f = xcl_bo_flags{0}; + f.use = XRT_BO_USE_LOG; + f.flags = XRT_BO_FLAGS_CACHEABLE; + f.access = XRT_BO_ACCESS_LOCAL; + f.dir = XRT_BO_ACCESS_READ_WRITE; + + m_log_bo->set_flags(f.all); + + std::map buf_sizes; + set_metadata(buf_sizes, m_col_cnt, column_size); + + // TODO: configure log BO on the hwctx once driver and fw support it + // we may use xrt.ini to control the config + //m_log_bo->config(this, buf_sizes); } void @@ -53,14 +64,10 @@ fini_log_buf(void) void hwctx_umq:: -set_metadata(int num_ucs, size_t size, uint64_t bo_paddr, enum umq_fw_flag flag) +set_metadata(std::map& buf_sizes, int num_ucs, size_t size) { - m_log_metadata.umq_fw_flag = flag; - m_log_metadata.num_ucs = num_ucs; for (int i = 0; i < num_ucs; i++) { - m_log_metadata.uc_info[i].paddr = bo_paddr + size * i + sizeof(m_log_metadata); - m_log_metadata.uc_info[i].size = size; - m_log_metadata.uc_info[i].index = i; + buf_sizes[i] = size; } } @@ -68,7 +75,7 @@ void hwctx_umq:: init_tcp_server(const device& dev) { - //check xrt.ini to start tcp server + //TODO:check xrt.ini to start tcp server m_tcp_server = std::make_unique(dev, this); m_thread_ = std::thread([&] () { m_tcp_server->start(); }); } @@ -79,6 +86,7 @@ fini_tcp_server() { if (m_thread_.joinable()) { + shim_debug("Kill TCP server..."); pthread_kill(m_thread_.native_handle(), SIGINT); m_thread_.join(); } diff --git a/src/shim/umq/hwctx.h b/src/shim/umq/hwctx.h index d65fa7f4..496c8886 100644 --- a/src/shim/umq/hwctx.h +++ b/src/shim/umq/hwctx.h @@ -7,10 +7,11 @@ #include #include #include +#include #include "../hwctx.h" #include "../buffer.h" #include "tcp_server.h" -#include "fw_buf_metadata.h" +#include "drm_local/amdxdna_accel.h" namespace shim_xdna { @@ -20,10 +21,10 @@ class hwctx_umq : public hwctx { ~hwctx_umq(); private: + const pdev& m_pdev; std::unique_ptr m_log_bo; uint32_t m_col_cnt = 0; - umq_fw_metadata m_log_metadata; void *m_log_buf = nullptr; std::unique_ptr m_tcp_server; @@ -34,7 +35,7 @@ class hwctx_umq : public hwctx { void init_log_buf(); void fini_log_buf(); - void set_metadata(int num_cols, size_t size, uint64_t bo_paddr, enum umq_fw_flag flag); + void set_metadata(std::map& buf_size, int num_cols, size_t size); }; } diff --git a/src/shim/umq/tcp_server.cpp b/src/shim/umq/tcp_server.cpp index 98521532..70f51b92 100644 --- a/src/shim/umq/tcp_server.cpp +++ b/src/shim/umq/tcp_server.cpp @@ -14,7 +14,7 @@ m_aie_attached(false), m_dbg_umq(dev), m_def_size(16), m_pdev(dev.get_pdev()) { m_hwctx = hwctx; auto def_buf_size = m_def_size * sizeof(uint32_t); - m_data_bo = std::make_unique(m_pdev, def_buf_size, AMDXDNA_BO_CMD); + m_data_bo = std::make_unique(m_pdev, def_buf_size, AMDXDNA_BO_SHARE); m_data_buf = m_data_bo->vaddr(); m_data_paddr = m_data_bo->paddr(); } @@ -22,6 +22,7 @@ m_aie_attached(false), m_dbg_umq(dev), m_def_size(16), m_pdev(dev.get_pdev()) tcp_server:: ~tcp_server() { + shim_debug("TCP server destructor"); } void @@ -90,7 +91,7 @@ start() ret.push_back(sizeof(uint32_t) * (cmd->cmd.read_mem.length + 1)); ret.insert(ret.end(), data->begin(), data->end()); - send(clientSocket, ret.data(), ret.size() + sizeof(uint32_t), 0); + send(clientSocket, ret.data(), ret.size() * sizeof(uint32_t), 0); break; } case WRITE_MEM_CMD: @@ -194,9 +195,9 @@ void tcp_server:: buffer_extend(size_t new_size) { - shim_debug("TCP server buffer extend to (%d)W\n", new_size); + shim_debug("TCP server buffer extend to (%dW)\n", new_size); auto n_buf_size = new_size * sizeof(uint32_t); - m_data_bo = std::make_unique(m_pdev, n_buf_size, AMDXDNA_BO_CMD); + m_data_bo = std::make_unique(m_pdev, n_buf_size, AMDXDNA_BO_SHARE); m_data_buf = m_data_bo->vaddr(); m_data_paddr = m_data_bo->paddr(); } @@ -206,15 +207,10 @@ tcp_server:: handle_attach(uint32_t uc_index) { // issue ioctl to attach the dbg hsa queue - // send a DBG_CMD_TEST opcode - m_ctrl_bo = std::make_unique(m_pdev, sizeof(umq_fw_metadata), AMDXDNA_BO_CMD); - auto debugger_metadata = reinterpret_cast(m_ctrl_bo->vaddr()); - debugger_metadata->umq_fw_flag = UMQ_DBG_QUEUE; - debugger_metadata->num_ucs = 1; - debugger_metadata->uc_info[0].paddr = m_dbg_umq.get_bo_paddr(); - debugger_metadata->uc_info[0].index = uc_index; - - m_ctrl_bo->bind_hwctx(*m_hwctx); + std::map buf_sizes; + buf_sizes[uc_index] = 0; //we don't care size + + m_dbg_umq.get_dbg_umq_bo()->config(m_hwctx, buf_sizes); shim_debug("TCP server ioctl: debugger attach\n"); m_aie_attached = true; @@ -226,8 +222,9 @@ tcp_server:: handle_detach() { m_dbg_umq.issue_exit_cmd(); - m_ctrl_bo = nullptr; // issue ioctl to detach the dbg hsa queue + m_dbg_umq.get_dbg_umq_bo()->unconfig(m_hwctx); + m_aie_attached = false; shim_debug("TCP server ioctl: debugger queue detach\n"); } diff --git a/src/shim/umq/tcp_server.h b/src/shim/umq/tcp_server.h index 7d82b89d..2b6f7bd5 100644 --- a/src/shim/umq/tcp_server.h +++ b/src/shim/umq/tcp_server.h @@ -7,7 +7,6 @@ #include "dbg_hwq.h" #include "../buffer.h" #include "../hwctx.h" -#include "fw_buf_metadata.h" namespace shim_xdna { From 9efdab4f9095c7c29002cfd0220229798fe9eb22 Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Tue, 20 May 2025 17:14:17 -0700 Subject: [PATCH 5/8] change function name --- src/shim/buffer.cpp | 6 +++--- src/shim/buffer.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shim/buffer.cpp b/src/shim/buffer.cpp index 8427aa3e..9fdc4d7d 100644 --- a/src/shim/buffer.cpp +++ b/src/shim/buffer.cpp @@ -11,7 +11,7 @@ namespace { uint8_t -use_to_type(uint8_t use) +use_to_fw_debug_type(uint8_t use) { switch (use) { case XRT_BO_USE_DEBUG: @@ -394,7 +394,7 @@ set_flags(uint64_t flags) uint64_t buffer:: -get_flags() +get_flags() const { return m_flags; } @@ -585,7 +585,7 @@ config(xrt_core::hwctx_handle* hwctx, const std::map& buf_size metadata->bo_handle = id().handle; metadata->command_id = 0; //support this later auto f = xcl_bo_flags{get_flags()}; - metadata->buf_type = use_to_type(f.use); + metadata->buf_type = use_to_fw_debug_type(f.use); metadata->num_ucs = buf_sizes.size(); int i = 0; for (const auto& pair : buf_sizes) diff --git a/src/shim/buffer.h b/src/shim/buffer.h index e186f96e..d84a40cf 100644 --- a/src/shim/buffer.h +++ b/src/shim/buffer.h @@ -96,7 +96,7 @@ class buffer : public xrt_core::buffer_handle // Save flags in buffer which later returns via get_properties() void set_flags(uint64_t flags); - uint64_t get_flags(); + uint64_t get_flags() const; virtual std::set get_arg_bo_ids() const; From bb523a303febda44ca7a4685ac3bf69702807620 Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Tue, 15 Jul 2025 09:49:38 -0700 Subject: [PATCH 6/8] add missing include --- src/shim/umq/tcp_server.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/shim/umq/tcp_server.cpp b/src/shim/umq/tcp_server.cpp index db78fa07..be2b6ec6 100644 --- a/src/shim/umq/tcp_server.cpp +++ b/src/shim/umq/tcp_server.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "tcp_server.h" From debb7e93694c499a669995c8f686f18802cbf8fd Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Tue, 15 Jul 2025 22:36:52 -0700 Subject: [PATCH 7/8] use SIGUSR1 to handle tcp server exit --- src/shim/umq/hwctx.cpp | 2 +- src/shim/umq/tcp_server.cpp | 22 ++++++++++++++++++++-- src/shim/umq/tcp_server.h | 3 +++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/shim/umq/hwctx.cpp b/src/shim/umq/hwctx.cpp index 1edbe5b5..03498d95 100644 --- a/src/shim/umq/hwctx.cpp +++ b/src/shim/umq/hwctx.cpp @@ -100,7 +100,7 @@ fini_tcp_server() if (m_thread_.joinable()) { shim_debug("Kill TCP server..."); - pthread_kill(m_thread_.native_handle(), SIGINT); + pthread_kill(m_thread_.native_handle(), SIGUSR1); m_thread_.join(); } } diff --git a/src/shim/umq/tcp_server.cpp b/src/shim/umq/tcp_server.cpp index be2b6ec6..61600fdc 100644 --- a/src/shim/umq/tcp_server.cpp +++ b/src/shim/umq/tcp_server.cpp @@ -19,6 +19,7 @@ m_aie_attached(false), m_dbg_umq(dev), m_def_size(16), m_pdev(dev.get_pdev()) m_data_bo = std::make_unique(m_pdev, def_buf_size, AMDXDNA_BO_SHARE); m_data_buf = m_data_bo->vaddr(); m_data_paddr = m_data_bo->paddr(); + m_srv_stop = 0; } tcp_server:: @@ -27,10 +28,27 @@ tcp_server:: shim_debug("TCP server destructor"); } +void +tcp_server:: +sigusr1_handler(int sig) +{ + if (sig == SIGUSR1) + { + shim_debug("SIGUSR1 received!"); + m_srv_stop = 1; + } +} + void tcp_server:: start() { + struct sigaction sa; + sa.sa_handler = tcp_server::sigusr1_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SIGUSR1, &sa, nullptr); + int serverSocket = socket(AF_INET, SOCK_STREAM, 0); // specifying the address @@ -54,7 +72,7 @@ start() int clientSocket = accept(serverSocket, nullptr, nullptr); if (clientSocket < 0) { - if (errno == EINTR) + if (errno == EINTR && m_srv_stop == 1) { shim_debug("Tcp thread exit!\n"); break; @@ -82,7 +100,7 @@ start() std::this_thread::sleep_for(std::chrono::seconds(1)); continue; } - else + else if (errno == EINTR && m_srv_stop == 1) { shim_debug("Tcp connection exit!\n"); loop = false; diff --git a/src/shim/umq/tcp_server.h b/src/shim/umq/tcp_server.h index 2b6f7bd5..756172f1 100644 --- a/src/shim/umq/tcp_server.h +++ b/src/shim/umq/tcp_server.h @@ -7,6 +7,7 @@ #include "dbg_hwq.h" #include "../buffer.h" #include "../hwctx.h" +#include namespace shim_xdna { @@ -28,6 +29,7 @@ class tcp_server { bool m_aie_attached; hwctx* m_hwctx; const pdev& m_pdev; + static inline volatile std::sig_atomic_t m_srv_stop; std::unique_ptr> handle_read_mem(uint32_t addr, uint32_t length); @@ -35,6 +37,7 @@ class tcp_server { void buffer_extend(size_t new_size); uint32_t handle_attach(uint32_t); void handle_detach(); + static void sigusr1_handler(int sig); }; } From c946960f8af715ea02cc22c0f1e9af9ac953944f Mon Sep 17 00:00:00 2001 From: Brian Xu Date: Wed, 13 Aug 2025 17:15:49 -0700 Subject: [PATCH 8/8] attach debugger queue before ctx runs --- src/shim/umq/dbg_cmd.h | 2 +- src/shim/umq/dbg_hwq.cpp | 18 ++++++++++----- src/shim/umq/hwctx.cpp | 44 +++++++++++++++++++++++++++++-------- src/shim/umq/hwctx.h | 3 ++- src/shim/umq/tcp_server.cpp | 13 +++++++++-- 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/src/shim/umq/dbg_cmd.h b/src/shim/umq/dbg_cmd.h index 58a3db4d..2832666e 100644 --- a/src/shim/umq/dbg_cmd.h +++ b/src/shim/umq/dbg_cmd.h @@ -11,7 +11,7 @@ extern "C" #define DBG_PKT_SUCCESS (1) #define DBG_PKT_EXIT (2) -#define DBG_PKT_INVALID (2) +#define DBG_PKT_INVALID (3) enum dbg_packet_opcode { diff --git a/src/shim/umq/dbg_hwq.cpp b/src/shim/umq/dbg_hwq.cpp index 7b62fc48..936d1b50 100644 --- a/src/shim/umq/dbg_hwq.cpp +++ b/src/shim/umq/dbg_hwq.cpp @@ -33,6 +33,7 @@ dbg_hwq_umq(const device& dev) m_dbg_umq_pkt->xrt_header.common_header.type = HOST_QUEUE_PACKET_TYPE_INVALID; m_dbg_umq_hdr->capacity = 1; + m_dbg_umq_hdr->data_address = m_dbg_umq_bo->paddr() + header_sz; set_use_flag(); shim_debug("Created DBG UMQ HW queue"); @@ -50,7 +51,6 @@ issue_exit_cmd() { auto hdr = &m_dbg_umq_pkt->xrt_header; // always case 1 - m_dbg_umq_hdr->write_index++; auto ehp = &m_dbg_umq_pkt->xrt_header; ehp->common_header.opcode = DBG_CMD_EXIT; ehp->common_header.count = 0; @@ -65,7 +65,6 @@ issue_rw_cmd(struct rw_mem &data, uint16_t opcode) { auto hdr = &m_dbg_umq_pkt->xrt_header; // always case 1 - m_dbg_umq_hdr->write_index++; auto ehp = &m_dbg_umq_pkt->xrt_header; ehp->common_header.opcode = opcode; ehp->common_header.count = sizeof (struct rw_mem); @@ -104,16 +103,23 @@ submit() { *m_dbg_umq_comp_ptr = 0; - /* Issue mfence instruction to make sure all writes to the slot before is done */ - std::atomic_thread_fence(std::memory_order::memory_order_seq_cst); m_dbg_umq_pkt->xrt_header.common_header.type = HOST_QUEUE_PACKET_TYPE_VENDOR_SPECIFIC; + /* Issue mfence instruction to make sure all writes to the slot before is done */ + std::atomic_thread_fence(std::memory_order::memory_order_seq_cst); + m_dbg_umq_hdr->write_index++; - shim_debug("dbg umq: submit cmd"); + shim_debug("dbg umq: submit cmd widx: %ld ridx: %ld", + m_dbg_umq_hdr->write_index, + m_dbg_umq_hdr->read_index); + shim_debug("dbg umq: cmd opcode: %d count: %d", + m_dbg_umq_pkt->xrt_header.common_header.opcode, + m_dbg_umq_pkt->xrt_header.common_header.count); while (1) { - if (*m_dbg_umq_comp_ptr) + if (*m_dbg_umq_comp_ptr && + m_dbg_umq_hdr->write_index == m_dbg_umq_hdr->read_index) { return (*m_dbg_umq_comp_ptr); } diff --git a/src/shim/umq/hwctx.cpp b/src/shim/umq/hwctx.cpp index 8c583d80..f7bc5a90 100644 --- a/src/shim/umq/hwctx.cpp +++ b/src/shim/umq/hwctx.cpp @@ -3,6 +3,9 @@ #include "hwctx.h" #include "hwq.h" +#include "core/common/config_reader.h" +#include +#include namespace shim_xdna { @@ -13,7 +16,7 @@ hwctx_umq(const device& device, const xrt::xclbin& xclbin, const qos_type& qos) { shim_debug("Created UMQ HW context (%d)", get_slotidx()); xclbin_parser xp(xclbin); - m_col_cnt = xp.get_column_cnt(); + m_uc_cnt = xp.get_column_cnt() * 2; //so far single-app mode only init_tcp_server(device); init_log_buf(); @@ -24,7 +27,7 @@ hwctx_umq(const device& device, uint32_t partition_size) : hwctx(device, partition_size, std::make_unique(device, 8)) , m_pdev(device.get_pdev()) { - m_col_cnt = partition_size; + m_uc_cnt = partition_size * 2; init_tcp_server(device); init_log_buf(); @@ -45,12 +48,18 @@ void hwctx_umq:: init_log_buf() { - size_t column_size = 4096; - auto log_buf_size = m_col_cnt * column_size; + //PoC to control log buf size and filename from xrt.ini + //this setting should be in xrt so it works for all platform + //this code will be gone once the support in xrt is ready + //auto column_size = xrt_core::config::get_cert_log_buffer_size(); + auto column_size = 4096; + // make sure size is multiple of 32B + //column_size = (column_size + 31) & (~31); + m_log_buf_size = m_uc_cnt * column_size; m_log_bo = std::make_unique - (m_pdev, log_buf_size, AMDXDNA_BO_CMD); - auto log_buf = m_log_bo->vaddr(); - std::memset(log_buf, 0, log_buf_size); + (m_pdev, m_log_buf_size, AMDXDNA_BO_CMD); + m_log_buf = m_log_bo->vaddr(); + std::memset(m_log_buf, 0, m_log_buf_size); auto f = xcl_bo_flags{0}; f.use = XRT_BO_USE_LOG; @@ -61,7 +70,8 @@ init_log_buf() m_log_bo->set_flags(f.all); std::map buf_sizes; - set_metadata(buf_sizes, m_col_cnt, column_size); + set_metadata(buf_sizes, m_uc_cnt, column_size); + shim_debug("cert log uc_cnt: %d size: %d", m_uc_cnt, column_size); // TODO: configure log BO on the hwctx once driver and fw support it // we may use xrt.ini to control the config @@ -72,7 +82,23 @@ void hwctx_umq:: fini_log_buf(void) { - // Nothing to do. +#if 0 + //PoC to support log buffer setting in xrt.ini + auto path = xrt_core::config::get_cert_log_file_path(); + if (path.empty()) { + shim_debug("Skip saving cert log"); + return; + } + + std::ofstream outf(path, std::ios::binary); + if (!outf) { + shim_debug("Error opening cert log file: %s", path.c_str()); + return; + } + outf.write(static_cast(m_log_buf), m_log_buf_size); + outf.close(); + shim_debug("cert log saved to %s", path.c_str()); +#endif } void diff --git a/src/shim/umq/hwctx.h b/src/shim/umq/hwctx.h index 75a59384..5a9a00af 100644 --- a/src/shim/umq/hwctx.h +++ b/src/shim/umq/hwctx.h @@ -24,9 +24,10 @@ class hwctx_umq : public hwctx { private: const pdev& m_pdev; std::unique_ptr m_log_bo; - uint32_t m_col_cnt = 0; + uint32_t m_uc_cnt = 0; void *m_log_buf = nullptr; + size_t m_log_buf_size; std::unique_ptr m_tcp_server; std::thread m_thread_; diff --git a/src/shim/umq/tcp_server.cpp b/src/shim/umq/tcp_server.cpp index b47877c4..7ef3c3fd 100644 --- a/src/shim/umq/tcp_server.cpp +++ b/src/shim/umq/tcp_server.cpp @@ -20,6 +20,13 @@ m_aie_attached(false), m_dbg_umq(dev), m_def_size(16), m_pdev(dev.get_pdev()) m_data_buf = m_data_bo->vaddr(); m_data_paddr = m_data_bo->paddr(); m_srv_stop = 0; + + // issue ioctl to attach the dbg hsa queue + std::map buf_sizes; + buf_sizes[0] = 32; //we don't care size + + m_dbg_umq.get_dbg_umq_bo()->config(m_hwctx, buf_sizes); + shim_debug("TCP server ioctl: debugger attach\n"); } tcp_server:: @@ -287,13 +294,15 @@ buffer_extend(size_t new_size) uint32_t tcp_server:: handle_attach(uint32_t uc_index) -{ +{ +#if 0 // issue ioctl to attach the dbg hsa queue std::map buf_sizes; buf_sizes[uc_index] = 0; //we don't care size m_dbg_umq.get_dbg_umq_bo()->config(m_hwctx, buf_sizes); shim_debug("TCP server ioctl: debugger attach\n"); +#endif m_aie_attached = true; return AIE_DBG_SUCCESS; @@ -305,7 +314,7 @@ handle_detach() { m_dbg_umq.issue_exit_cmd(); // issue ioctl to detach the dbg hsa queue - m_dbg_umq.get_dbg_umq_bo()->unconfig(m_hwctx); + //m_dbg_umq.get_dbg_umq_bo()->unconfig(m_hwctx); m_aie_attached = false; shim_debug("TCP server ioctl: debugger queue detach\n");