From 65ee0d66565d7eab41db37d6b19ab55d98c31d3e Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Wed, 15 Jan 2020 14:08:44 -0500 Subject: [PATCH 01/13] Fix the RPM spec file for openSUSE This commit also fixed a warning in the RPM build in openSUSE. Signed-off-by: Liming Sun --- rshim.spec | 2 +- rshim.spec.in | 2 +- src/rshim.c | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rshim.spec b/rshim.spec index ac22528..272e217 100644 --- a/rshim.spec +++ b/rshim.spec @@ -26,7 +26,7 @@ Vendor: Mellanox Technologies Obsoletes: %{name} < 2.0 %if "%{_vendor}" == "suse" -BuildRequires: libpci-dev, libusb-dev, libfuse-dev +BuildRequires: pciutils-devel, libusb-devel, fuse-devel %else BuildRequires: pciutils-devel, libusbx-devel, fuse-devel %endif diff --git a/rshim.spec.in b/rshim.spec.in index 7512321..87d3879 100644 --- a/rshim.spec.in +++ b/rshim.spec.in @@ -26,7 +26,7 @@ Vendor: Mellanox Technologies Obsoletes: %{name} < 2.0 %if "%{_vendor}" == "suse" -BuildRequires: libpci-dev, libusb-dev, libfuse-dev +BuildRequires: pciutils-devel, libusb-devel, fuse-devel %else BuildRequires: pciutils-devel, libusbx-devel, fuse-devel %endif diff --git a/src/rshim.c b/src/rshim.c index 3ad2d1f..d487861 100644 --- a/src/rshim.c +++ b/src/rshim.c @@ -2625,6 +2625,7 @@ static void rshim_rshim_ioctl(fuse_req_t req, int cmd, void *arg, rshim_backend_t *bd = fuse_req_userdata(req); rshim_ioctl_msg msg; struct iovec iov; + uint64_t data = 0; int rc = 0; switch (cmd) { @@ -2662,7 +2663,8 @@ static void rshim_rshim_ioctl(fuse_req_t req, int cmd, void *arg, rc = bd->read_rshim(bd, (msg.addr >> 16) & 0xF, /* channel # */ msg.addr & 0xFFFF, /* addr */ - &msg.data); + &data); + msg.data = data; pthread_mutex_unlock(&bd->mutex); } From 76c5bf8caceea73283f49c17d35a032ba4095423 Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Wed, 15 Jan 2020 14:38:47 -0500 Subject: [PATCH 02/13] Set default signal handlers Signed-off-by: Liming Sun --- README.md | 2 +- man/bfrshim.1 | 4 ++-- src/rshim.c | 20 +++++++++++++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4bf5167..6144c19 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ syntax: bfrshim [--help|-h] [--backend|-b usb|pcie|pcie_lf] cat /dev/rshim/misc BOOT_MODE 1 # eMMC boot mode (0:USB/PCIe, 1: eMMC) SW_RESET 0 # Set to 1 to initiate SW RESET - DRV_NAME rshim_usb # Backend driver name (display-only) + DEV_NAME rshim_usb # Backend driver name (display-only) Turning on the 'rshim_adv_cfg' flag with the '-a' command line argument could display more parameters. diff --git a/man/bfrshim.1 b/man/bfrshim.1 index fc11011..761f609 100644 --- a/man/bfrshim.1 +++ b/man/bfrshim.1 @@ -38,7 +38,7 @@ cat /dev/rshim0/misc BOOT_MODE 1 (0:rshim, 1:emmc, 2:emmc-boot-swap) BOOT_TIMEOUT 100 (seconds) SW_RESET 0 (1: reset) - DRV_NAME pcie-04:00.2 + DEV_NAME pcie-04:00.2 .fi .in @@ -61,7 +61,7 @@ cat /dev/rshim0/misc BOOT_MODE 1 (0:rshim, 1:emmc, 2:emmc-boot-swap) BOOT_TIMEOUT 100 (seconds) SW_RESET 0 (1: reset) - DRV_NAME pcie-04:00.2 + DEV_NAME pcie-04:00.2 PEER_MAC 00:1a:ca:ff:ff:01 (rw) PXE_ID 0x00000000 (rw) VLAN_ID 0 0 (rw) diff --git a/src/rshim.c b/src/rshim.c index d487861..3ab3f62 100644 --- a/src/rshim.c +++ b/src/rshim.c @@ -2363,7 +2363,7 @@ static int rshim_misc_read(struct cuse_dev *cdev, int fflags, void *peer_ptr, len -= n; /* Display the driver name. */ - n = snprintf(p, len, "%-16s%s\n", "DRV_NAME", bd->dev_name); + n = snprintf(p, len, "%-16s%s\n", "DEV_NAME", bd->dev_name); p += n; len -= n; @@ -3412,6 +3412,22 @@ static void rshim_main(int argc, char *argv[]) } } +static void rshim_signal_handler(int sig) +{ +} + +static void set_signals(void) +{ + struct sigaction sa; + + memset(&sa, 0, sizeof(struct sigaction)); + sa.sa_handler = rshim_signal_handler; + sigaction(SIGHUP, &sa, NULL); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + sigaction(SIGPIPE, &sa, NULL); +} + static void print_help(void) { printf("Usage: bfrshim [options]\n"); @@ -3508,6 +3524,8 @@ int main(int argc, char *argv[]) openlog("rshim", LOG_CONS, LOG_USER); + set_signals(); + rshim_main(argc, argv); closelog(); From ed741c381169a286c547dbea1c1f086e29916603 Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Thu, 23 Jan 2020 15:23:40 -0500 Subject: [PATCH 03/13] Add log levels for rshim logging Signed-off-by: Liming Sun --- src/rshim_log.c | 65 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/src/rshim_log.c b/src/rshim_log.c index 8fdd6de..d09edf0 100644 --- a/src/rshim_log.c +++ b/src/rshim_log.c @@ -12,6 +12,9 @@ const char * const rshim_log_mod[] = { "others", "BL1", "BL2", "BL2R", "BL31", "UEFI" }; +/* Log level */ +const char * const rshim_log_levels[] = { "INFO", "WARN", "ERR" }; + /* Log type. */ #define BF_RSH_LOG_TYPE_UNKNOWN 0x00ULL #define BF_RSH_LOG_TYPE_PANIC 0x01ULL @@ -20,14 +23,22 @@ const char * const rshim_log_mod[] = { #define BF_RSH_LOG_TYPE_MSG 0x04ULL /* Utility macro. */ -#define BF_RSH_LOG_MOD_MASK 0x0FULL -#define BF_RSH_LOG_MOD_SHIFT 60 -#define BF_RSH_LOG_TYPE_MASK 0x0FULL -#define BF_RSH_LOG_TYPE_SHIFT 56 -#define BF_RSH_LOG_LEN_MASK 0x7FULL -#define BF_RSH_LOG_LEN_SHIFT 48 -#define BF_RSH_LOG_PC_MASK 0xFFFFFFFFULL -#define BF_RSH_LOG_PC_SHIFT 0 +#define BF_RSH_LOG_MOD_MASK 0x0FULL +#define BF_RSH_LOG_MOD_SHIFT 60 +#define BF_RSH_LOG_TYPE_MASK 0x0FULL +#define BF_RSH_LOG_TYPE_SHIFT 56 +#define BF_RSH_LOG_LEN_MASK 0x7FULL +#define BF_RSH_LOG_LEN_SHIFT 48 +#define BF_RSH_LOG_ARG_MASK 0xFFFFFFFFULL +#define BF_RSH_LOG_ARG_SHIFT 16 +#define BF_RSH_LOG_HAS_ARG_MASK 0xFFULL +#define BF_RSH_LOG_HAS_ARG_SHIFT 8 +#define BF_RSH_LOG_LEVEL_MASK 0xFFULL +#define BF_RSH_LOG_LEVEL_SHIFT 0 +#define BF_RSH_LOG_PC_MASK 0xFFFFFFFFULL +#define BF_RSH_LOG_PC_SHIFT 0 +#define BF_RSH_LOG_SYNDROME_MASK 0xFFFFFFFFULL +#define BF_RSH_LOG_SYNDROME_SHIFT 0 #define BF_RSH_LOG_HEADER_GET(f, h) \ (((h) >> BF_RSH_LOG_##f##_SHIFT) & BF_RSH_LOG_##f##_MASK) @@ -170,7 +181,7 @@ static int rshim_log_show_crash(rshim_backend_t *bd, uint64_t hdr, char *buf, int rc = 0, i, module, type, len, n = 0; uint64_t opcode, data; char *p = buf; - uint32_t pc; + uint32_t pc, syndrome, ec; module = BF_RSH_LOG_HEADER_GET(MOD, hdr); if (module >= sizeof(rshim_log_mod) / sizeof(rshim_log_mod[0])) @@ -178,11 +189,17 @@ static int rshim_log_show_crash(rshim_backend_t *bd, uint64_t hdr, char *buf, type = BF_RSH_LOG_HEADER_GET(TYPE, hdr); len = BF_RSH_LOG_HEADER_GET(LEN, hdr); - if (type == BF_RSH_LOG_TYPE_EXCEPTION) - n = snprintf(p, size, "Exception(%s):\n", rshim_log_mod[module]); + if (type == BF_RSH_LOG_TYPE_EXCEPTION) { + syndrome = BF_RSH_LOG_HEADER_GET(SYNDROME, hdr); + ec = syndrome >> 26; + n = snprintf(p, size, " Exception(%s): syndrome = 0x%x%s\n", + rshim_log_mod[module], syndrome, + (ec == 0x24 || ec == 0x25) ? "(Data Abort)" : + (ec == 0x2f) ? "(SError)" : ""); + } else if (type == BF_RSH_LOG_TYPE_PANIC) { pc = BF_RSH_LOG_HEADER_GET(PC, hdr); - n = snprintf(p, size, "PANIC(%s): PC = 0x%x\n", rshim_log_mod[module], pc); + n = snprintf(p, size, " PANIC(%s): PC = 0x%x\n", rshim_log_mod[module], pc); } p += n; size -= n; @@ -198,7 +215,7 @@ static int rshim_log_show_crash(rshim_backend_t *bd, uint64_t hdr, char *buf, opcode = (le64toh(opcode) >> AARCH64_MRS_REG_SHIFT) & AARCH64_MRS_REG_MASK; - n = snprintf(p, size, " %-16s0x%llx\n", rshim_log_get_reg_name(opcode), + n = snprintf(p, size, " %-16s0x%llx\n", rshim_log_get_reg_name(opcode), (unsigned long long)data); p += n; size -= n; @@ -210,13 +227,23 @@ static int rshim_log_show_crash(rshim_backend_t *bd, uint64_t hdr, char *buf, static int rshim_log_show_msg(rshim_backend_t *bd, uint64_t hdr, char *buf, int size) { - int rc, len = BF_RSH_LOG_HEADER_GET(LEN, hdr); + int rc; + int module = BF_RSH_LOG_HEADER_GET(MOD, hdr); + int len = BF_RSH_LOG_HEADER_GET(LEN, hdr); + int level = BF_RSH_LOG_HEADER_GET(LEVEL, hdr); + int has_arg = BF_RSH_LOG_HEADER_GET(HAS_ARG, hdr); + uint32_t arg = BF_RSH_LOG_HEADER_GET(ARG, hdr); uint64_t data; char *tmp, *p; if (len <= 0) return -EINVAL; + if (module >= sizeof(rshim_log_mod) / sizeof(rshim_log_mod[0])) + module = 0; + if (level >= sizeof(rshim_log_levels) / sizeof(rshim_log_levels[0])) + level = 0; + tmp = malloc(len * sizeof(uint64_t) + 1); if (!tmp) return 0; @@ -230,7 +257,15 @@ static int rshim_log_show_msg(rshim_backend_t *bd, uint64_t hdr, char *buf, p += sizeof(data); } *p = '\0'; - len = snprintf(buf, size, " %s\n", tmp); + if (!has_arg) { + len = snprintf(buf, size, " %s[%s]: %s\n", rshim_log_levels[level], + rshim_log_mod[module], tmp); + } else { + len = snprintf(buf, size, " %s[%s]: ", rshim_log_levels[level], + rshim_log_mod[module]); + len += snprintf(buf + len, size - len, tmp, arg); + len += snprintf(buf + len, size - len, "\n"); + } free(tmp); return len; From d760c17a1b74d8d19e97daae66d42d3a7fbec0ab Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Tue, 28 Jan 2020 20:55:10 -0500 Subject: [PATCH 04/13] Fix the rshim index base when multiple instances exist Signed-off-by: Liming Sun --- man/bfrshim.1 | 5 +++++ src/rshim.c | 9 +++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/man/bfrshim.1 b/man/bfrshim.1 index 761f609..30e974a 100644 --- a/man/bfrshim.1 +++ b/man/bfrshim.1 @@ -92,6 +92,11 @@ cat /dev/rshim0/misc -f, --foreground Run in forground. +-i, --index + Specify the index to create device path /dev/rshim. It's also used + to create network interface name tmfifo_net. This option is needed + when multiple bfrshim instances are running. + -l, --log-level Log level (0:none, 1:error, 2:warning, 3:notice, 4:debug) Log messages will be printed to standard output when running in foreground, or in syslog when running as a daemon. diff --git a/src/rshim.c b/src/rshim.c index 3ab3f62..10fcf8e 100644 --- a/src/rshim.c +++ b/src/rshim.c @@ -3443,12 +3443,13 @@ static void print_help(void) int main(int argc, char *argv[]) { - static const char short_options[] = "b:d:fhl:m:v"; + static const char short_options[] = "b:d:fhi:l:v"; static struct option long_options[] = { { "backend", required_argument, NULL, 'b' }, { "device", required_argument, NULL, 'd' }, { "foreground", no_argument, NULL, 'f' }, { "help", no_argument, NULL, 'h' }, + { "index", required_argument, NULL, 'h' }, { "log-level", required_argument, NULL, 'l' }, { NULL, 0, NULL, 0 } }; @@ -3467,6 +3468,9 @@ int main(int argc, char *argv[]) case 'f': rshim_daemon_mode = false; break; + case 'i': + rshim_index_base = atoi(optarg); + break; case 'l': rshim_log_level = atoi(optarg); if (rshim_log_level == 1) @@ -3478,9 +3482,6 @@ int main(int argc, char *argv[]) else if (rshim_log_level >= 4) rshim_log_level = LOG_DEBUG; break; - case 'm': - rshim_index_base = atoi(optarg); - break; case 'v': #if defined(PACKAGE_NAME) && defined(VERSION) printf(PACKAGE_NAME " " VERSION "\n"); From 19e18c9174982317275e94666c478d9b9e754d0f Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Wed, 19 Feb 2020 17:06:43 -0500 Subject: [PATCH 05/13] Add support for BlueField-2 Signed-off-by: Liming Sun --- src/rshim.c | 19 +++++++++++++++- src/rshim_usb.c | 58 ++++++++++++++++++++++++++++++------------------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/src/rshim.c b/src/rshim.c index 10fcf8e..a1483b6 100644 --- a/src/rshim.c +++ b/src/rshim.c @@ -48,6 +48,9 @@ static int rshim_keepalive_period = 300; /* Boot timeout in seconds. */ static int rshim_boot_timeout = 100; +/* Skip SW_RESET while pushing boot stream. */ +static int rshim_skip_boot_reset; + #define RSH_KEEPALIVE_MAGIC_NUM 0x5089836482ULL /* Circular buffer macros. */ @@ -478,7 +481,7 @@ static int wait_for_boot_done(rshim_backend_t *bd) { int rc; - if (!bd->has_reprobe) + if (!bd->has_reprobe || rshim_skip_boot_reset) return 0; if (!bd->has_rshim || bd->is_booting) { @@ -625,6 +628,9 @@ static int rshim_boot_open(struct cuse_dev *cdev, int fflags) bd->write_rshim(bd, RSH_MMIO_ADDRESS_SPACE__CHANNEL_VAL_WDOG1, RSH_ARM_WDG_CONTROL_WCS, 0); + if (rshim_skip_boot_reset) + goto boot_open_done; + /* SW reset. */ rc = rshim_write_reset_control(bd); @@ -671,6 +677,7 @@ static int rshim_boot_open(struct cuse_dev *cdev, int fflags) if (rc) RSHIM_ERR("boot_open: got error %d on reset write\n", rc); +boot_open_done: pthread_mutex_unlock(&bd->mutex); if (!bd->has_reprobe) @@ -2370,6 +2377,12 @@ static int rshim_misc_read(struct cuse_dev *cdev, int fflags, void *peer_ptr, if (rshim_misc_level == 1) { gettimeofday(&tp, NULL); + /* Skip SW_RESET while pushing boot stream. */ + n = snprintf(p, len, "%-16s%d (1: skip)\n", "BOOT_RESET_SKIP", + rshim_skip_boot_reset); + p += n; + len -= n; + /* * Display the target-side information. Send a request and wait for * some time for the response. @@ -2513,6 +2526,10 @@ static int rshim_misc_write(struct cuse_dev *cdev, int fflags, pthread_mutex_unlock(&bd->mutex); } } + } else if (strcmp(key, "BOOT_RESET_SKIP") == 0) { + if (sscanf(p, "%x", &value) != 1) + goto invalid; + rshim_skip_boot_reset = value; } else if (strcmp(key, "PEER_MAC") == 0) { if (sscanf(p, "%x:%x:%x:%x:%x:%x", &mac[0], &mac[1], &mac[2], &mac[3], &mac[4], &mac[5]) != 6) diff --git a/src/rshim_usb.c b/src/rshim_usb.c index fca50f8..97e85ae 100644 --- a/src/rshim_usb.c +++ b/src/rshim_usb.c @@ -13,8 +13,9 @@ #include "rshim.h" /* Our USB vendor/product IDs. */ -#define USB_TILERA_VENDOR_ID 0x22dc /* Tilera Corporation */ -#define USB_BLUEFIELD_PRODUCT_ID 0x0004 /* Mellanox Bluefield-1 */ +#define USB_TILERA_VENDOR_ID 0x22dc /* Tilera Corporation */ +#define USB_BLUEFIELD_1_PRODUCT_ID 0x0004 /* Mellanox Bluefield-1 */ +#define USB_BLUEFIELD_2_PRODUCT_ID 0x0214 /* Mellanox Bluefield-2 */ #define READ_RETRIES 5 #define WRITE_RETRIES 5 @@ -56,6 +57,11 @@ static libusb_context *rshim_usb_ctx; static int rshim_usb_epoll_fd; static bool rshim_usb_need_probe; +static int rshim_usb_product_ids[] = { + USB_BLUEFIELD_1_PRODUCT_ID, + USB_BLUEFIELD_2_PRODUCT_ID +}; + static void rshim_usb_delete(rshim_backend_t *bd) { rshim_usb_t *dev = container_of(bd, rshim_usb_t, bd); @@ -880,7 +886,7 @@ static bool rshim_usb_probe(void) { libusb_context *ctx = rshim_usb_ctx; libusb_device **devs, *dev; - int rc, i = 0; + int rc, i = 0, j, num; rc = libusb_get_device_list(ctx, &devs); if (rc < 0) { @@ -895,9 +901,14 @@ static bool rshim_usb_probe(void) if (rc) continue; - if (desc.idVendor == USB_TILERA_VENDOR_ID && - desc.idProduct == USB_BLUEFIELD_PRODUCT_ID) - rshim_usb_probe_one(ctx, dev); + if (desc.idVendor != USB_TILERA_VENDOR_ID) + continue; + + num = sizeof(rshim_usb_product_ids) / sizeof(rshim_usb_product_ids[0]); + for (j = 0; j < num; j++) { + if (desc.idProduct == rshim_usb_product_ids[j]) + rshim_usb_probe_one(ctx, dev); + } } rc = rshim_usb_add_poll(ctx); @@ -910,7 +921,7 @@ static bool rshim_usb_probe(void) int rshim_usb_init(int epoll_fd) { libusb_context *ctx = NULL; - int rc; + int rc, i, num; rc = libusb_init(&ctx); if (rc < 0) { @@ -927,21 +938,24 @@ int rshim_usb_init(int epoll_fd) rshim_usb_epoll_fd = epoll_fd; #if LIBUSB_API_VERSION >= 0x01000102 - rc = libusb_hotplug_register_callback(ctx, - LIBUSB_HOTPLUG_EVENT_DEVICE_ARRIVED | - LIBUSB_HOTPLUG_EVENT_DEVICE_LEFT, - LIBUSB_HOTPLUG_ENUMERATE, - USB_TILERA_VENDOR_ID, - USB_BLUEFIELD_PRODUCT_ID, - LIBUSB_HOTPLUG_MATCH_ANY, - rshim_hotplug_callback, - NULL, - &rshim_hotplug_handle); - if (rc != LIBUSB_SUCCESS) { - RSHIM_ERR("failed to register hotplug callback\n"); - libusb_exit(ctx); - rshim_usb_ctx = NULL; - return rc; + num = sizeof(rshim_usb_product_ids) / sizeof(rshim_usb_product_ids[0]); + for (i = 0; i < num; i++) { + rc = libusb_hotplug_register_callback(ctx, + LIBUSB_HOTPLUG_EVENT_DEVICE_ARRIVED | + LIBUSB_HOTPLUG_EVENT_DEVICE_LEFT, + LIBUSB_HOTPLUG_ENUMERATE, + USB_TILERA_VENDOR_ID, + rshim_usb_product_ids[i], + LIBUSB_HOTPLUG_MATCH_ANY, + rshim_hotplug_callback, + NULL, + &rshim_hotplug_handle); + if (rc != LIBUSB_SUCCESS) { + RSHIM_ERR("failed to register hotplug callback\n"); + libusb_exit(ctx); + rshim_usb_ctx = NULL; + return rc; + } } #else rshim_usb_probe(); From 27f10fe6c9b7003a661db94aaf12e601b58566bf Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Thu, 20 Feb 2020 10:50:42 -0500 Subject: [PATCH 06/13] Fix potential stuck when pushing boot stream over USB When BlueField crashes when pushing boot stream over USB, the cat command could get stuck and not able to be killed by 'ctrl + c'. This commit fixes this issue by returning an error code. Signed-off-by: Liming Sun --- src/rshim.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/rshim.c b/src/rshim.c index a1483b6..304b262 100644 --- a/src/rshim.c +++ b/src/rshim.c @@ -773,9 +773,8 @@ static int rshim_boot_write(struct cuse_dev *cdev, int fflags, bytes_written += len; bd->boot_rem_cnt = 0; } else if (rc == 0) { - /* Wait for some time instead of busy polling. */ - usleep(1000); - continue; + rc = -EINTR; + break; } if (rc != buf_bytes) break; From 9b5806b434d463c73dfc6008d68218b7acf4abcc Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Thu, 27 Feb 2020 14:20:42 -0500 Subject: [PATCH 07/13] rshim/usb: fix a timeout issue when running in VM This commit increases the timeout value to fix an issue found when running the driver in VM. Signed-off-by: Liming Sun --- src/rshim_usb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rshim_usb.c b/src/rshim_usb.c index 97e85ae..e86e45f 100644 --- a/src/rshim_usb.c +++ b/src/rshim_usb.c @@ -19,7 +19,7 @@ #define READ_RETRIES 5 #define WRITE_RETRIES 5 -#define RSHIM_USB_TIMEOUT 2000 +#define RSHIM_USB_TIMEOUT 20000 /* Structure to hold all of our device specific stuff. */ typedef struct { From 8301c532e0ee4ec40436f96d56d10556c9e1ab96 Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Tue, 3 Mar 2020 09:07:48 -0500 Subject: [PATCH 08/13] Enable compilation in Fedora mock The Fedora mock removes gcc by default. The .spec file needs to be updated accordingly for the required packages during build. Signed-off-by: Liming Sun --- configure.ac | 2 ++ rshim.spec | 2 ++ rshim.spec.in | 2 ++ 3 files changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index c26d064..84421c9 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,9 @@ # AC_INIT([Rshim Driver for BlueField SoC], [2.0]) +AC_CONFIG_AUX_DIR(config) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) +AC_LANG(C) AC_PROG_CC AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/rshim.spec b/rshim.spec index 272e217..3281575 100644 --- a/rshim.spec +++ b/rshim.spec @@ -25,6 +25,8 @@ Vendor: Mellanox Technologies Obsoletes: %{name} < 2.0 +BuildRequires: gcc, autoconf, automake + %if "%{_vendor}" == "suse" BuildRequires: pciutils-devel, libusb-devel, fuse-devel %else diff --git a/rshim.spec.in b/rshim.spec.in index 87d3879..4695a35 100644 --- a/rshim.spec.in +++ b/rshim.spec.in @@ -25,6 +25,8 @@ Vendor: Mellanox Technologies Obsoletes: %{name} < 2.0 +BuildRequires: gcc, autoconf, automake + %if "%{_vendor}" == "suse" BuildRequires: pciutils-devel, libusb-devel, fuse-devel %else From 63460829d5ec8ac83d3034303cba72d1d7cff30f Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Fri, 6 Mar 2020 19:59:35 -0500 Subject: [PATCH 09/13] Add the LICENSE file Signed-off-by: Liming Sun --- LICENSE | 13 +++++++++++++ rshim.spec | 1 + rshim.spec.in | 1 + 3 files changed, 15 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fe50396 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License version 2 as +published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt + diff --git a/rshim.spec b/rshim.spec index 3281575..d0c4391 100644 --- a/rshim.spec +++ b/rshim.spec @@ -74,6 +74,7 @@ killall -9 bfrshim %endif %files +%license LICENSE %defattr(-,root,root,-) %%doc README.md %if "%{WITH_SYSTEMD}" == "1" diff --git a/rshim.spec.in b/rshim.spec.in index 4695a35..6c1b207 100644 --- a/rshim.spec.in +++ b/rshim.spec.in @@ -74,6 +74,7 @@ killall -9 bfrshim %endif %files +%license LICENSE %defattr(-,root,root,-) %%doc README.md %if "%{WITH_SYSTEMD}" == "1" From b4bd28efc01ab9d387cb6cf47b08b677bd12dd48 Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Sat, 7 Mar 2020 10:19:58 -0500 Subject: [PATCH 10/13] Add --enable-usb and --enable-pcie to configure This commit adds the '--enable-' support to compile out the USB or PCIe backends if needed. Signed-off-by: Liming Sun --- Makefile.freebsd | 2 +- configure.ac | 41 +++++++++++++++++++++++++++++------------ src/Makefile.am | 11 ++++++++++- src/rshim.c | 1 - src/rshim.h | 18 ++++++++++++------ 5 files changed, 52 insertions(+), 21 deletions(-) diff --git a/Makefile.freebsd b/Makefile.freebsd index 1fb207b..1a9da40 100644 --- a/Makefile.freebsd +++ b/Makefile.freebsd @@ -21,8 +21,8 @@ CFLAGS = \ -D_FILE_OFFSET_BITS=64 \ -DHAVE_RSHIM_NET \ -DHAVE_RSHIM_CUSE \ + -DHAVE_RSHIM_USB \ -DHAVE_RSHIM_PCIE \ - -DHAVE_RSHIM_PCIE_LF \ -g LIBS = -L$(LOCALBASE)/lib \ diff --git a/configure.ac b/configure.ac index 84421c9..d5d415b 100644 --- a/configure.ac +++ b/configure.ac @@ -16,6 +16,18 @@ AC_CONFIG_FILES([ ]) AC_CANONICAL_HOST +AC_ARG_ENABLE([usb], + AS_HELP_STRING([--enable-usb], [Enable rshim over USB (default is yes) ]), + [build_usb=$enableval], [build_usb=yes]) + +AM_CONDITIONAL([BUILD_RSHIM_USB], [test "x$build_usb" = "xyes"]) + +AC_ARG_ENABLE([pcie], + AS_HELP_STRING([--enable-pcie], [Enable rshim over PCIe (default is yes) ]), + [build_pcie=$enableval], [build_pcie=yes]) + +AM_CONDITIONAL([BUILD_RSHIM_PCIE], [test "x$build_pcie" = "xyes"]) + case $host in *-linux*) AC_MSG_RESULT([Linux]) @@ -33,19 +45,24 @@ AC_SUBST(CPPFLAGS, "$CPPFLAGS -D_FILE_OFFSET_BITS=64") AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_NET") AC_SUBST(CPPFLAGS, "$CPPFLAGS -I/usr/local/include") -AC_CHECK_HEADERS([pci/pci.h],[],[AC_MSG_ERROR([Missing pciutils header files])]) -AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_PCIE -DHAVE_RSHIM_PCIE_LF") -AC_CHECK_LIB(pci, pci_init, [], [AC_MSG_ERROR([Missing libpci])]) +AS_IF([test "x$build_pcie" = "xyes"], [ + AC_CHECK_HEADERS([pci/pci.h],[],[AC_MSG_ERROR([Missing pciutils header files])]) + AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_PCIE") + AC_CHECK_LIB(pci, pci_init, [], [AC_MSG_ERROR([Missing libpci])]) +]) -if test $backend = freebsd; then -AC_CHECK_HEADERS([libusb.h],[],[AC_MSG_ERROR([Missing libusb header files])]) -AC_CHECK_LIB(usb, libusb_init, [], [AC_MSG_ERROR([Missing libusb])]) -else -AC_CHECK_HEADERS([libusb-1.0/libusb.h],[],[AC_MSG_ERROR([Missing libusb header files])]) -AC_SUBST(CPPFLAGS, "$CPPFLAGS -I/usr/include/libusb-1.0 -I/usr/local/include/libusb-1.0") -AC_CHECK_LIB(usb-1.0, libusb_init, [], [AC_MSG_ERROR([Missing libusb-1.0])]) -fi -AC_CHECK_FUNCS_ONCE([libusb_get_port_numbers libusb_get_device_address]) +AS_IF([test "x$build_usb" = "xyes"], [ + if test $backend = freebsd; then + AC_CHECK_HEADERS([libusb.h],[],[AC_MSG_ERROR([Missing libusb header files])]) + AC_CHECK_LIB(usb, libusb_init, [], [AC_MSG_ERROR([Missing libusb])]) + else + AC_CHECK_HEADERS([libusb-1.0/libusb.h],[],[AC_MSG_ERROR([Missing libusb header files])]) + AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_USB") + AC_SUBST(CPPFLAGS, "$CPPFLAGS -I/usr/include/libusb-1.0 -I/usr/local/include/libusb-1.0") + AC_CHECK_LIB(usb-1.0, libusb_init, [], [AC_MSG_ERROR([Missing libusb-1.0])]) + fi + AC_CHECK_FUNCS([libusb_get_port_numbers libusb_get_device_address]) +]) AC_CHECK_HEADERS([fuse/cuse_lowlevel.h],[],[AC_MSG_ERROR([Missing fuse header files])]) if test $backend = freebsd; then diff --git a/src/Makefile.am b/src/Makefile.am index fa6c113..93b9bab 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -3,4 +3,13 @@ # bin_PROGRAMS = bfrshim -bfrshim_SOURCES = rshim.c rshim_log.c rshim_net.c rshim_pcie.c rshim_pcie_lf.c rshim_usb.c + +bfrshim_SOURCES = rshim.c rshim_log.c rshim_net.c + +if BUILD_RSHIM_USB +bfrshim_SOURCES += rshim_usb.c +endif + +if BUILD_RSHIM_PCIE +bfrshim_SOURCES += rshim_pcie.c rshim_pcie_lf.c +endif diff --git a/src/rshim.c b/src/rshim.c index 304b262..c147b32 100644 --- a/src/rshim.c +++ b/src/rshim.c @@ -6,7 +6,6 @@ #include #include -#include #include #include #include diff --git a/src/rshim.h b/src/rshim.h index 9441962..df621ca 100644 --- a/src/rshim.h +++ b/src/rshim.h @@ -424,23 +424,29 @@ int rshim_log_show(rshim_backend_t *bd, char *buf, int len); bool rshim_allow_device(const char *devname); /* USB backend APIs. */ +#ifdef HAVE_RSHIM_USB int rshim_usb_init(int epoll_fd); void rshim_usb_poll(void); +#else +static inline int rshim_usb_init(int epoll_fd) +{ + return -1; +} +static inline void rshim_usb_poll(void) +{ +} +#endif -/* PCIe backend APIs. */ +/* PCIe & PCIe livefish backend APIs. */ #ifdef HAVE_RSHIM_PCIE int rshim_pcie_init(void); +int rshim_pcie_lf_init(void); #else static inline int rshim_pcie_init(void) { return -1; } -#endif -/* PCIe livefish backend APIs. */ -#ifdef HAVE_RSHIM_PCIE_LF -int rshim_pcie_lf_init(void); -#else static inline int rshim_pcie_lf_init(void) { return -1; From d28f5c1b666218333a01445983929e4ca5f232af Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Sat, 7 Mar 2020 13:36:45 -0500 Subject: [PATCH 11/13] Fix the hard-coded header path in configure.ac This commit removes the hard-coded path in configure.ac which causes compiling issues when doing cross-compiling. It uses PKG_CHECK_MODULES to figure out the correct path and library name. Signed-off-by: Liming Sun --- README.md | 2 +- configure.ac | 27 +++++++++------------------ src/Makefile.am | 12 ++++++++++++ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 6144c19..a645faa 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ internal rshim registers. FreeBSD: Require FreeBSD 12.0+ with packages autoconf, automake, gmake, libepoll-shim, - libpciaccess, libpci. + libpciaccess, libpci, pkgconf. Follow the same steps as build in Linux, or use 'gmake -f Makefile.freebsd' to build it. Use 'gmake install' to install it. diff --git a/configure.ac b/configure.ac index d5d415b..16f6029 100644 --- a/configure.ac +++ b/configure.ac @@ -7,6 +7,7 @@ AC_CONFIG_AUX_DIR(config) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_LANG(C) AC_PROG_CC +AM_PROG_CC_C_O AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ Makefile @@ -41,37 +42,27 @@ case $host in AC_MSG_ERROR([unsupported operating system $host]) esac -AC_SUBST(CPPFLAGS, "$CPPFLAGS -D_FILE_OFFSET_BITS=64") -AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_NET") -AC_SUBST(CPPFLAGS, "$CPPFLAGS -I/usr/local/include") - AS_IF([test "x$build_pcie" = "xyes"], [ - AC_CHECK_HEADERS([pci/pci.h],[],[AC_MSG_ERROR([Missing pciutils header files])]) - AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_PCIE") - AC_CHECK_LIB(pci, pci_init, [], [AC_MSG_ERROR([Missing libpci])]) + PKG_CHECK_MODULES(libpci, libpci, [], [AC_MSG_ERROR([Can't find libpci])]) ]) AS_IF([test "x$build_usb" = "xyes"], [ + PKG_CHECK_MODULES(libusb, libusb-1.0 >= 1.0, [], [AC_MSG_ERROR([Can't find libusb-1.0])]) if test $backend = freebsd; then - AC_CHECK_HEADERS([libusb.h],[],[AC_MSG_ERROR([Missing libusb header files])]) - AC_CHECK_LIB(usb, libusb_init, [], [AC_MSG_ERROR([Missing libusb])]) + AC_CHECK_LIB(usb, libusb_init, [], [AC_MSG_ERROR([Missing libusb_init in libusb])]) else - AC_CHECK_HEADERS([libusb-1.0/libusb.h],[],[AC_MSG_ERROR([Missing libusb header files])]) - AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_USB") - AC_SUBST(CPPFLAGS, "$CPPFLAGS -I/usr/include/libusb-1.0 -I/usr/local/include/libusb-1.0") - AC_CHECK_LIB(usb-1.0, libusb_init, [], [AC_MSG_ERROR([Missing libusb-1.0])]) + AC_CHECK_LIB(usb-1.0, libusb_init, [], [AC_MSG_ERROR([Missing libusb_init in libusb-1.0])]) fi AC_CHECK_FUNCS([libusb_get_port_numbers libusb_get_device_address]) ]) -AC_CHECK_HEADERS([fuse/cuse_lowlevel.h],[],[AC_MSG_ERROR([Missing fuse header files])]) +PKG_CHECK_MODULES(fuse, fuse, [], [AC_MSG_ERROR([Can't find fuse])]) if test $backend = freebsd; then -AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_CUSE") -AC_CHECK_LIB(cuse, cuse_dev_create) + AC_SUBST(CPPFLAGS, "$CPPFLAGS -DHAVE_RSHIM_CUSE") + AC_CHECK_LIB(cuse, cuse_dev_create) else -AC_SUBST(CPPFLAGS, "$CPPFLAGS -DFUSE_USE_VERSION=30 -DHAVE_RSHIM_FUSE") + AC_SUBST(CPPFLAGS, "$CPPFLAGS -DFUSE_USE_VERSION=30 -DHAVE_RSHIM_FUSE") fi -AC_CHECK_LIB(fuse, cuse_lowlevel_setup, [], [AC_MSG_ERROR([Missing fuse library])]) if test $backend = freebsd; then AC_CHECK_HEADERS([libepoll-shim/sys/epoll.h],[],[AC_MSG_ERROR([Missing libepoll-shim])]) diff --git a/src/Makefile.am b/src/Makefile.am index 93b9bab..53dd5ed 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -5,11 +5,23 @@ bin_PROGRAMS = bfrshim bfrshim_SOURCES = rshim.c rshim_log.c rshim_net.c +bfrshim_CPPFLAGS = -DHAVE_RSHIM_NET +bfrshim_LDFLAGS = +# USB if BUILD_RSHIM_USB bfrshim_SOURCES += rshim_usb.c +bfrshim_CPPFLAGS += $(libusb_CFLAGS) -DHAVE_RSHIM_USB +bfrshim_LDFLAGS += $(libusb_LIBS) endif +# PCIe if BUILD_RSHIM_PCIE bfrshim_SOURCES += rshim_pcie.c rshim_pcie_lf.c +bfrshim_CPPFLAGS += $(libpci_CFLAGS) -DHAVE_RSHIM_PCIE +bfrshim_LDFLAGS += $(libpci_LIBS) endif + +# FUSE / CUSE +bfrshim_CPPFLAGS += $(fuse_CFLAGS) +bfrshim_LDFLAGS += $(fuse_LIBS) From f3d92bc7cf3af41e866267abae502f6619766e3c Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Sun, 8 Mar 2020 12:36:43 -0400 Subject: [PATCH 12/13] Update the systemd service to be restartable by default Signed-off-by: Liming Sun --- bfrshim.service | 1 + 1 file changed, 1 insertion(+) diff --git a/bfrshim.service b/bfrshim.service index 35db935..d51ea0f 100644 --- a/bfrshim.service +++ b/bfrshim.service @@ -7,6 +7,7 @@ Documentation=man:bfrshim(8) After=network.target [Service] +Restart=always Type=forking ExecStart=-/usr/bin/bfrshim $OPTIONS KillMode=process From a29ecfb5ce1c9757a6c651d74a6004d3877c1c97 Mon Sep 17 00:00:00 2001 From: Liming Sun Date: Tue, 10 Mar 2020 10:24:04 -0400 Subject: [PATCH 13/13] Add retry when bring up the network interface During SW reset, the network interface could still be busy. This commit adds a robust retry when the returning code is EBUSY. Signed-off-by: Liming Sun --- src/rshim.c | 5 ++--- src/rshim_net.c | 12 +++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/rshim.c b/src/rshim.c index c147b32..2519406 100644 --- a/src/rshim.c +++ b/src/rshim.c @@ -3187,10 +3187,9 @@ int rshim_register(rshim_backend_t *bd) sizeof(init_console_termios)); bd->index = index; - if (rshim_dev_names[index] != bd->dev_name) { + if (rshim_dev_names[index]) free(rshim_dev_names[index]); - rshim_dev_names[index] = bd->dev_name; - } + rshim_dev_names[index] = strdup(bd->dev_name); rshim_devs[index] = bd; for (i = 0; i < 2; i++) { diff --git a/src/rshim_net.c b/src/rshim_net.c index 319f9c3..39834b4 100644 --- a/src/rshim_net.c +++ b/src/rshim_net.c @@ -51,7 +51,7 @@ static int rshim_if_open(char *ifname, int index) { char cmd[128]; struct ifreq ifr; - int s, fd, rc; + int s, fd, rc, retry; rc = system("modprobe tun"); if (rc == -1) @@ -67,8 +67,14 @@ static int rshim_if_open(char *ifname, int index) ifr.ifr_flags = IFF_TAP | IFF_NO_PI; strncpy(ifr.ifr_name, ifname, IFNAMSIZ); - if (ioctl(fd, TUNSETIFF, (void *) &ifr) < 0) { - RSHIM_ERR("ioctl failed: %m\n"); + retry = 16; + do { + rc = ioctl(fd, TUNSETIFF, (void *) &ifr); + if (rc == -1 && retry) + sleep(1); + } while (rc == -1 && errno == EBUSY && retry--); + if (rc < 0) { + RSHIM_ERR("ioctl failed: %m, errno=%d\n", errno); close(fd); return -1; }