From 3f5398cfbf051dc1850b4f64fbe5267cbd699ce0 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Sat, 6 Apr 2019 15:41:35 +0300 Subject: [PATCH] habanalabs: improve IOCTLs behavior when disabled or reset This patch makes some improvement in how IOCTLs behave when the device is disabled or under reset. The new code checks, at the start of every IOCTL, if the device is disabled or in reset. If so, it prints an appropriate kernel message and returns -EBUSY to user-space. In addition, the code modifies the location of where the hard_reset_pending flag is being set or cleared: 1. It is now cleared immediately after the reset *tear-down* flow is finished but before the re-initialization flow begins. 2. It is being set in the remove function of the device, to make the behavior the same with the hard-reset flow There are two exceptions to the disable or in reset check: 1. The HL_INFO_DEVICE_STATUS opcode in the INFO IOCTL. This opcode allows the user to inquire about the status of the device, whether it is operational, in reset or malfunction (disabled). If the driver will block this IOCTL, the user won't be able to retrieve the status in case of malfunction or in reset. 2. The WAIT_FOR_CS IOCTL. This IOCTL allows the user to inquire about the status of a CS. We want to allow the user to continue to do so, even if we started a soft-reset process because it will allow the user to get the correct error code for each CS he submitted. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/command_buffer.c | 7 +++++++ drivers/misc/habanalabs/device.c | 9 +++++---- drivers/misc/habanalabs/habanalabs_ioctl.c | 3 ++- drivers/misc/habanalabs/memory.c | 3 ++- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/misc/habanalabs/command_buffer.c b/drivers/misc/habanalabs/command_buffer.c index 85f75806a9a7..b1ffca47d748 100644 --- a/drivers/misc/habanalabs/command_buffer.c +++ b/drivers/misc/habanalabs/command_buffer.c @@ -214,6 +214,13 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data) u64 handle; int rc; + if (hl_device_disabled_or_in_reset(hdev)) { + dev_warn_ratelimited(hdev->dev, + "Device is %s. Can't execute CB IOCTL\n", + atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); + return -EBUSY; + } + switch (args->in.op) { case HL_CB_OP_CREATE: rc = hl_cb_create(hdev, &hpriv->cb_mgr, args->in.cb_size, diff --git a/drivers/misc/habanalabs/device.c b/drivers/misc/habanalabs/device.c index 56d3ba53c661..25bfb093ff26 100644 --- a/drivers/misc/habanalabs/device.c +++ b/drivers/misc/habanalabs/device.c @@ -641,6 +641,8 @@ again: if ((hard_reset) && (!from_hard_reset_thread)) { struct hl_device_reset_work *device_reset_work; + hdev->hard_reset_pending = true; + if (!hdev->pdev) { dev_err(hdev->dev, "Reset action is NOT supported in simulator\n"); @@ -648,8 +650,6 @@ again: goto out_err; } - hdev->hard_reset_pending = true; - device_reset_work = kzalloc(sizeof(*device_reset_work), GFP_ATOMIC); if (!device_reset_work) { @@ -718,6 +718,7 @@ again: if (hard_reset) { hdev->device_cpu_disabled = false; + hdev->hard_reset_pending = false; if (hdev->kernel_ctx) { dev_crit(hdev->dev, @@ -779,8 +780,6 @@ again: } hl_set_max_power(hdev, hdev->max_power); - - hdev->hard_reset_pending = false; } else { rc = hdev->asic_funcs->soft_reset_late_init(hdev); if (rc) { @@ -1069,6 +1068,8 @@ void hl_device_fini(struct hl_device *hdev) hdev->asic_funcs->hw_queues_lock(hdev); hdev->asic_funcs->hw_queues_unlock(hdev); + hdev->hard_reset_pending = true; + device_kill_open_processes(hdev); hl_hwmon_fini(hdev); diff --git a/drivers/misc/habanalabs/habanalabs_ioctl.c b/drivers/misc/habanalabs/habanalabs_ioctl.c index 810e65446075..eeefb22023e9 100644 --- a/drivers/misc/habanalabs/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/habanalabs_ioctl.c @@ -202,7 +202,8 @@ static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data) if (hl_device_disabled_or_in_reset(hdev)) { dev_warn_ratelimited(hdev->dev, - "Device is disabled or in reset. Can't execute INFO IOCTL\n"); + "Device is %s. Can't execute INFO IOCTL\n", + atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); return -EBUSY; } diff --git a/drivers/misc/habanalabs/memory.c b/drivers/misc/habanalabs/memory.c index 05919f913300..43ef3ad8438a 100644 --- a/drivers/misc/habanalabs/memory.c +++ b/drivers/misc/habanalabs/memory.c @@ -1159,7 +1159,8 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) if (hl_device_disabled_or_in_reset(hdev)) { dev_warn_ratelimited(hdev->dev, - "Device is disabled or in reset. Can't execute memory IOCTL\n"); + "Device is %s. Can't execute MEMORY IOCTL\n", + atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); return -EBUSY; }