diff --git a/drivers/staging/greybus/es1.c b/drivers/staging/greybus/es1.c
index 858c4aa9e28a..75253d9a942b 100644
--- a/drivers/staging/greybus/es1.c
+++ b/drivers/staging/greybus/es1.c
@@ -390,7 +390,7 @@ static void es1_destroy(struct es1_ap_dev *es1)
 	}
 
 	udev = es1->usb_dev;
-	gb_hd_remove(es1->hd);
+	gb_hd_put(es1->hd);
 
 	usb_put_dev(udev);
 }
@@ -403,6 +403,8 @@ static void ap_disconnect(struct usb_interface *interface)
 	for (i = 0; i < NUM_CPORT_IN_URB; ++i)
 		usb_kill_urb(es1->cport_in_urb[i]);
 
+	gb_hd_del(es1->hd);
+
 	es1_destroy(es1);
 }
 
@@ -692,6 +694,10 @@ static int ap_probe(struct usb_interface *interface,
 							gb_debugfs_get(), es1,
 							&apb1_log_enable_fops);
 
+	retval = gb_hd_add(hd);
+	if (retval)
+		goto error;
+
 	for (i = 0; i < NUM_CPORT_IN_URB; ++i) {
 		retval = usb_submit_urb(es1->cport_in_urb[i], GFP_KERNEL);
 		if (retval)
@@ -703,6 +709,7 @@ static int ap_probe(struct usb_interface *interface,
 err_kill_in_urbs:
 	for (--i; i >= 0; --i)
 		usb_kill_urb(es1->cport_in_urb[i]);
+	gb_hd_del(hd);
 error:
 	es1_destroy(es1);
 
diff --git a/drivers/staging/greybus/es2.c b/drivers/staging/greybus/es2.c
index 96ba1a74bb69..aad6451d90de 100644
--- a/drivers/staging/greybus/es2.c
+++ b/drivers/staging/greybus/es2.c
@@ -585,7 +585,7 @@ static void es2_destroy(struct es2_ap_dev *es2)
 
 	udev = es2->usb_dev;
 	cport_to_ep = es2->cport_to_ep;
-	gb_hd_remove(es2->hd);
+	gb_hd_put(es2->hd);
 
 	kfree(cport_to_ep);
 	usb_put_dev(udev);
@@ -599,6 +599,8 @@ static void ap_disconnect(struct usb_interface *interface)
 	for (i = 0; i < NUM_BULKS; ++i)
 		es2_cport_in_disable(es2, &es2->cport_in[i]);
 
+	gb_hd_del(es2->hd);
+
 	es2_destroy(es2);
 }
 
@@ -942,6 +944,10 @@ static int ap_probe(struct usb_interface *interface,
 							gb_debugfs_get(), es2,
 							&apb_log_enable_fops);
 
+	retval = gb_hd_add(hd);
+	if (retval)
+		goto error;
+
 	for (i = 0; i < NUM_BULKS; ++i) {
 		retval = es2_cport_in_enable(es2, &es2->cport_in[i]);
 		if (retval)
@@ -953,6 +959,7 @@ static int ap_probe(struct usb_interface *interface,
 err_disable_cport_in:
 	for (--i; i >= 0; --i)
 		es2_cport_in_disable(es2, &es2->cport_in[i]);
+	gb_hd_del(hd);
 error:
 	es2_destroy(es2);
 
diff --git a/drivers/staging/greybus/hd.c b/drivers/staging/greybus/hd.c
index 6f29eb49993a..b22d5470508a 100644
--- a/drivers/staging/greybus/hd.c
+++ b/drivers/staging/greybus/hd.c
@@ -77,6 +77,12 @@ struct gb_host_device *gb_hd_create(struct gb_hd_driver *driver,
 	hd->buffer_size_max = buffer_size_max;
 	hd->num_cports = num_cports;
 
+	return hd;
+}
+EXPORT_SYMBOL_GPL(gb_hd_create);
+
+int gb_hd_add(struct gb_host_device *hd)
+{
 	/*
 	 * Initialize AP's SVC protocol connection:
 	 *
@@ -87,16 +93,14 @@ struct gb_host_device *gb_hd_create(struct gb_hd_driver *driver,
 	 * time we will create a fully initialized svc-connection, as we need
 	 * endo-id and AP's interface id for that.
 	 */
-	if (!gb_ap_svc_connection_create(hd)) {
-		kref_put_mutex(&hd->kref, free_hd, &hd_mutex);
-		return ERR_PTR(-ENOMEM);
-	}
+	if (!gb_ap_svc_connection_create(hd))
+		return -ENOMEM;
 
-	return hd;
+	return 0;
 }
-EXPORT_SYMBOL_GPL(gb_hd_create);
+EXPORT_SYMBOL_GPL(gb_hd_add);
 
-void gb_hd_remove(struct gb_host_device *hd)
+void gb_hd_del(struct gb_host_device *hd)
 {
 	/*
 	 * Tear down all interfaces, modules, and the endo that is associated
@@ -109,7 +113,11 @@ void gb_hd_remove(struct gb_host_device *hd)
 	/* Is the SVC still using the partially uninitialized connection ? */
 	if (hd->initial_svc_connection)
 		gb_connection_destroy(hd->initial_svc_connection);
+}
+EXPORT_SYMBOL_GPL(gb_hd_del);
 
+void gb_hd_put(struct gb_host_device *hd)
+{
 	kref_put_mutex(&hd->kref, free_hd, &hd_mutex);
 }
-EXPORT_SYMBOL_GPL(gb_hd_remove);
+EXPORT_SYMBOL_GPL(gb_hd_put);
diff --git a/drivers/staging/greybus/hd.h b/drivers/staging/greybus/hd.h
index 91fcccb46df1..6724cfea1e19 100644
--- a/drivers/staging/greybus/hd.h
+++ b/drivers/staging/greybus/hd.h
@@ -52,6 +52,8 @@ struct gb_host_device *gb_hd_create(struct gb_hd_driver *driver,
 					struct device *parent,
 					size_t buffer_size_max,
 					size_t num_cports);
-void gb_hd_remove(struct gb_host_device *hd);
+int gb_hd_add(struct gb_host_device *hd);
+void gb_hd_del(struct gb_host_device *hd);
+void gb_hd_put(struct gb_host_device *hd);
 
 #endif	/* __HD_H */