diff --git a/gdb/ChangeLog b/gdb/ChangeLog
index e883ffcd6e..54642e1a53 100644
--- a/gdb/ChangeLog
+++ b/gdb/ChangeLog
@@ -1,3 +1,10 @@
+2015-11-27  Yao Qi  <yao.qi@linaro.org>
+
+	* nat/aarch64-linux-hw-point.c (aarch64_dr_state_remove_one_point):
+	Don't assert on alignment.
+	(aarch64_handle_breakpoint): Only check alignment when IS_INSERT
+	is true.
+
 2015-11-27  Yao Qi  <yao.qi@linaro.org>
 
 	* aarch64-tdep.c (is_hfa): Rename to ...
diff --git a/gdb/nat/aarch64-linux-hw-point.c b/gdb/nat/aarch64-linux-hw-point.c
index dcbfa98c3b..466823a432 100644
--- a/gdb/nat/aarch64-linux-hw-point.c
+++ b/gdb/nat/aarch64-linux-hw-point.c
@@ -411,7 +411,6 @@ aarch64_dr_state_remove_one_point (struct aarch64_debug_reg_state *state,
 
   /* Set up state pointers.  */
   is_watchpoint = (type != hw_execute);
-  gdb_assert (aarch64_point_is_aligned (is_watchpoint, addr, len));
   if (is_watchpoint)
     {
       num_regs = aarch64_num_wp_regs;
@@ -460,13 +459,21 @@ aarch64_handle_breakpoint (enum target_hw_bp_type type, CORE_ADDR addr,
 			   int len, int is_insert,
 			   struct aarch64_debug_reg_state *state)
 {
-  /* The hardware breakpoint on AArch64 should always be 4-byte
-     aligned, but on AArch32, it can be 2-byte aligned.  */
-  if (!aarch64_point_is_aligned (0 /* is_watchpoint */ , addr, len))
-    return -1;
-
   if (is_insert)
-    return aarch64_dr_state_insert_one_point (state, type, addr, len);
+    {
+      /* The hardware breakpoint on AArch64 should always be 4-byte
+	 aligned, but on AArch32, it can be 2-byte aligned.  Note that
+	 we only check the alignment on inserting breakpoint because
+	 aarch64_point_is_aligned needs the inferior_ptid inferior's
+	 regcache to decide whether the inferior is 32-bit or 64-bit.
+	 However when GDB follows the parent process and detach breakpoints
+	 from child process, inferior_ptid is the child ptid, but the
+	 child inferior doesn't exist in GDB's view yet.  */
+      if (!aarch64_point_is_aligned (0 /* is_watchpoint */ , addr, len))
+	return -1;
+
+      return aarch64_dr_state_insert_one_point (state, type, addr, len);
+    }
   else
     return aarch64_dr_state_remove_one_point (state, type, addr, len);
 }