diff --git a/bfd/elf.c b/bfd/elf.c
index ebcf40a925..2cc64e8d52 100644
--- a/bfd/elf.c
+++ b/bfd/elf.c
@@ -5271,7 +5271,9 @@ assign_file_positions_for_load_sections (bfd *abfd,
 	  p->p_memsz = bed->s->sizeof_ehdr;
 	  if (m->count > 0)
 	    {
-	      if (p->p_vaddr < (bfd_vma) off)
+	      if (p->p_vaddr < (bfd_vma) off
+		  || (!m->p_paddr_valid
+		      && p->p_paddr < (bfd_vma) off))
 		{
 		  (*_bfd_error_handler)
 		    (_("%B: Not enough room for program headers, try linking with -N"),
diff --git a/ld/ChangeLog b/ld/ChangeLog
index b744275132..a2493d0daf 100644
--- a/ld/ChangeLog
+++ b/ld/ChangeLog
@@ -1,3 +1,9 @@
+2016-07-20  H.J. Lu  <hongjiu.lu@intel.com>
+
+	PR ld/20376
+	* elf.c (assign_file_positions_for_load_sections): Also check
+	p_paddr for program header space.
+
 2016-07-20  Alan Modra  <amodra@gmail.com>
 
 	* ldexp.c (exp_unop, exp_binop, exp_trinop, exp_nameop): Don't