OpenE2K/gcc
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

PR libgfortran/83649 Chunk large reads and writes

Browse Source 
Backport from trunk.

It turns out that Linux never reads or writes more than 2147479552
bytes in a single syscall. For writes this is not a problem as
libgfortran already contains a loop around write() to handle short
writes. But for reads we cannot do this, since then read will hang if
we have a short read when reading from the terminal.  Also, there are
reports that macOS fails I/O's larger than 2 GB.  Thus, to work around
these issues do large reads/writes in chunks.

The testcase from the PR

program largewr
  integer(kind=1) :: a(2_8**31+1)
  a = 0
  a(size(a, kind=8)) = 1
  open(10, file="largewr.dat", access="stream", form="unformatted")
  write (10) a
  close(10)
  a(size(a, kind=8)) = 2
  open(10, file="largewr.dat", access="stream", form="unformatted")
  read (10) a
  if (a(size(a, kind=8)) == 1) then
    print *, "All is well"
  else
    print *, "Oh no"
  end if
end program largewr

fails on trunk but works with the patch.

Regtested on x86_64-pc-linux-gnu, committed to trunk.

libgfortran/ChangeLog:

2018-01-03  Janne Blomqvist  <jb@gcc.gnu.org>

	PR libgfortran/83649
	* io/unix.c (MAX_CHUNK): New define.
	(raw_read): For reads larger than MAX_CHUNK, loop.
	(raw_write): Write no more than MAX_CHUNK bytes per iteration.
---
 libgfortran/io/unix.c | 50 ++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 8 deletions(-)

diff --git a/libgfortran/io/unix.c b/libgfortran/io/unix.c
index a07a3c9..7a982b3 100644
--- a/libgfortran/io/unix.c
+++ b/libgfortran/io/unix.c
@@ -292,18 +292,49 @@ raw_flush (unix_stream *s  __attribute__ ((unused)))
   return 0;
 }
 
+/* Write/read at most 2 GB - 4k chunks at a time. Linux never reads or
+   writes more than this, and there are reports that macOS fails for
+   larger than 2 GB as well.  */
+#define MAX_CHUNK 2147479552
+
 static ssize_t
 raw_read (unix_stream *s, void *buf, ssize_t nbyte)
 {
   /* For read we can't do I/O in a loop like raw_write does, because
      that will break applications that wait for interactive I/O.  We
-     still can loop around EINTR, though.  */
-  while (true)
+     still can loop around EINTR, though.  This however causes a
+     problem for large reads which must be chunked, see comment above.
+     So assume that if the size is larger than the chunk size, we're
+     reading from a file and not the terminal.  */
+  if (nbyte <= MAX_CHUNK)
     {
-      ssize_t trans = read (s->fd, buf, nbyte);
-      if (trans == -1 && errno == EINTR)
-	continue;
-      return trans;
+      while (true)
+	{
+	  ssize_t trans = read (s->fd, buf, nbyte);
+	  if (trans == -1 && errno == EINTR)
+	    continue;
+	  return trans;
+	}
+    }
+  else
+    {
+      ssize_t bytes_left = nbyte;
+      char *buf_st = buf;
+      while (bytes_left > 0)
+	{
+	  ssize_t to_read = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK;
+	  ssize_t trans = read (s->fd, buf_st, to_read);
+	  if (trans == -1)
+	    {
+	      if (errno == EINTR)
+		continue;
+	      else
+		return trans;
+	    }
+	  buf_st += trans;
+	  bytes_left -= trans;
+	}
+      return nbyte - bytes_left;
     }
 }
 
@@ -317,10 +348,13 @@ raw_write (unix_stream *s, const void *buf, ssize_t nbyte)
   buf_st = (char *) buf;
 
   /* We must write in a loop since some systems don't restart system
-     calls in case of a signal.  */
+     calls in case of a signal.  Also some systems might fail outright
+     if we try to write more than 2 GB in a single syscall, so chunk
+     up large writes.  */
   while (bytes_left > 0)
     {
-      trans = write (s->fd, buf_st, bytes_left);
+      ssize_t to_write = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK;
+      trans = write (s->fd, buf_st, to_write);
       if (trans == -1)
 	{
 	  if (errno == EINTR)
-- 
2.7.4

From-SVN: r256172
This commit is contained in:
Janne Blomqvist 2018-01-03 13:46:38 +02:00
parent cb1df2ac95
commit 8a159915b9
2 changed files with 51 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
libgfortran/ChangeLog
View File

@ -1,3 +1,11 @@
2018-01-03 Janne Blomqvist <jb@gcc.gnu.org>
Backport from trunk
PR libgfortran/83649
* io/unix.c (MAX_CHUNK): New define.
(raw_read): For reads larger than MAX_CHUNK, loop.
(raw_write): Write no more than MAX_CHUNK bytes per iteration.
2017-12-29 Jerry DeLisle <jvdelisle@gcc.gnu.org>
Backport from trunk
@ -51,7 +59,7 @@
in complex_write.
* io/write.c (write_complex): Suppress the leading blanks when
namelist_mode bit is not set to 1.
2017-12-02 Jerry DeLisle <jvdelisle@gcc.gnu.org>
Backport from trunk

50
libgfortran/io/unix.c
View File

@ -292,18 +292,49 @@ raw_flush (unix_stream *s __attribute__ ((unused)))
return 0;
}
/* Write/read at most 2 GB - 4k chunks at a time. Linux never reads or
writes more than this, and there are reports that macOS fails for
larger than 2 GB as well. */
#define MAX_CHUNK 2147479552
static ssize_t
raw_read (unix_stream *s, void *buf, ssize_t nbyte)
{
/* For read we can't do I/O in a loop like raw_write does, because
that will break applications that wait for interactive I/O. We
still can loop around EINTR, though. */
while (true)
still can loop around EINTR, though. This however causes a
problem for large reads which must be chunked, see comment above.
So assume that if the size is larger than the chunk size, we're
reading from a file and not the terminal. */
if (nbyte <= MAX_CHUNK)
{
ssize_t trans = read (s->fd, buf, nbyte);
if (trans == -1 && errno == EINTR)
continue;
return trans;
while (true)
{
ssize_t trans = read (s->fd, buf, nbyte);
if (trans == -1 && errno == EINTR)
continue;
return trans;
}
}
else
{
ssize_t bytes_left = nbyte;
char *buf_st = buf;
while (bytes_left > 0)
{
ssize_t to_read = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK;
ssize_t trans = read (s->fd, buf_st, to_read);
if (trans == -1)
{
if (errno == EINTR)
continue;
else
return trans;
}
buf_st += trans;
bytes_left -= trans;
}
return nbyte - bytes_left;
}
}
@ -317,10 +348,13 @@ raw_write (unix_stream *s, const void *buf, ssize_t nbyte)
buf_st = (char *) buf;
/* We must write in a loop since some systems don't restart system
calls in case of a signal. */
calls in case of a signal. Also some systems might fail outright
if we try to write more than 2 GB in a single syscall, so chunk
up large writes. */
while (bytes_left > 0)
{
trans = write (s->fd, buf_st, bytes_left);
ssize_t to_write = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK;
trans = write (s->fd, buf_st, to_write);
if (trans == -1)
{
if (errno == EINTR)