Fixed overflow that broke remote debugger reopening ##debug (#15525)

opt->sz is initialized with r_buf_size at r_bin_open_io using an io
buffer if r_bin_open_io can't open a file buffer. Since the debuggers
returned unsigned values to opt->sz which is signed, opt->sz would
overflow and contain a negative value, causing r_bin_open_buf to fail.
Went ahead and modified CUR_END values for all debuggers even though this
should only affect remote debuggers. ST64_MAX should be enough.

Author: yossizap

libr/io/p/io_gdb.c

@@ -156,7 +156,7 @@ static ut64 __lseek(RIO *io, RIODesc *fd, ut64 offset, int whence) {
 		io->off += offset;
 		break;
 	case R_IO_SEEK_END:
-		io->off = UT64_MAX;
+		io->off = ST64_MAX;
 	}
 	return io->off;
 }

libr/io/p/io_mach.c

@@ -444,15 +444,14 @@ static RIODesc *__open(RIO *io, const char *file, int rw, int mode) {
 
 static ut64 __lseek(RIO *io, RIODesc *fd, ut64 offset, int whence) {
 	switch (whence) {
-	case 0: // abs
+	case R_IO_SEEK_SET:
 		io->off = offset;
 		break;
-	case 1: // cur
-		io->off += (int)offset;
-		break;
-	case 2: // end
-		io->off = UT64_MAX;
+	case R_IO_SEEK_CUR:
+		io->off += offset;
 		break;
+	case R_IO_SEEK_END:
+		io->off = ST64_MAX;
 	}
 	return io->off;
 }

libr/io/p/io_ptrace.c

@@ -251,15 +251,14 @@ static RIODesc *__open(RIO *io, const char *file, int rw, int mode) {
 
 static ut64 __lseek(RIO *io, RIODesc *fd, ut64 offset, int whence) {
 	switch (whence) {
-	case 0: // abs
+	case R_IO_SEEK_SET:
 		io->off = offset;
 		break;
-	case 1: // cur
-		io->off += (int)offset;
-		break;
-	case 2: // end
-		io->off = UT64_MAX;
+	case R_IO_SEEK_CUR:
+		io->off += offset;
 		break;
+	case R_IO_SEEK_END:
+		io->off = ST64_MAX;
 	}
 	return io->off;
 }

libr/io/p/io_w32.c

@@ -36,7 +36,7 @@ static int w32__close(RIODesc *fd) {
 // TODO: handle filesize and so on
 static ut64 w32__lseek(RIO *io, RIODesc *fd, ut64 offset, int whence) {
 	SetFilePointer (RIOW32_HANDLE (fd), offset, 0, !whence?FILE_BEGIN:whence==1?FILE_CURRENT:FILE_END);
-	return (!whence)?offset:whence==1?io->off+offset:UT64_MAX;
+	return (!whence)?offset:whence==1?io->off+offset:ST64_MAX;
 }
 
 static bool w32__plugin_open(RIO *io, const char *pathname, bool many) {

libr/io/p/io_windbg.c

@@ -66,7 +66,7 @@ static ut64 __lseek(RIO *io, RIODesc *fd, ut64 offset, int whence) {
 	case R_IO_SEEK_CUR:
 		return io->off + offset;
 	case R_IO_SEEK_END:
-		return UT64_MAX;
+		return ST64_MAX;
 	default:
 		return offset;
 	}

libr/io/p/io_winedbg.c

@@ -141,13 +141,14 @@ static int __close(RIODesc *fd) {
 
 static ut64 __lseek(RIO *io, RIODesc *fd, ut64 offset, int whence) {
 	switch (whence) {
-	case SEEK_SET:
+	case R_IO_SEEK_SET:
 		io->off = offset;
-		return offset;
-	case SEEK_CUR:
-		return io->off + offset;
-	case SEEK_END:
-		return UT64_MAX;
+		break;
+	case R_IO_SEEK_CUR:
+		io->off += offset;
+		break;
+	case R_IO_SEEK_END:
+		io->off = ST64_MAX;
 	}
 	io->off = offset;
 	return offset;