Fix cjpeg segfault when Windows BMP width/height<0

rdbmp.c used the ambiguous INT32 datatype, which is sometimes typedef'ed
to long.  Windows bitmap headers use 32-bit signed integers for the
width and height, because height can sometimes be negative (this
indicates a top-down bitmap.)  If biWidth or biHeight was negative and
INT32 was a 64-bit long, then biWidth and biHeight were read as a
positive integer > INT32_MAX, which failed the test in line 385:

    if (biWidth <= 0 || biHeight <= 0)
        ERREXIT(cinfo, JERR_BMP_EMPTY);

This commit refactors rdbmp.c so that it uses the datatypes specified by
Microsoft for the Windows BMP header.

This closes flutter#9 and also provides a better solution for mozilla/mozjpeg#153.

Author: dcommander

ChangeLog.txt

@@ -1,3 +1,12 @@
+1.4.2
+=====
+
+[1] Fixed an issue whereby cjpeg would segfault if a Windows bitmap with a
+negative width or height was used as an input image (Windows bitmaps can have
+a negative height if they are stored in top-down order, but such files are
+rare and not supported by libjpeg-turbo.)
+
+
 1.4.1
 =====
 

rdbmp.c

@@ -6,6 +6,7 @@
  * Modified 2009-2010 by Guido Vollbeding.
  * libjpeg-turbo Modifications:
  * Modified 2011 by Siarhei Siamashka.
+ * Copyright (C) 2015, D. R. Commander.
  * For conditions of distribution and use, see the accompanying README file.
  *
  * This file contains routines to read input images in Microsoft "BMP"
@@ -279,58 +280,58 @@ start_input_bmp (j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
   bmp_source_ptr source = (bmp_source_ptr) sinfo;
   U_CHAR bmpfileheader[14];
   U_CHAR bmpinfoheader[64];
-#define GET_2B(array,offset)  ((unsigned int) UCH(array[offset]) + \
-                               (((unsigned int) UCH(array[offset+1])) << 8))
-#define GET_4B(array,offset)  ((INT32) UCH(array[offset]) + \
-                               (((INT32) UCH(array[offset+1])) << 8) + \
-                               (((INT32) UCH(array[offset+2])) << 16) + \
-                               (((INT32) UCH(array[offset+3])) << 24))
-  INT32 bfOffBits;
-  INT32 headerSize;
-  INT32 biWidth;
-  INT32 biHeight;
-  unsigned int biPlanes;
-  INT32 biCompression;
-  INT32 biXPelsPerMeter,biYPelsPerMeter;
-  INT32 biClrUsed = 0;
+#define GET_2B(array,offset)  ((unsigned short) UCH(array[offset]) + \
+                               (((unsigned short) UCH(array[offset+1])) << 8))
+#define GET_4B(array,offset)  ((unsigned int) UCH(array[offset]) + \
+                               (((unsigned int) UCH(array[offset+1])) << 8) + \
+                               (((unsigned int) UCH(array[offset+2])) << 16) + \
+                               (((unsigned int) UCH(array[offset+3])) << 24))
+  unsigned int bfOffBits;
+  unsigned int headerSize;
+  int biWidth;
+  int biHeight;
+  unsigned short biPlanes;
+  unsigned int biCompression;
+  int biXPelsPerMeter,biYPelsPerMeter;
+  unsigned int biClrUsed = 0;
   int mapentrysize = 0;         /* 0 indicates no colormap */
-  INT32 bPad;
+  int bPad;
   JDIMENSION row_width;
 
   /* Read and verify the bitmap file header */
   if (! ReadOK(source->pub.input_file, bmpfileheader, 14))
     ERREXIT(cinfo, JERR_INPUT_EOF);
   if (GET_2B(bmpfileheader,0) != 0x4D42) /* 'BM' */
     ERREXIT(cinfo, JERR_BMP_NOT);
-  bfOffBits = (INT32) GET_4B(bmpfileheader,10);
+  bfOffBits = GET_4B(bmpfileheader,10);
   /* We ignore the remaining fileheader fields */
 
   /* The infoheader might be 12 bytes (OS/2 1.x), 40 bytes (Windows),
    * or 64 bytes (OS/2 2.x).  Check the first 4 bytes to find out which.
    */
   if (! ReadOK(source->pub.input_file, bmpinfoheader, 4))
     ERREXIT(cinfo, JERR_INPUT_EOF);
-  headerSize = (INT32) GET_4B(bmpinfoheader,0);
+  headerSize = GET_4B(bmpinfoheader,0);
   if (headerSize < 12 || headerSize > 64)
     ERREXIT(cinfo, JERR_BMP_BADHEADER);
   if (! ReadOK(source->pub.input_file, bmpinfoheader+4, headerSize-4))
     ERREXIT(cinfo, JERR_INPUT_EOF);
 
-  switch ((int) headerSize) {
+  switch (headerSize) {
   case 12:
     /* Decode OS/2 1.x header (Microsoft calls this a BITMAPCOREHEADER) */
-    biWidth = (INT32) GET_2B(bmpinfoheader,4);
-    biHeight = (INT32) GET_2B(bmpinfoheader,6);
+    biWidth = (int) GET_2B(bmpinfoheader,4);
+    biHeight = (int) GET_2B(bmpinfoheader,6);
     biPlanes = GET_2B(bmpinfoheader,8);
     source->bits_per_pixel = (int) GET_2B(bmpinfoheader,10);
 
     switch (source->bits_per_pixel) {
     case 8:                     /* colormapped image */
       mapentrysize = 3;         /* OS/2 uses RGBTRIPLE colormap */
-      TRACEMS2(cinfo, 1, JTRC_BMP_OS2_MAPPED, (int) biWidth, (int) biHeight);
+      TRACEMS2(cinfo, 1, JTRC_BMP_OS2_MAPPED, biWidth, biHeight);
       break;
     case 24:                    /* RGB image */
-      TRACEMS2(cinfo, 1, JTRC_BMP_OS2, (int) biWidth, (int) biHeight);
+      TRACEMS2(cinfo, 1, JTRC_BMP_OS2, biWidth, biHeight);
       break;
     default:
       ERREXIT(cinfo, JERR_BMP_BADDEPTH);
@@ -341,26 +342,26 @@ start_input_bmp (j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
   case 64:
     /* Decode Windows 3.x header (Microsoft calls this a BITMAPINFOHEADER) */
     /* or OS/2 2.x header, which has additional fields that we ignore */
-    biWidth = GET_4B(bmpinfoheader,4);
-    biHeight = GET_4B(bmpinfoheader,8);
+    biWidth = (int) GET_4B(bmpinfoheader,4);
+    biHeight = (int) GET_4B(bmpinfoheader,8);
     biPlanes = GET_2B(bmpinfoheader,12);
     source->bits_per_pixel = (int) GET_2B(bmpinfoheader,14);
     biCompression = GET_4B(bmpinfoheader,16);
-    biXPelsPerMeter = GET_4B(bmpinfoheader,24);
-    biYPelsPerMeter = GET_4B(bmpinfoheader,28);
+    biXPelsPerMeter = (int) GET_4B(bmpinfoheader,24);
+    biYPelsPerMeter = (int) GET_4B(bmpinfoheader,28);
     biClrUsed = GET_4B(bmpinfoheader,32);
     /* biSizeImage, biClrImportant fields are ignored */
 
     switch (source->bits_per_pixel) {
     case 8:                     /* colormapped image */
       mapentrysize = 4;         /* Windows uses RGBQUAD colormap */
-      TRACEMS2(cinfo, 1, JTRC_BMP_MAPPED, (int) biWidth, (int) biHeight);
+      TRACEMS2(cinfo, 1, JTRC_BMP_MAPPED, biWidth, biHeight);
       break;
     case 24:                    /* RGB image */
-      TRACEMS2(cinfo, 1, JTRC_BMP, (int) biWidth, (int) biHeight);
+      TRACEMS2(cinfo, 1, JTRC_BMP, biWidth, biHeight);
       break;
     case 32:                    /* RGB image + Alpha channel */
-      TRACEMS2(cinfo, 1, JTRC_BMP, (int) biWidth, (int) biHeight);
+      TRACEMS2(cinfo, 1, JTRC_BMP, biWidth, biHeight);
       break;
     default:
       ERREXIT(cinfo, JERR_BMP_BADDEPTH);