plug-ins: convert endianness of bitmap/channel_count as soon as possible in file-psp.

Conversion of endianness should happen as soon as possible inside the function
where the values are read otherwise we get inconsistencies with certain
values being converted right away and others not.

plug-ins/common/file-psp.c

@@ -1557,28 +1557,273 @@ upscale_indexed_sub_8 (FILE    *f,
 }
 
 static int
-read_channel_data (FILE        *f,
-                   PSPimage    *ia,
+read_uncompressed_channel (FILE        *f,
+                           guchar     **pixels,
+                           gint         line_width, /* width in bytes of one scanline */
+                           gint         height,
+                           guint        bytespp,    /* bytes per pixel */
+                           guint        bytesps,    /* bytes per sample */
+                           guint        offset,
+                           GError     **error)
+{
+  guchar *buf;
+  gint    y, i;
+
+  if (bytespp == 1)
+    {
+      if (fread (pixels[0], height * line_width, 1, f) < 1)
+        {
+          g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                       _("Error reading channel"));
+          return -1;
+        }
+    }
+  else
+    {
+      buf = g_malloc (line_width);
+      if (bytesps == 1)
+        {
+          for (y = 0; y < height; y++)
+            {
+              guchar *p, *q;
+
+              if (fread (buf, line_width, 1, f) < 1)
+                {
+                  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                               _("Error reading channel"));
+                  return -1;
+                }
+              /* Contrary to what the PSP specification seems to suggest
+                 scanlines are not stored on a 4-byte boundary. */
+              p = buf;
+              q = pixels[y] + offset;
+              for (i = 0; i < line_width; i++)
+                {
+                  *q = *p++;
+                  q += bytespp;
+                }
+            }
+        }
+      else if (bytesps == 2)
+        {
+          for (y = 0; y < height; y++)
+            {
+              guint16 *p, *q;
+              gint     width = line_width / 2;
+
+              if (fread (buf, line_width, 1, f) < 1)
+                {
+                  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                               _("Error reading channel"));
+                  return -1;
+                }
+              /* Contrary to what the PSP specification seems to suggest
+                 scanlines are not stored on a 4-byte boundary. */
+              p = (guint16 *) buf;
+              q = (guint16 *) (pixels[y] + offset);
+              for (i = 0; i < width; i++)
+                {
+                  *q = GUINT16_FROM_LE (*p++);
+                  q += bytespp / 2;
+                }
+            }
+        }
+
+      g_free (buf);
+    }
+  return 0;
+}
+
+static int
+read_rle_channel (FILE        *f,
+                  guchar     **pixels,
+                  gint         line_width, /* width in bytes of one scanline */
+                  gint         height,
+                  guint        bytespp,    /* bytes per pixel */
+                  guint        bytesps,    /* bytes per sample */
+                  guint        offset,
+                  GError     **error)
+{
+  gint    i;
+  guchar *q, *endq;
+  guchar *buf;
+  guchar  runcount, byte;
+
+  q = pixels[0] + offset;
+  endq = q + line_width / bytesps * height * bytespp;
+
+  buf = g_malloc (127);
+  while (q < endq)
+    {
+      if (fread (&runcount, 1, 1, f) < 1)
+        {
+          g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                       _("Error reading channel"));
+          return -1;
+        }
+      if (runcount > 128)
+        {
+          runcount -= 128;
+          if (fread (&byte, 1, 1, f) < 1)
+            {
+              g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                           _("Error reading channel"));
+              return -1;
+            }
+          memset (buf, byte, runcount);
+        }
+      else if (fread (buf, runcount, 1, f) < 1)
+        {
+          g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                       _("Error reading channel"));
+          return -1;
+        }
+
+      /* prevent buffer overflow for bogus data */
+      if (runcount > (endq - q) / bytespp + bytesps)
+        {
+          g_printerr ("Buffer overflow decompressing RLE data.\n");
+          break;
+        }
+
+      if (bytespp == 1)
+        {
+          memmove (q, buf, runcount);
+          q += runcount;
+        }
+      else if (bytesps == 1)
+        {
+          guchar *p = buf;
+
+          for (i = 0; i < runcount; i++)
+            {
+              *q = *p++;
+              q += bytespp;
+            }
+        }
+      else if (bytesps == 2)
+        {
+          guint16 *p = (guint16 *) buf;
+          guint16 *r = (guint16 *) q;
+
+          for (i = 0; i < runcount / 2; i++)
+            {
+              *r = GUINT16_FROM_LE (*p++);
+              r += bytespp / 2;
+            }
+          q = (guchar *) r;
+        }
+    }
+  g_free (buf);
+  return 0;
+}
+
+static int
+read_lz77_channel (FILE        *f,
                    guchar     **pixels,
-                   guint        bytespp,
+                   gint         line_width,     /* width in bytes of one scanline */
+                   gint         height,
+                   guint        bytespp,        /* bytes per pixel */
+                   guint        bytesps,        /* bytes per sample */
                    guint        offset,
-                   GeglBuffer  *buffer,
-                   guint32      compressed_len,
+                   guint32      compressed_len, /* length of compressed data */
                    GError     **error)
 {
-  gint i, y, line_width;
-  gint width = gegl_buffer_get_width (buffer);
-  gint height = gegl_buffer_get_height (buffer);
-  gint npixels = width * height;
-  guchar *buf;
-  guchar *buf2 = NULL;  /* please the compiler */
-  guchar runcount, byte;
+  guint32  npixels, nbytes;
+  gint     i;
+  guchar  *buf, *buf2;
   z_stream zstream;
 
+  nbytes = line_width * height;
+  npixels = nbytes / bytesps;
+
+  buf = g_malloc (compressed_len);
+  if (fread (buf, compressed_len, 1, f) < 1)
+    {
+      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                   _("Error reading channel"));
+      return -1;
+    }
+  zstream.next_in = buf;
+  zstream.avail_in = compressed_len;
+  zstream.zalloc = psp_zalloc;
+  zstream.zfree = psp_zfree;
+  zstream.opaque = f;
+  if (inflateInit (&zstream) != Z_OK)
+    {
+      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                   _("zlib error"));
+      return -1;
+    }
+  if (bytespp == 1)
+    zstream.next_out = pixels[0];
+  else
+    {
+      buf2 = g_malloc (nbytes);
+      zstream.next_out = buf2;
+    }
+  zstream.avail_out = nbytes;
+  if (inflate (&zstream, Z_FINISH) != Z_STREAM_END)
+    {
+      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                   _("zlib error"));
+      inflateEnd (&zstream);
+      return -1;
+    }
+  inflateEnd (&zstream);
+  g_free (buf);
+
+  if (bytespp > 1)
+    {
+      if (bytesps == 1)
+        {
+          guchar *p, *q;
+
+          p = buf2;
+          q = pixels[0] + offset;
+          for (i = 0; i < npixels; i++)
+            {
+              *q = *p++;
+              q += bytespp;
+            }
+          g_free (buf2);
+        }
+      else if (bytesps == 2)
+        {
+          guint16 *p, *q;
+
+          p = (guint16 *) buf2;
+          q = (guint16 *) (pixels[0] + offset);
+          for (i = 0; i < npixels; i++)
+            {
+              *q = GUINT16_FROM_LE (*p++);
+              q += bytespp / 2;
+            }
+          g_free (buf2);
+        }
+    }
+  return 0;
+}
+
+static int
+read_channel_data (FILE      *f,
+                   PSPimage  *ia,
+                   guchar   **pixels,
+                   gint       width,          /* width of layer or mask channel */
+                   gint       height,         /* height of layer or mask channel */
+                   guint      bytespp,
+                   guint      offset,
+                   guint32    compressed_len,
+                   GError   **error)
+{
+  gint decompress_result = 0;
+  gint line_width;
+
   g_assert (ia->bytes_per_sample <= 2);
 
   if (ia->depth < 8)
     {
+      /* Note that bytespp is assumed to be set to 1 for depth < 8. */
       /* Scanlines for 1 and 4 bit only end on a 4-byte boundary. */
       line_width = (((width * ia->depth + 7) / 8) + ia->depth - 1) / 4 * 4;
     }
@@ -1590,187 +1835,35 @@ read_channel_data (FILE        *f,
   switch (ia->compression)
     {
     case PSP_COMP_NONE:
-      if (bytespp == 1)
-        {
-          fread (pixels[0], height * line_width, 1, f);
-        }
-      else
-        {
-          buf = g_malloc (line_width);
-          if (ia->bytes_per_sample == 1)
-            {
-              for (y = 0; y < height; y++)
-                {
-                  guchar *p, *q;
-
-                  fread (buf, width, 1, f);
-                  /* Contrary to what the PSP specification seems to suggest
-                    scanlines are not stored on a 4-byte boundary. */
-                  p = buf;
-                  q = pixels[y] + offset;
-                  for (i = 0; i < width; i++)
-                    {
-                      *q = *p++;
-                      q += bytespp;
-                    }
-                }
-            }
-          else if (ia->bytes_per_sample == 2)
-            {
-              for (y = 0; y < height; y++)
-                {
-                  guint16 *p, *q;
-
-                  fread (buf, width * ia->bytes_per_sample, 1, f);
-                  /* Contrary to what the PSP specification seems to suggest
-                    scanlines are not stored on a 4-byte boundary. */
-                  p = (guint16 *) buf;
-                  q = (guint16 *) (pixels[y] + offset);
-                  for (i = 0; i < width; i++)
-                    {
-                      *q = GUINT16_FROM_LE (*p++);
-                      q += bytespp / 2;
-                    }
-                }
-            }
-
-          g_free (buf);
-        }
+      decompress_result = read_uncompressed_channel (f, pixels, line_width, height,
+                                                     bytespp, ia->bytes_per_sample,
+                                                     offset, error);
       break;
 
     case PSP_COMP_RLE:
-      {
-        guchar *q, *endq;
-
-        q = pixels[0] + offset;
-        if (ia->depth >= 8)
-          endq = q + npixels * bytespp;
-        else
-          endq = q + line_width * height;
-
-        buf = g_malloc (127);
-        while (q < endq)
-          {
-            fread (&runcount, 1, 1, f);
-            if (runcount > 128)
-              {
-                runcount -= 128;
-                fread (&byte, 1, 1, f);
-                memset (buf, byte, runcount);
-              }
-            else
-              fread (buf, runcount, 1, f);
-
-            /* prevent buffer overflow for bogus data */
-            if (runcount > (endq - q) / bytespp + ia->bytes_per_sample - 1)
-              {
-                g_printerr ("Buffer overflow decompressing RLE data.\n");
-                break;
-              }
-
-            if (bytespp == 1)
-              {
-                memmove (q, buf, runcount);
-                q += runcount;
-              }
-            else if (ia->bytes_per_sample == 1)
-              {
-                guchar *p = buf;
-
-                for (i = 0; i < runcount; i++)
-                  {
-                    *q = *p++;
-                    q += bytespp;
-                  }
-              }
-            else if (ia->bytes_per_sample == 2)
-              {
-                guint16 *p = (guint16 *) buf;
-                guint16 *r = (guint16 *) q;
-
-                for (i = 0; i < runcount / 2; i++)
-                  {
-                    *r = GUINT16_FROM_LE (*p++);
-                    r += bytespp / 2;
-                  }
-                q = (guchar *) r;
-              }
-          }
-        g_free (buf);
-      }
+      decompress_result = read_rle_channel (f, pixels, line_width, height,
+                                            bytespp, ia->bytes_per_sample,
+                                            offset, error);
       break;
 
     case PSP_COMP_LZ77:
-      buf = g_malloc (compressed_len);
-      fread (buf, compressed_len, 1, f);
-      zstream.next_in = buf;
-      zstream.avail_in = compressed_len;
-      zstream.zalloc = psp_zalloc;
-      zstream.zfree = psp_zfree;
-      zstream.opaque = f;
-      if (inflateInit (&zstream) != Z_OK)
-        {
-          g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
-                       _("zlib error"));
-          return -1;
-        }
-      if (bytespp == 1)
-        zstream.next_out = pixels[0];
-      else
-        {
-          buf2 = g_malloc (npixels * ia->bytes_per_sample);
-          zstream.next_out = buf2;
-        }
-      zstream.avail_out = npixels * ia->bytes_per_sample;
-      if (inflate (&zstream, Z_FINISH) != Z_STREAM_END)
-        {
-          g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
-                       _("zlib error"));
-          inflateEnd (&zstream);
-          return -1;
-        }
-      inflateEnd (&zstream);
-      g_free (buf);
+      decompress_result = read_lz77_channel (f, pixels, line_width, height,
+                                             bytespp, ia->bytes_per_sample,
+                                             offset, compressed_len, error);
+      break;
 
-      if (bytespp > 1)
-        {
-          if (ia->bytes_per_sample == 1)
-            {
-              guchar *p, *q;
-
-              p = buf2;
-              q = pixels[0] + offset;
-              for (i = 0; i < npixels; i++)
-                {
-                  *q = *p++;
-                  q += bytespp;
-                }
-              g_free (buf2);
-            }
-          else if (ia->bytes_per_sample == 2)
-            {
-              guint16 *p, *q;
-
-              p = (guint16 *) buf2;
-              q = (guint16 *) (pixels[0] + offset);
-              for (i = 0; i < npixels; i++)
-                {
-                  *q = GUINT16_FROM_LE (*p++);
-                  q += bytespp / 2;
-                }
-              g_free (buf2);
-            }
-        }
+    default:
+      decompress_result = -1;
       break;
     }
 
-  if (ia->base_type == GIMP_INDEXED && ia->depth < 8)
+  if (decompress_result == 0 && ia->base_type == GIMP_INDEXED && ia->depth < 8)
     {
       /* We need to convert 1 and 4 bit to 8 bit indexed */
       upscale_indexed_sub_8 (f, width, height, ia->depth, pixels[0]);
     }
 
-  return 0;
+  return decompress_result;
 }
 
 static gboolean
@@ -1835,6 +1928,8 @@ read_raster_layer_info (FILE      *f,
                      _("Error reading layer extension information"));
         return FALSE;
       }
+    *bitmap_count = GUINT16_FROM_LE (*bitmap_count);
+    *channel_count = GUINT16_FROM_LE (*channel_count);
     if (try_fseek (f, layer_extension_start + layer_extension_len, SEEK_SET, error) < 0)
       {
         return FALSE;
@@ -1843,6 +1938,89 @@ read_raster_layer_info (FILE      *f,
   return TRUE;
 }
 
+static const Babl*
+get_mask_format (PSPimage *ia)
+{
+  const Babl *format = NULL;
+
+  switch (ia->depth)
+    {
+    case 48:
+      format = babl_format ("Y u16");
+      break;
+
+    case 8:
+    case 24:
+      format = babl_format ("Y u8");
+      break;
+
+    default:
+      break;
+    }
+
+  return format;
+}
+
+static gboolean
+read_layer_mask (FILE          *f,
+                 GimpLayer     *layer,
+                 GeglRectangle *mask_rect,       /* dimensions of mask data */
+                 GeglRectangle *layer_mask_rect, /* part of mask inside the layer boundaries */
+                 PSPimage      *ia,
+                 gboolean       mask_disabled,
+                 guint32        compressed_len,
+                 GError       **error)
+{
+  GimpLayerMask *mask;
+  GeglBuffer    *buffer;
+  guchar       **pixels, *pixel;
+  gint           i;
+
+  if (! (ia->base_type == GIMP_RGB || ia->base_type == GIMP_GRAY))
+    {
+      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
+                   _("Invalid image base type for layer mask."));
+      return FALSE;
+    }
+
+  if (layer_mask_rect->height > 0 && layer_mask_rect->width > 0)
+    {
+      pixel = g_malloc0 (mask_rect->height * mask_rect->width * ia->bytes_per_sample);
+      pixels = g_new (guchar *, mask_rect->height);
+
+      for (i = 0; i < mask_rect->height; i++)
+        pixels[i] = pixel + mask_rect->width * ia->bytes_per_sample * i;
+
+      /* Mask is a grayscale layer which means bytes per sample is the same
+         as bytes per pixel. */
+      if (read_channel_data (f, ia, pixels, mask_rect->width, mask_rect->height,
+                             ia->bytes_per_sample, 0,
+                             compressed_len, error) == -1)
+        {
+          g_free (pixels);
+          g_free (pixel);
+          return FALSE;
+        }
+
+      mask = gimp_layer_create_mask (layer, GIMP_ADD_MASK_WHITE);
+      gimp_layer_add_mask (layer, mask);
+      buffer = gimp_drawable_get_buffer (GIMP_DRAWABLE (mask));
+      gegl_buffer_set (buffer, mask_rect,
+                       0, get_mask_format (ia),
+                       pixel +
+                         ((layer_mask_rect->y - mask_rect->y) * mask_rect->width +
+                         (layer_mask_rect->x - mask_rect->x)) * ia->bytes_per_sample,
+                       mask_rect->width * ia->bytes_per_sample);
+      gimp_layer_set_apply_mask (layer, ! mask_disabled);
+
+      g_object_unref (buffer);
+      g_free (pixels);
+      g_free (pixel);
+    }
+
+  return TRUE;
+}
+
 static GimpLayer *
 read_layer_block (FILE      *f,
                   GimpImage *image,
@@ -1873,6 +2051,7 @@ read_layer_block (FILE      *f,
   gint width, height, bytespp, offset;
   guchar **pixels, *pixel;
   GeglBuffer *buffer;
+  GeglRectangle mask_data_rect, layer_mask_rect;
 
   block_start = ftell (f);
 
@@ -1986,6 +2165,8 @@ read_layer_block (FILE      *f,
           if (type == PSP_LAYER_FLOATING_SELECTION)
             g_message ("Floating selection restored as normal layer");
           type = keGLTRaster;
+          bitmap_count = GUINT16_FROM_LE (bitmap_count);
+          channel_count = GUINT16_FROM_LE (channel_count);
           can_handle_layer = TRUE;
           if (try_fseek (f, sub_block_start + sub_init_len, SEEK_SET, error) < 0)
             {
@@ -1998,8 +2179,6 @@ read_layer_block (FILE      *f,
       swab_rect (saved_image_rect);
       swab_rect (mask_rect);
       swab_rect (saved_mask_rect);
-      bitmap_count = GUINT16_FROM_LE (bitmap_count);
-      channel_count = GUINT16_FROM_LE (channel_count);
 
       layer_mode = gimp_layer_mode_from_psp_blend_mode (blend_mode);
       if ((int) layer_mode == -1)
@@ -2109,8 +2288,6 @@ read_layer_block (FILE      *f,
                 pixels[i] = pixel + width * bytespp * i;
             }
 
-          buffer = gimp_drawable_get_buffer (GIMP_DRAWABLE (layer));
-
           /* Read the layer channel sub-blocks */
           while (ftell (f) < sub_block_start + sub_total_len)
             {
@@ -2156,8 +2333,22 @@ read_layer_block (FILE      *f,
                 }
               else if (bitmap_type == PSP_DIB_USER_MASK)
                 {
-                  /* FIXME: Add as layer mask */
-                  g_message ("Conversion of layer mask is not supported");
+                  /* Set dimensions of layer mask. */
+                  mask_data_rect.x = mask_rect[0] + saved_mask_rect[0];
+                  mask_data_rect.y = mask_rect[1] + saved_mask_rect[1];
+                  mask_data_rect.width = saved_mask_rect[2] - saved_mask_rect[0];
+                  mask_data_rect.height = saved_mask_rect[3] - saved_mask_rect[1];
+                  /* Crop mask to layer boundaries. */
+                  if (gegl_rectangle_intersect (&layer_mask_rect,
+                                                GEGL_RECTANGLE (0, 0, width, height),
+                                                &mask_data_rect))
+                    {
+                      if (! read_layer_mask (f, layer, &mask_data_rect, &layer_mask_rect, ia,
+                                             mask_disabled, compressed_len, error))
+                        {
+                          g_message ("Reading layer mask failed.");
+                        }
+                    }
                 }
               else
                 {
@@ -2187,8 +2378,8 @@ read_layer_block (FILE      *f,
                           return NULL;
                         }
 
-                      if (read_channel_data (f, ia, pixels, bytespp, offset,
-                                            buffer, compressed_len, error) == -1)
+                      if (read_channel_data (f, ia, pixels, width, height, bytespp,
+                                             offset, compressed_len, error) == -1)
                         {
                           return NULL;
                         }
@@ -2200,9 +2391,9 @@ read_layer_block (FILE      *f,
                 }
             }
 
+          buffer = gimp_drawable_get_buffer (GIMP_DRAWABLE (layer));
           gegl_buffer_set (buffer, GEGL_RECTANGLE (0, 0, width, height), 0,
                           NULL, pixel, GEGL_AUTO_ROWSTRIDE);
-
           g_object_unref (buffer);
 
           g_free (pixels);