[PATCH v3 2/2] windowscodecs/tiff: Skip frames marked with SUBFILETYPE(0x1).

[Nikolay Sivov]

From: Nikolay Sivov

This is important for RAW formats that use TIFF container, but mark
directories as 'reduced-size'. Those files should be processed with
appropriate RAW decoder, instead of displaying auxiliary frame with
the TIFF decoder.

Unfortunately libtiff skips old tag OSUBFILETYPE == 0xff,
it's not available via TIFFGetField() and SUBFILETYPE is not synthesized
from it. On Windows both tags have the same effect on frames
being skipped by the decoder.

Signed-off-by: Nikolay Sivov 
---
 dlls/windowscodecs/libtiff.c          | 58 ++++++++++++++++++++++++++-
 dlls/windowscodecs/tests/tiffformat.c |  9 -----
 2 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/dlls/windowscodecs/libtiff.c b/dlls/windowscodecs/libtiff.c
index 60c5ffe3404..2e62e42c7bf 100644
--- a/dlls/windowscodecs/libtiff.c
+++ b/dlls/windowscodecs/libtiff.c
@@ -161,6 +161,7 @@ struct tiff_decoder
     IStream *stream;
     TIFF *tiff;
     DWORD frame_count;
+    unsigned int *frame_map;
     DWORD cached_frame;
     tiff_decode_info cached_decode_info;
     INT cached_tile_x, cached_tile_y;
@@ -576,6 +577,56 @@ static HRESULT tiff_get_decode_info(TIFF *tiff, tiff_decode_info *decode_info)
     return S_OK;
 }

+static bool tiff_decoder_should_skip_directory(TIFF *tiff)
+{
+    unsigned int value;
+
+    if (TIFFGetField(tiff, TIFFTAG_SUBFILETYPE, &value))
+        return value & FILETYPE_REDUCEDIMAGE;
+
+    return false;
+}
+
+static HRESULT tiff_decoder_collect_frames(struct tiff_decoder *decoder)
+{
+    unsigned int dir_count = TIFFNumberOfDirectories(decoder->tiff), frame_count, dir;
+    unsigned int *map;
+
+    if (!dir_count)
+        return S_OK;
+
+    if (!(map = calloc(dir_count, sizeof(*map))))
+        return E_OUTOFMEMORY;
+
+    frame_count = 0;
+    for (dir = 0; dir < dir_count; ++dir)
+    {
+        if (!TIFFSetDirectory(decoder->tiff, dir))
+        {
+            free(map);
+            return E_FAIL;
+        }
+
+        if (tiff_decoder_should_skip_directory(decoder->tiff)) continue;
+
+        map[frame_count++] = dir;
+    }
+
+    if (!frame_count)
+    {
+        free(map);
+        return WINCODEC_ERR_BADIMAGE;
+    }
+
+    /* Back to the first useful directory. */
+    TIFFSetDirectory(decoder->tiff, map[0]);
+
+    decoder->frame_count = frame_count;
+    decoder->frame_map = map;
+
+    return S_OK;
+}
+
 static HRESULT CDECL tiff_decoder_initialize(struct decoder* iface, IStream *stream, struct decoder_stat *st)
 {
     struct tiff_decoder *This = impl_from_decoder(iface);
@@ -585,7 +636,9 @@ static HRESULT CDECL tiff_decoder_initialize(struct decoder* iface, IStream *str
     if (!This->tiff)
         return E_FAIL;

-    This->frame_count = TIFFNumberOfDirectories(This->tiff);
+    if (FAILED(hr = tiff_decoder_collect_frames(This)))
+        goto fail;
+
     This->cached_frame = 0;
     hr = tiff_get_decode_info(This->tiff, &This->cached_decode_info);
     if (FAILED(hr))
@@ -617,7 +670,7 @@ static HRESULT tiff_decoder_select_frame(struct tiff_decoder* This, DWORD frame)

prev_tile_size = This->cached_tile ? This->cached_decode_info.tile_size : 0;

-    res = TIFFSetDirectory(This->tiff, frame);
+    res = TIFFSetDirectory(This->tiff, This->frame_map[frame]);
     if (!res)
         return E_INVALIDARG;

@@ -1143,6 +1196,7 @@ static void CDECL tiff_decoder_destroy(struct decoder* iface)
     struct tiff_decoder *This = impl_from_decoder(iface);
     if (This->tiff) TIFFClose(This->tiff);
     free(This->cached_tile);
+    free(This->frame_map);
     free(This);
 }

diff --git a/dlls/windowscodecs/tests/tiffformat.c b/dlls/windowscodecs/tests/tiffformat.c
index d2348bdc97d..a83cf4ed64e 100644
--- a/dlls/windowscodecs/tests/tiffformat.c
+++ b/dlls/windowscodecs/tests/tiffformat.c
@@ -791,13 +791,7 @@ static void test_tiff_1bpp_palette(void)

hr = IWICBitmapDecoder_GetFrameCount(decoder, &count);
         ok(hr == S_OK, "Unexpected hr %#lx.\n", hr);
-        todo_wine
         ok(count == test_data[i].count, "Unexpected count %u.\n", count);
-        if (count != test_data[i].count)
-        {
-            IWICBitmapDecoder_Release(decoder);
-            continue;
-        }

for (int j = 0; j < count; ++j)
         {
@@ -825,10 +819,7 @@ static void test_tiff_1bpp_palette(void)
     tiff_set_field(&tiff, 2, TIFF_SUBFILETYPE, 0x1);

hr = create_decoder(&tiff, sizeof(tiff), &decoder);
-    todo_wine
     ok(FAILED(hr), "Unexpected hr %#lx.\n", hr);
-    if (SUCCEEDED(hr))
-        IWICBitmapDecoder_Release(decoder);
 }

static void test_QueryCapability(void)