diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a868e794..ce099659 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,9 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v5 + - name: Clone ImageTestSuite + shell: bash + run: git clone --depth 1 https://github.com/treeform/imagetestsuite ../imagetestsuite - uses: treeform/setup-nim-action@v6 - name: Install dependencies shell: bash @@ -22,3 +25,4 @@ jobs: nimby install -g pixie/pixie.nimble - run: nim r tests/tests.nim - run: nim cpp -r tests/tests.nim + - run: nim r tests/imagetestsuite.nim diff --git a/src/pixie/fileformats/gif.nim b/src/pixie/fileformats/gif.nim index d3b330f7..40052e8d 100644 --- a/src/pixie/fileformats/gif.nim +++ b/src/pixie/fileformats/gif.nim @@ -39,17 +39,13 @@ proc decodeGif*(data: string): Gif {.raises: [PixieError].} = hasGlobalColorTable = (globalFlags and 0b10000000) != 0 globalColorTableSize = 2 ^ ((globalFlags and 0b00000111) + 1) bgColorIndex = data.readUint8(11).int - pixelAspectRatio = data.readUint8(12) - if bgColorIndex >= globalColorTableSize: + if hasGlobalColorTable and bgColorIndex >= globalColorTableSize: failInvalid() - if pixelAspectRatio != 0: - raise newException(PixieError, "Unsupported GIF, pixel aspect ratio") - var pos = 13 - if pos + globalColorTableSize * 3 > data.len: + if hasGlobalColorTable and pos + globalColorTableSize * 3 > data.len: failInvalid() var @@ -79,6 +75,8 @@ proc decodeGif*(data: string): Gif {.raises: [PixieError].} = break pos += subBlockSize + if pos > data.len: + failInvalid() var controlExtension: ControlExtension while true: @@ -98,7 +96,7 @@ proc decodeGif*(data: string): Gif {.raises: [PixieError].} = imageTopPos = data.readUint16(pos + 2).int imageWidth = data.readUint16(pos + 4).int imageHeight = data.readUint16(pos + 6).int - imageFlags = data.readUint16(pos + 8) + imageFlags = data.readUint8(pos + 8) hasLocalColorTable = (imageFlags and 0b10000000) != 0 interlaced = (imageFlags and 0b01000000) != 0 localColorTableSize = 2 ^ ((imageFlags and 0b00000111) + 1) @@ -108,7 +106,7 @@ proc decodeGif*(data: string): Gif {.raises: [PixieError].} = if imageWidth > screenWidth or imageHeight > screenHeight: raise newException(PixieError, "Invalid GIF frame dimensions") - if pos + localColorTableSize * 3 > data.len: + if hasLocalColorTable and pos + localColorTableSize * 3 > data.len: failInvalid() var localColorTable: seq[ColorRGBX] @@ -129,7 +127,7 @@ proc decodeGif*(data: string): Gif {.raises: [PixieError].} = let minCodeSize = data.readUint8(pos).int inc pos - if minCodeSize > 11: + if minCodeSize < 2 or minCodeSize > 8: failInvalid() # The image data is contained in a sequence of sub-blocks diff --git a/src/pixie/fileformats/jpeg.nim b/src/pixie/fileformats/jpeg.nim index 0d5cbd01..ebd2b539 100644 --- a/src/pixie/fileformats/jpeg.nim +++ b/src/pixie/fileformats/jpeg.nim @@ -1,5 +1,5 @@ -import chroma, flatty/binny, ../common, ../images, ../internal, - ../simd, std/decls, std/sequtils, std/strutils +import chroma, flatty/binny, ../common, ../images, ../simd, std/decls, + std/sequtils, std/strutils # This JPEG decoder is loosely based on stb_image which is public domain. @@ -38,6 +38,8 @@ const 0.uint32, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767, 65535 ] + maxMarkerResync = 64 + maxRestartResync = 8192 let jpegStartOfImage* = [0xFF.uint8, 0xD8] @@ -69,6 +71,7 @@ type bitsBuffered: int bitBuffer: uint32 foundSOF: bool + foundSOS: bool imageHeight, imageWidth: int progressive: bool quantizationTables: array[4, array[64, uint8]] @@ -176,9 +179,47 @@ proc readStr(state: var DecoderState, n: int): string = proc skipBytes(state: var DecoderState, n: int) = ## Skips a number of bytes. if state.pos + n > state.len: + if state.foundSOS: + state.pos = state.len + return failInvalid() state.pos += n +proc readMarker(state: var DecoderState): uint8 = + ## Reads a JPEG marker, resyncing past junk between marker segments. + var skipped: int + while true: + if state.pos >= state.len: + failInvalid("missing marker") + + let marker = state.readUint8() + if marker != 0xFF: + if state.progressive and not state.foundSOS and marker == 0xDA: + return 0xDA + inc skipped + if skipped > maxMarkerResync: + if state.foundSOS: + return 0xD9 + failInvalid("invalid chunk marker") + continue + + while state.pos < state.len and state.buffer[state.pos] == 0xFF: + inc state.pos + + if state.pos >= state.len: + failInvalid("missing marker id") + + result = state.readUint8() + if result == 0: + inc skipped + if skipped > maxMarkerResync: + if state.foundSOS: + return 0xD9 + failInvalid("invalid chunk marker") + continue + + return + proc skipChunk(state: var DecoderState) = ## Skips current chunk. let len = state.readUint16be() - 2 @@ -186,27 +227,51 @@ proc skipChunk(state: var DecoderState) = proc decodeDRI(state: var DecoderState) = ## Decode Define Restart Interval - let len = state.readUint16be() - 2 - if len != 2: - failInvalid("invalid DRI length") + let len = state.readUint16be().int - 2 + if len < 2: + state.skipBytes(len.int) + state.restartInterval = 0 + return state.restartInterval = state.readUint16be().int + if len > 2: + state.skipBytes(len.int - 2) proc decodeDQT(state: var DecoderState) = ## Decode Define Quantization Table(s) - var len = state.readUint16be() - 2 + var len = state.readUint16be().int - 2 + let chunkEnd = state.pos + len + if chunkEnd > state.len and state.foundSOS: + state.pos = state.len + return while len > 0: let info = state.readUint8() tableId = info and 15 precision = info shr 4 - if precision != 0: - failInvalid("unsupported quantization table precision") if tableId > 3: + if state.foundSOS: + state.pos = min(chunkEnd, state.len) + return failInvalid() - for i in 0 ..< 64: - state.quantizationTables[tableId][deZigZag[i]] = state.readUint8() - len -= 65 + case precision: + of 0: + for i in 0 ..< 64: + state.quantizationTables[tableId][deZigZag[i]] = state.readUint8() + len -= 65 + of 1: + for i in 0 ..< 64: + let value = state.readUint16be().int + state.quantizationTables[tableId][deZigZag[i]] = min(value, 255).uint8 + len -= 129 + else: + if state.foundSOS: + state.pos = min(chunkEnd, state.len) + return + failInvalid("unsupported quantization table precision") if len != 0: + if state.foundSOS: + state.pos = min(chunkEnd, state.len) + return failInvalid("DQT table length did not match") proc buildHuffman(huffman: var Huffman, counts: array[16, uint8]) = @@ -249,7 +314,11 @@ proc buildHuffman(huffman: var Huffman, counts: array[16, uint8]) = proc decodeDHT(state: var DecoderState) = ## Decode Define Huffman Table - var len = state.readUint16be() - 2 + var len = state.readUint16be().int - 2 + let chunkEnd = state.pos + len + if chunkEnd > state.len and state.foundSOS: + state.pos = state.len + return while len > 0: let info = state.readUint8() @@ -257,26 +326,34 @@ proc decodeDHT(state: var DecoderState) = tableCurrent = info shr 4 # DC or AC if tableCurrent > 1 or tableId > 3: + if state.foundSOS: + state.pos = min(chunkEnd, state.len) + return failInvalid() var counts: array[16, uint8] - numSymbols: uint8 + numSymbols: int for i in 0 ..< 16: counts[i] = state.readUint8() - numSymbols += counts[i] + numSymbols += counts[i].int + if numSymbols > 256: + failInvalid() len -= 17 state.huffmanTables[tableCurrent][tableId] = Huffman() state.huffmanTables[tableCurrent][tableId].buildHuffman(counts) - for i in 0.uint8 ..< numSymbols: + for i in 0 ..< numSymbols: state.huffmanTables[tableCurrent][tableId].symbols[i] = state.readUint8() len -= numSymbols if len != 0: + if state.foundSOS: + state.pos = min(chunkEnd, state.len) + return failInvalid() proc decodeSOF0(state: var DecoderState) = @@ -377,12 +454,19 @@ proc decodeSOF1(state: var DecoderState) = proc decodeSOF2(state: var DecoderState) = ## Decode Start of Image (Progressive DCT format) # Same as SOF0 - state.decodeSOF0() state.progressive = true + state.decodeSOF0() proc decodeExif(state: var DecoderState) = ## Decode Exif header var len = state.readUint16be().int - 2 + let chunkEnd = state.pos + len + if chunkEnd > state.len: + failInvalid() + + template skipInvalidExif() = + state.pos = chunkEnd + return let exifHeader = state.readStr(6) @@ -403,13 +487,13 @@ proc decodeExif(state: var DecoderState) = elif tiffHeader == 0x4949: true else: - failInvalid("invalid Tiff header") + skipInvalidExif() len -= 2 # Verify we got the endianess right. if state.readUint16be().maybeSwap(littleEndian) != 0x002A.uint16: - failInvalid("invalid Tiff header endianess") + skipInvalidExif() len -= 2 @@ -419,23 +503,28 @@ proc decodeExif(state: var DecoderState) = len -= 4 if offsetToFirstIFD < 8: - failInvalid("invalid Tiff offset") + skipInvalidExif() + if state.pos + offsetToFirstIFD - 8 > chunkEnd: + skipInvalidExif() state.skipBytes(offsetToFirstIFD - 8) len -= (offsetToFirstIFD - 8) # Read the IFD0 (main image) tags. + if state.pos + 2 > chunkEnd: + skipInvalidExif() let numTags = state.readUint16be().maybeSwap(littleEndian).int len -= 2 for i in 0 ..< numTags: - let - tagNumber = state.readUint16be().maybeSwap(littleEndian) - dataFormat = state.readUint16be().maybeSwap(littleEndian) - numberComponents = state.readUint32be().maybeSwap(littleEndian) - dataOffset = state.readUint32be().maybeSwap(littleEndian).int + if state.pos + 12 > chunkEnd: + skipInvalidExif() + let tagNumber = state.readUint16be().maybeSwap(littleEndian) + discard state.readUint16be().maybeSwap(littleEndian) # Data format + discard state.readUint32be().maybeSwap(littleEndian) # Number of components + let dataOffset = state.readUint32be().maybeSwap(littleEndian).int len -= 12 @@ -447,7 +536,7 @@ proc decodeExif(state: var DecoderState) = discard # Skip all of the data we do not want to read, IFD1, thumbnail, etc. - state.skipBytes(len) # Skip any remaining len + state.pos = chunkEnd # Skip any remaining len proc reset(state: var DecoderState) = ## Rests the decoder state need for restart markers. @@ -464,16 +553,14 @@ proc reset(state: var DecoderState) = proc decodeSOS(state: var DecoderState) = ## Decode Start of Scan - header before the block data. - var len = state.readUint16be() - 2 + var len = state.readUint16be().int - 2 + state.foundSOS = true state.scanComponents = state.readUint8().int - if state.scanComponents > state.components.len: + if state.scanComponents <= 0 or state.scanComponents > state.components.len: failInvalid("extra components") - if cast[uint8](state.scanComponents) notin {1'u8, 3}: - failInvalid("unsupported scan component count") - state.componentOrder.setLen(0) for i in 0 ..< state.scanComponents: @@ -483,9 +570,6 @@ proc decodeSOS(state: var DecoderState) = huffmanAC = info and 15 huffmanDC = info shr 4 - if huffmanAC > 3 or huffmanDC > 3: - failInvalid() - var component: int while component < state.components.len: if state.components[component].id == id: @@ -512,6 +596,11 @@ proc decodeSOS(state: var DecoderState) = failInvalid() if state.successiveApproxHigh > 13 or state.successiveApproxLow > 13: failInvalid() + if state.successiveApproxHigh != 0 and + state.successiveApproxLow != state.successiveApproxHigh - 1: + failInvalid("invalid progressive parameters") + if state.spectralStart != 0 and state.scanComponents != 1: + failInvalid("invalid progressive AC scan component count") else: if state.spectralStart != 0: failInvalid() @@ -519,13 +608,56 @@ proc decodeSOS(state: var DecoderState) = failInvalid() state.spectralEnd = 63 - len -= 4 + 2 * state.scanComponents.uint16 + for component in state.componentOrder: + if state.spectralStart == 0 and state.components[component].huffmanDC > 3: + failInvalid() + if (not state.progressive or state.spectralStart != 0) and + state.components[component].huffmanAC > 3: + failInvalid() + + if state.scanComponents == state.components.len: + for i, component in state.componentOrder: + if component != i: + failInvalid() + + len -= 4 + 2 * state.scanComponents if len != 0: failInvalid() state.reset() +proc isEntropyMarker(marker: uint8): bool {.inline.} = + marker in {0xC0'u8..0xC2'u8, 0xC4, 0xD0..0xD7, 0xD9, 0xDA, 0xDB, + 0xDC, 0xDD, 0xE0..0xEF, 0xFE} + +proc seekEntropyMarker(state: var DecoderState): bool = + ## Finds the next marker after damaged entropy-coded data. + var pos = state.pos + while pos < state.len - 1: + if state.buffer[pos] != 0xFF: + inc pos + continue + + var markerPos = pos + while markerPos < state.len and state.buffer[markerPos] == 0xFF: + inc markerPos + if markerPos >= state.len: + break + + let marker = state.buffer[markerPos] + if marker == 0: + pos = markerPos + 1 + continue + if marker.isEntropyMarker(): + state.pos = pos + state.hitEnd = true + state.bitsBuffered = 0 + state.bitBuffer = 0 + return true + + pos = markerPos + 1 + proc fillBitBuffer(state: var DecoderState) = ## When we are low on bits, we need to call this to populate some more. while state.bitsBuffered <= 24: @@ -533,15 +665,31 @@ proc fillBitBuffer(state: var DecoderState) = if state.hitEnd: 0.uint32 else: - state.readUint8().uint32 + if state.pos >= state.len: + state.hitEnd = true + 0.uint32 + else: + state.readUint8().uint32 + if state.hitEnd: + state.bitBuffer = state.bitBuffer or (b shl (24 - state.bitsBuffered)) + state.bitsBuffered += 8 + continue if b == 0xFF: + if state.pos >= state.len: + state.hitEnd = true + return var c = state.readUint8() while c == 0xFF: + if state.pos >= state.len: + state.hitEnd = true + return c = state.readUint8() if c != 0: - state.pos -= 2 - state.hitEnd = true - return + if c.isEntropyMarker(): + state.pos -= 2 + state.hitEnd = true + return + dec state.pos state.bitBuffer = state.bitBuffer or (b shl (24 - state.bitsBuffered)) state.bitsBuffered += 8 @@ -558,6 +706,10 @@ proc huffmanDecode(state: var DecoderState, tableCurrent, table: int): uint8 = if fast < 255: let size = huffman.sizes[fast].int if size > state.bitsBuffered: + if state.hitEnd: + return 0 + if state.seekEntropyMarker(): + return 0 failInvalid() state.bitBuffer = state.bitBuffer shl size state.bitsBuffered -= size @@ -572,6 +724,10 @@ proc huffmanDecode(state: var DecoderState, tableCurrent, table: int): uint8 = inc i if i == 17 or i > state.bitsBuffered: + if state.hitEnd: + return 0 + if state.seekEntropyMarker(): + return 0 failInvalid() let symbolId = (state.bitBuffer shr (32 - i)).int + huffman.deltas[i] @@ -597,6 +753,8 @@ proc readBits(state: var DecoderState, n: int): int = failInvalid() if state.bitsBuffered < n: state.fillBitBuffer() + if state.bitsBuffered < n and state.hitEnd: + state.bitsBuffered = n let k = lrot(state.bitBuffer, n) result = (k and bitMasks[n]).int state.bitBuffer = k and (not bitMasks[n]) @@ -617,9 +775,9 @@ proc decodeRegularBlock( state: var DecoderState, component: int, data: var array[64, int16] ) = ## Decodes a whole block. - let t = state.huffmanDecode(0, state.components[component].huffmanDC).int + var t = state.huffmanDecode(0, state.components[component].huffmanDC).int if t > 15: - failInvalid("bad huffman code") + t = 0 let diff = if t == 0: @@ -643,7 +801,7 @@ proc decodeRegularBlock( else: i += r.int if i >= 64: - failInvalid() + break let zig = deZigZag[i] data[zig] = cast[int16](state.receiveExtend(s.int)) inc i @@ -656,9 +814,9 @@ proc decodeProgressiveBlock( failInvalid("can't merge dc and ac") if state.successiveApproxHigh == 0: - let t = state.huffmanDecode(0, state.components[component].huffmanDC).int + var t = state.huffmanDecode(0, state.components[component].huffmanDC).int if t > 15: - failInvalid("bad huffman code") + t = 0 let diff = if t > 0: @@ -703,11 +861,11 @@ proc decodeProgressiveContinuationBlock( else: k += r.int if k >= 64: - failInvalid() + break let zig = deZigZag[k] inc k if s >= 15: - failInvalid() + break data[zig] = cast[int16](state.receiveExtend(s.int) * (1 shl shift)) else: @@ -741,11 +899,13 @@ proc decodeProgressiveContinuationBlock( discard else: if s != 1: - failInvalid("bad huffman code") - if state.readBit() != 0: - s = bit + r = 64 + s = 0 else: - s = -bit + if state.readBit() != 0: + s = bit + else: + s = -bit while k <= state.spectralEnd: let zig = deZigZag[k] @@ -893,17 +1053,57 @@ proc decodeBlock(state: var DecoderState, comp, row, column: int) = else: state.decodeRegularBlock(comp, data) +proc resyncRestart(state: var DecoderState): bool = + ## Leniently resync to the next restart marker after damaged entropy data. + let maxPos = min(state.len - 1, state.pos + maxRestartResync) + var pos = state.pos + while pos < maxPos: + if state.buffer[pos] != 0xFF: + inc pos + continue + + var markerPos = pos + while markerPos < state.len and state.buffer[markerPos] == 0xFF: + inc markerPos + if markerPos >= state.len: + break + + let marker = state.buffer[markerPos] + if marker in 0xD0'u8 .. 0xD7'u8: + state.pos = markerPos + 1 + state.reset() + return true + + pos = markerPos + 1 + proc checkRestart(state: var DecoderState) = ## Check if we might have run into a restart marker, then deal with it. dec state.todoBeforeRestart if state.todoBeforeRestart <= 0: if state.pos + 1 > state.len: + if state.progressive and state.hitEnd: + state.todoBeforeRestart = int.high + return failInvalid() # Handle getting a restart marker right at the end. if state.buffer[state.pos] == 0xFF and state.buffer[state.pos+1] == 0xD9: return + if state.progressive and state.hitEnd and state.buffer[state.pos] == 0xFF and + state.buffer[state.pos + 1] notin 0xD0'u8 .. 0xD7'u8: + state.todoBeforeRestart = int.high + return if state.buffer[state.pos] != 0xFF or state.buffer[state.pos + 1] notin 0xD0'u8 .. 0xD7'u8: + if state.resyncRestart(): + return + if state.progressive and state.pos + 16 >= state.len: + state.pos = state.len + state.hitEnd = true + state.todoBeforeRestart = int.high + return + if state.progressive and state.seekEntropyMarker(): + state.todoBeforeRestart = int.high + return failInvalid("did not get expected restart marker") state.pos += 2 state.reset() @@ -1102,10 +1302,11 @@ proc decodeJpeg*(data: string): Image {.raises: [PixieError].} = state.len = data.len while true: - if state.readUint8() != 0xFF: - failInvalid("invalid chunk marker") - - let chunkId = state.readUint8() + if state.pos >= state.len and state.foundSOS: + break + let chunkId = state.readMarker() + if state.foundSOS and not state.progressive and chunkId != 0xD9: + break case chunkId: of 0xC0: # Start Of Frame (Baseline DCT) @@ -1127,10 +1328,15 @@ proc decodeJpeg*(data: string): Image {.raises: [PixieError].} = break of 0xD0 .. 0xD7: # Restart markers - failInvalid("invalid restart marker") + discard of 0xDB: # Define Quantization Table(s) state.decodeDQT() + of 0xDC: + # Define Number of Lines + if state.foundSOS and state.pos + 2 > state.len: + break + state.skipChunk() of 0xDD: # Define Restart Interval state.decodeDRI() @@ -1169,10 +1375,7 @@ proc decodeJpegDimensions*( state.len = len while true: - if state.readUint8() != 0xFF: - failInvalid("invalid chunk marker") - - let chunkId = state.readUint8() + let chunkId = state.readMarker() case chunkId: of 0xD8: # SOI - Start of Image @@ -1189,9 +1392,15 @@ proc decodeJpegDimensions*( of 0xC4: # Define Huffman Table state.decodeDHT() + of 0xD0..0xD7: + # Restart markers + discard of 0xDB: # Define Quantization Table(s) state.skipChunk() + of 0xDC: + # Define Number of Lines + state.skipChunk() of 0xDD: # Define Restart Interval state.skipChunk() diff --git a/src/pixie/fileformats/png.nim b/src/pixie/fileformats/png.nim index b24aed4b..1cc08781 100644 --- a/src/pixie/fileformats/png.nim +++ b/src/pixie/fileformats/png.nim @@ -1,4 +1,4 @@ -import chroma, flatty/binny, math, ../common, ../images, ../internal, +import chroma, flatty/binny, ../common, ../images, ../internal, ../simd, zippy, crunchy # See http://www.libpng.org/pub/png/spec/1.2/PNG-Contents.html @@ -14,9 +14,13 @@ type width, height: int bitDepth, colorType, compressionMethod, filterMethod, interlaceMethod: uint8 + ColorRGBA16* = object + r*, g*, b*, a*: uint16 + Png* = ref object - width*, height*, channels*: int + width*, height*, channels*, bitDepth*: int data*: seq[ColorRGBA] + data16*: seq[ColorRGBA16] template failInvalid() = raise newException(PixieError, "Invalid PNG buffer, unable to load") @@ -189,196 +193,422 @@ proc unfilter( value += paethPredictor(up, left, upLeft).uint8 result[unfilteredStartIx + x] = value else: - discard # Not possible, parseHeader validates + raise newException(PixieError, "Invalid PNG row filter") -proc decodeImageData( - data: ptr UncheckedArray[uint8], - header: PngHeader, - palette: seq[ColorRGB], - transparency: string, - idats: seq[(int, int)] -): seq[ColorRGBA] = - if idats.len == 0: - failInvalid() +proc valuesPerPixel(header: PngHeader): int = + case header.colorType: + of 0: 1 + of 2: 3 + of 3: 1 + of 4: 2 + of 6: 4 + else: 0 # Not possible, decodeHeader validates + +proc scanlineBytes(width: int, header: PngHeader): int = + let bitsPerPixel = header.valuesPerPixel * header.bitDepth.int + (width * bitsPerPixel + 7) div 8 + +proc filterBytesPerPixel(header: PngHeader): int = + max((header.valuesPerPixel * header.bitDepth.int + 7) div 8, 1) + +proc adam7Size(size, start, step: int): int = + if size <= start: + 0 + else: + (size - start + step - 1) div step + +proc packedSample( + unfiltered: openArray[uint8], rowStart, x: int, bitDepth: uint8 +): uint8 = + case bitDepth: + of 1: + (unfiltered[rowStart + x div 8] shr (7 - (x mod 8))) and 1 + of 2: + (unfiltered[rowStart + x div 4] shr (6 - (x mod 4) * 2)) and 3 + of 4: + (unfiltered[rowStart + x div 2] shr (4 - (x mod 2) * 4)) and 15 + of 8: + unfiltered[rowStart + x] + else: + 0 # 16 bit is rejected before image data is decoded. - result.setLen(header.width * header.height) +proc expandSample(value, bitDepth: uint8): uint8 = + case bitDepth: + of 1: value * 255 + of 2: value * 85 + of 4: value * 17 + else: value - let - uncompressed = - if idats.len > 1: - var imageData: string - for (start, len) in idats: - let op = imageData.len - imageData.setLen(imageData.len + len) - copyMem(imageData[op].addr, data[start].addr, len) - try: uncompress(imageData) except ZippyError: failInvalid() - else: - let - (start, len) = idats[0] - p = data[start].unsafeAddr - try: uncompress(p, len) except ZippyError: failInvalid() - valuesPerPixel = - case header.colorType: - of 0: 1 - of 2: 3 - of 3: 1 - of 4: 2 - of 6: 4 - else: 0 # Not possible, parseHeader validates - valuesPerByte = 8 div header.bitDepth.int - rowBytes = ceil((header.width.int * valuesPerPixel) / valuesPerByte).int - totalBytes = rowBytes * header.height.int - - # Uncompressed image data should be the total bytes of pixel data plus - # a filter byte for each row. - if uncompressed.len != totalBytes + header.height.int: - failInvalid() +proc readUint16be(data: openArray[uint8], offset: int): uint16 {.inline.} = + (data[offset].uint16 shl 8) or data[offset + 1].uint16 + +proc readUint16be(data: string, offset: int): uint16 {.inline.} = + (data.readUint8(offset).uint16 shl 8) or data.readUint8(offset + 1).uint16 - let unfiltered = unfilter( - uncompressed.cstring, - uncompressed.len, - header.height, - rowBytes, - max(valuesPerPixel div valuesPerByte, 1) +proc rgba16*(r, g, b, a: uint16): ColorRGBA16 {.inline.} = + ColorRGBA16(r: r, g: g, b: b, a: a) + +proc toUint8(value: uint16): uint8 {.inline.} = + ((value.uint32 * 255 + 32767) div 65535).uint8 + +proc toRgba(color: ColorRGBA16): ColorRGBA {.inline.} = + rgba( + color.r.toUint8, + color.g.toUint8, + color.b.toUint8, + color.a.toUint8 ) - case header.colorType: - of 0: - let special = if transparency.len == 2: transparency[1].int else: -1 - var bytePos, bitPos: int - for y in 0 ..< header.height: - for x in 0 ..< header.width: - var value = unfiltered[bytePos] - case header.bitDepth: - of 1: - value = (value shr (7 - bitPos)) and 1 - value *= 255 - inc bitPos - of 2: - value = (value shr (6 - bitPos)) and 3 - value *= 85 - inc(bitPos, 2) - of 4: - value = (value shr (4 - bitPos)) and 15 - value *= 17 - inc(bitPos, 4) - of 8: - inc bytePos - else: - discard # Not possible, parseHeader validates - - if bitPos == 8: - inc bytePos - bitPos = 0 - - let alpha = if value.int == special: 0 else: 255 - result[x + y * header.width] = rgba(value, value, value, alpha.uint8) - - # If we move to a new row, skip to the next full byte - if bitPos > 0: - inc bytePos - bitPos = 0 - of 2: - var special: ColorRGBA - if transparency.len == 6: # Need to apply transparency check, slower. - special.r = transparency.readUint8(1) - special.g = transparency.readUint8(3) - special.b = transparency.readUint8(5) - special.a = 255 - - # While we can read an extra byte safely, do so. Much faster. - for i in 0 ..< header.height * header.width - 1: - copyMem(result[i].addr, unfiltered[i * 3].unsafeAddr, 4) - result[i].a = 255 - if result[i] == special: - result[i].a = 0 - else: - # While we can read an extra byte safely, do so. Much faster. - for i in 0 ..< header.height * header.width - 1: - copyMem(result[i].addr, unfiltered[i * 3].unsafeAddr, 4) - result[i].a = 255 - - let lastOffset = header.height * header.width - 1 - var rgba = rgba( - unfiltered[lastOffset * 3 + 0].uint8, - unfiltered[lastOffset * 3 + 1].uint8, - unfiltered[lastOffset * 3 + 2].uint8, +proc writePixels( + image: var seq[ColorRGBA], + header: PngHeader, + palette: seq[ColorRGB], + transparency: string, + unfiltered: openArray[uint8], + passWidth, passHeight, startX, startY, stepX, stepY: int +) = + let + rowBytes = scanlineBytes(passWidth, header) + specialGray = if transparency.len == 2: transparency[1].int else: -1 + + var specialRgb: ColorRGBA + if transparency.len == 6: + specialRgb = rgba( + transparency.readUint8(1), + transparency.readUint8(3), + transparency.readUint8(5), 255 ) - if rgba == special: - rgba.a = 0 - result[header.height * header.width - 1] = rgba - of 3: - var bytePos, bitPos: int - for y in 0 ..< header.height: - for x in 0 ..< header.width: - var value = unfiltered[bytePos] - case header.bitDepth: - of 1: - value = (value shr (7 - bitPos)) and 1 - inc bitPos - of 2: - value = (value shr (6 - bitPos)) and 3 - inc(bitPos, 2) - of 4: - value = (value shr (4 - bitPos)) and 15 - inc(bitPos, 4) - of 8: - inc bytePos - else: - discard # Not possible, parseHeader validates - - if bitPos == 8: - inc bytePos - bitPos = 0 + + for passY in 0 ..< passHeight: + let + rowStart = passY * rowBytes + y = startY + passY * stepY + + for passX in 0 ..< passWidth: + let + x = startX + passX * stepX + dst = x + y * header.width + + case header.colorType: + of 0: + let value = expandSample( + packedSample(unfiltered, rowStart, passX, header.bitDepth), + header.bitDepth + ) + let alpha = if value.int == specialGray: 0.uint8 else: 255.uint8 + image[dst] = rgba(value, value, value, alpha) + of 2: + let + src = rowStart + passX * 3 + color = rgba( + unfiltered[src + 0], + unfiltered[src + 1], + unfiltered[src + 2], + 255 + ) + image[dst] = + if transparency.len == 6 and color == specialRgb: + rgba(color.r, color.g, color.b, 0) + else: + color + of 3: + let value = packedSample(unfiltered, rowStart, passX, header.bitDepth) if value.int >= palette.len: failInvalid() let rgb = palette[value] - transparency = + alpha = if transparency.len > value.int: transparency.readUint8(value.int) else: 255 - result[x + y * header.width] = rgba(rgb.r, rgb.g, rgb.b, transparency) + image[dst] = rgba(rgb.r, rgb.g, rgb.b, alpha) + of 4: + let src = rowStart + passX * 2 + image[dst] = rgba( + unfiltered[src], + unfiltered[src], + unfiltered[src], + unfiltered[src + 1] + ) + of 6: + let src = rowStart + passX * 4 + image[dst] = rgba( + unfiltered[src + 0], + unfiltered[src + 1], + unfiltered[src + 2], + unfiltered[src + 3] + ) + else: + discard # Not possible, decodeHeader validates - # If we move to a new row, skip to the next full byte - if bitPos > 0: - inc bytePos - bitPos = 0 - of 4: - for i in 0 ..< header.height * header.width: - let bytePos = i * 2 - result[i] = rgba( - unfiltered[bytePos], - unfiltered[bytePos], - unfiltered[bytePos], - unfiltered[bytePos + 1] +proc writePixels16( + image: var seq[ColorRGBA16], + header: PngHeader, + transparency: string, + unfiltered: openArray[uint8], + passWidth, passHeight, startX, startY, stepX, stepY: int +) = + let + rowBytes = scanlineBytes(passWidth, header) + specialGray = + if transparency.len == 2: transparency.readUint16be(0) else: 0.uint16 + + var specialRgb: ColorRGBA16 + if transparency.len == 6: + specialRgb = rgba16( + transparency.readUint16be(0), + transparency.readUint16be(2), + transparency.readUint16be(4), + uint16.high + ) + + for passY in 0 ..< passHeight: + let + rowStart = passY * rowBytes + y = startY + passY * stepY + + for passX in 0 ..< passWidth: + let + x = startX + passX * stepX + dst = x + y * header.width + + case header.colorType: + of 0: + let value = unfiltered.readUint16be(rowStart + passX * 2) + let alpha = + if transparency.len == 2 and value == specialGray: + 0.uint16 + else: + uint16.high + image[dst] = rgba16(value, value, value, alpha) + of 2: + let src = rowStart + passX * 6 + var color = rgba16( + unfiltered.readUint16be(src + 0), + unfiltered.readUint16be(src + 2), + unfiltered.readUint16be(src + 4), + uint16.high + ) + if transparency.len == 6 and color == specialRgb: + color.a = 0 + image[dst] = color + of 4: + let + src = rowStart + passX * 4 + value = unfiltered.readUint16be(src + 0) + image[dst] = rgba16( + value, + value, + value, + unfiltered.readUint16be(src + 2) + ) + of 6: + let src = rowStart + passX * 8 + image[dst] = rgba16( + unfiltered.readUint16be(src + 0), + unfiltered.readUint16be(src + 2), + unfiltered.readUint16be(src + 4), + unfiltered.readUint16be(src + 6) + ) + else: + discard # 16-bit paletted PNG is not a valid color/bit-depth combo. + +proc uncompressIdats( + data: ptr UncheckedArray[uint8], idats: seq[(int, int)] +): string = + if idats.len > 1: + var imageData: string + for (start, len) in idats: + let op = imageData.len + imageData.setLen(imageData.len + len) + copyMem(imageData[op].addr, data[start].addr, len) + result = try: uncompress(imageData) except ZippyError: failInvalid() + else: + let + (start, len) = idats[0] + p = data[start].unsafeAddr + result = try: uncompress(p, len) except ZippyError: failInvalid() + +proc decodeImageData( + data: ptr UncheckedArray[uint8], + header: PngHeader, + palette: seq[ColorRGB], + transparency: string, + idats: seq[(int, int)] +): seq[ColorRGBA] = + if idats.len == 0: + failInvalid() + + result.setLen(header.width * header.height) + + let uncompressed = uncompressIdats(data, idats) + + if header.interlaceMethod == 0: + let + rowBytes = scanlineBytes(header.width, header) + totalBytes = rowBytes * header.height + + # Uncompressed image data should be the total bytes of pixel data plus + # a filter byte for each row. + if uncompressed.len != totalBytes + header.height: + failInvalid() + + let unfiltered = unfilter( + uncompressed.cstring, + uncompressed.len, + header.height, + rowBytes, + header.filterBytesPerPixel + ) + result.writePixels( + header, palette, transparency, unfiltered, + header.width, header.height, 0, 0, 1, 1 + ) + else: + const + startXs = [0, 4, 0, 2, 0, 1, 0] + startYs = [0, 0, 4, 0, 2, 0, 1] + stepXs = [8, 8, 4, 4, 2, 2, 1] + stepYs = [8, 8, 8, 4, 4, 2, 2] + + var pos: int + for pass in 0 ..< 7: + let + passWidth = adam7Size(header.width, startXs[pass], stepXs[pass]) + passHeight = adam7Size(header.height, startYs[pass], stepYs[pass]) + + if passWidth == 0 or passHeight == 0: + continue + + let + rowBytes = scanlineBytes(passWidth, header) + passLen = (rowBytes + 1) * passHeight + + if pos + passLen > uncompressed.len: + failInvalid() + + let unfiltered = unfilter( + uncompressed[pos].unsafeAddr, + passLen, + passHeight, + rowBytes, + header.filterBytesPerPixel ) - of 6: - copyMem(result[0].addr, unfiltered[0].unsafeAddr, unfiltered.len) + result.writePixels( + header, palette, transparency, unfiltered, + passWidth, passHeight, + startXs[pass], startYs[pass], stepXs[pass], stepYs[pass] + ) + inc(pos, passLen) + + if pos != uncompressed.len: + failInvalid() + +proc decodeImageData16( + data: ptr UncheckedArray[uint8], + header: PngHeader, + transparency: string, + idats: seq[(int, int)] +): seq[ColorRGBA16] = + if idats.len == 0: + failInvalid() + + result.setLen(header.width * header.height) + + let uncompressed = uncompressIdats(data, idats) + + if header.interlaceMethod == 0: + let + rowBytes = scanlineBytes(header.width, header) + totalBytes = rowBytes * header.height + + # Uncompressed image data should be the total bytes of pixel data plus + # a filter byte for each row. + if uncompressed.len != totalBytes + header.height: + failInvalid() + + let unfiltered = unfilter( + uncompressed.cstring, + uncompressed.len, + header.height, + rowBytes, + header.filterBytesPerPixel + ) + result.writePixels16( + header, transparency, unfiltered, + header.width, header.height, 0, 0, 1, 1 + ) else: - discard # Not possible, parseHeader validates + const + startXs = [0, 4, 0, 2, 0, 1, 0] + startYs = [0, 0, 4, 0, 2, 0, 1] + stepXs = [8, 8, 4, 4, 2, 2, 1] + stepYs = [8, 8, 8, 4, 4, 2, 2] + + var pos: int + for pass in 0 ..< 7: + let + passWidth = adam7Size(header.width, startXs[pass], stepXs[pass]) + passHeight = adam7Size(header.height, startYs[pass], stepYs[pass]) + + if passWidth == 0 or passHeight == 0: + continue + + let + rowBytes = scanlineBytes(passWidth, header) + passLen = (rowBytes + 1) * passHeight + + if pos + passLen > uncompressed.len: + failInvalid() + + let unfiltered = unfilter( + uncompressed[pos].unsafeAddr, + passLen, + passHeight, + rowBytes, + header.filterBytesPerPixel + ) + result.writePixels16( + header, transparency, unfiltered, + passWidth, passHeight, + startXs[pass], startYs[pass], stepXs[pass], stepYs[pass] + ) + inc(pos, passLen) + + if pos != uncompressed.len: + failInvalid() proc newImage*(png: Png): Image {.raises: [PixieError].} = ## Creates a new Image from the PNG. result = newImage(png.width, png.height) - copyMem(result.data[0].addr, png.data[0].addr, png.data.len * 4) - result.data.toPremultipliedAlpha() + if png.data.len > 0: + copyMem(result.data[0].addr, png.data[0].addr, png.data.len * 4) + result.data.toPremultipliedAlpha() + else: + for i, color in png.data16: + result.data[i] = color.toRgba.rgbx() proc convertToImage*(png: Png): Image {.raises: [].} = ## Converts a PNG into an Image by moving the data. This is faster but can ## only be done once. type Movable = ref object - width, height, channels: int + width, height, channels, bitDepth: int data: seq[ColorRGBX] + data16: seq[ColorRGBA16] result = Image() result.width = png.width result.height = png.height - result.data = move cast[Movable](png).data - result.data.toPremultipliedAlpha() + if png.data.len > 0: + result.data = move cast[Movable](png).data + result.data.toPremultipliedAlpha() + else: + result.data.setLen(png.data16.len) + for i, color in png.data16: + result.data[i] = color.toRgba.rgbx() proc decodePngDimensions*( data: pointer, len: int @@ -443,12 +673,6 @@ proc decodePng*(data: pointer, len: int): Png {.raises: [PixieError].} = failCRC() inc(pos, 4) # CRC - # Not yet supported: - if header.bitDepth == 16: - raise newException(PixieError, "PNG 16 bit depth not supported yet") - if header.interlaceMethod != 0: - raise newException(PixieError, "Interlaced PNG not supported yet") - while true: if pos + 8 > len: failInvalid() @@ -524,7 +748,11 @@ proc decodePng*(data: pointer, len: int): Png {.raises: [PixieError].} = result.width = header.width result.height = header.height result.channels = 4 - result.data = decodeImageData(data, header, palette, transparency, idats) + result.bitDepth = header.bitDepth.int + if header.bitDepth == 16: + result.data16 = decodeImageData16(data, header, transparency, idats) + else: + result.data = decodeImageData(data, header, palette, transparency, idats) proc decodePng*(data: string): Png {.inline, raises: [PixieError].} = ## Decodes the PNG data. @@ -647,7 +875,15 @@ proc encodePng*( result.addUint32(crc32(result[result.len - 4].addr, 4).swap()) proc encodePng*(png: Png): string {.raises: [PixieError].} = - encodePng(png.width, png.height, 4, png.data[0].addr, png.data.len * 4) + if png.data.len > 0: + encodePng(png.width, png.height, 4, png.data[0].addr, png.data.len * 4) + elif png.data16.len > 0: + var data = newSeq[ColorRGBA](png.data16.len) + for i, color in png.data16: + data[i] = color.toRgba() + encodePng(png.width, png.height, 4, data[0].addr, data.len * 4) + else: + raise newException(PixieError, "PNG has no data") proc encodePng*(image: Image): string {.raises: [PixieError].} = ## Encodes the image data into the PNG file format. diff --git a/src/pixie/fileformats/qoi.nim b/src/pixie/fileformats/qoi.nim index c7d65d99..4fce389b 100644 --- a/src/pixie/fileformats/qoi.nim +++ b/src/pixie/fileformats/qoi.nim @@ -1,4 +1,4 @@ -import chroma, flatty/binny, ../common, ../images, ../internal +import chroma, flatty/binny, math, ../common, ../images, ../internal # See: https://qoiformat.org/qoi-specification.pdf @@ -29,10 +29,40 @@ type proc hash(p: ColorRGBA): int = (p.r.int * 3 + p.g.int * 5 + p.b.int * 7 + p.a.int * 11) mod indexLen +proc srgbToLinear(value: uint8): uint8 {.inline.} = + let c = value.float32 / 255 + let linear = + if c <= 0.04045: + c / 12.92 + else: + pow((c + 0.055) / 1.055, 2.4) + round(linear * 255).uint8 + +proc srgbToLinear(color: var ColorRGBA) {.inline.} = + color.r = color.r.srgbToLinear() + color.g = color.g.srgbToLinear() + color.b = color.b.srgbToLinear() + +proc srgbToLinear(color: var ColorRGBX) {.inline.} = + color.r = color.r.srgbToLinear() + color.g = color.g.srgbToLinear() + color.b = color.b.srgbToLinear() + +proc srgbToLinear(data: var seq[ColorRGBX]) = + for color in data.mitems: + color.srgbToLinear() + +proc linearPixel(qoi: Qoi, px: ColorRGBA): ColorRGBA {.inline.} = + result = px + if qoi.colorspace == sRBG: + result.srgbToLinear() + proc newImage*(qoi: Qoi): Image = ## Creates a new Image from the QOI. result = newImage(qoi.width, qoi.height) copyMem(result.data[0].addr, qoi.data[0].addr, qoi.data.len * 4) + if qoi.colorspace == sRBG: + result.data.srgbToLinear() result.data.toPremultipliedAlpha() proc convertToImage*(qoi: Qoi): Image {.raises: [].} = @@ -47,6 +77,8 @@ proc convertToImage*(qoi: Qoi): Image {.raises: [].} = result.width = qoi.width result.height = qoi.height result.data = move cast[Movable](qoi).data + if qoi.colorspace == sRBG: + result.data.srgbToLinear() result.data.toPremultipliedAlpha() proc decodeQoi*(data: string): Qoi {.raises: [PixieError].} = @@ -166,13 +198,14 @@ proc encodeQoi*(qoi: Qoi): string {.raises: [PixieError].} = result.addUint32(qoi.width.uint32.swap()) result.addUint32(qoi.height.uint32.swap()) result.addUint8(qoi.channels.uint8) - result.addUint8(qoi.colorspace.uint8) + result.addUint8(Linear.uint8) var index: Index run: uint8 pxPrev = rgba(0, 0, 0, 255) - for off, px in qoi.data: + for off, qoiPx in qoi.data: + let px = qoi.linearPixel(qoiPx) if px == pxPrev: inc run if run == 62 or off == qoi.data.high: @@ -231,6 +264,7 @@ proc encodeQoi*(image: Image): string {.raises: [PixieError].} = qoi.width = image.width qoi.height = image.height qoi.channels = 4 + qoi.colorspace = Linear qoi.data.setLen(image.data.len) copyMem(qoi.data[0].addr, image.data[0].addr, image.data.len * 4) diff --git a/src/pixie/fileformats/tiff.nim b/src/pixie/fileformats/tiff.nim index 03ed2452..1852caa7 100644 --- a/src/pixie/fileformats/tiff.nim +++ b/src/pixie/fileformats/tiff.nim @@ -1,29 +1,168 @@ -import chroma, flatty/binny, ../common, ../images, ../internal +import chroma, flatty/binny, ../common, ../images, ../internal, + zippy, zippy/inflate const tiffSignatures* = [ [0x4d.uint8, 0x4d, 0x00, 0x2a], [0x49.uint8, 0x49, 0x2a, 0x00] ] + + ImageWidthTag = 0x0100.uint16 + ImageLengthTag = 0x0101.uint16 + BitsPerSampleTag = 0x0102.uint16 + CompressionTag = 0x0103.uint16 + PhotometricInterpretationTag = 0x0106.uint16 + StripOffsetsTag = 0x0111.uint16 + RowsPerStripTag = 0x0116.uint16 + StripByteCountsTag = 0x0117.uint16 + PredictorTag = 0x013d.uint16 + ColorMapTag = 0x0140.uint16 + TileWidthTag = 0x0142.uint16 + TileLengthTag = 0x0143.uint16 + TileOffsetsTag = 0x0144.uint16 + TileByteCountsTag = 0x0145.uint16 + SampleFormatTag = 0x0153.uint16 + knownTags = [ - 0x0100.uint16, # ImageWidth - 0x0101, # ImageLength - 0x0102, # BitsPerSample - 0x0103, # Compression - 0x0106, # PhotometricInterpretation - 0x0111, # StripOffsets - 0x0116, # RowsPerStrip - 0x0117, # StripByteCounts - 0x0140, # ColorMap + ImageWidthTag, + ImageLengthTag, + BitsPerSampleTag, + CompressionTag, + PhotometricInterpretationTag, + StripOffsetsTag, + RowsPerStripTag, + StripByteCountsTag, + PredictorTag, + ColorMapTag, + TileWidthTag, + TileLengthTag, + TileOffsetsTag, + TileByteCountsTag, + SampleFormatTag ] type + TiffDataFormat* = enum + tiffRgba + tiffGray16 + tiffGrayInt16 + tiffFloat32 + Tiff* = ref object width*, height*: int + dataFormat*: TiffDataFormat data*: seq[ColorRGBA] + dataFloat32*: seq[float32] + dataGray16*: seq[uint16] + dataGrayInt16*: seq[int16] + +template failInvalid(reason = "") = + let msg = + if reason.len > 0: + "Invalid TIFF buffer, unable to load: " & reason + else: + "Invalid TIFF buffer, unable to load" + raise newException(PixieError, msg) + +proc decodeTiffDimensions*(data: string): ImageDimensions = + ## Decodes a TIFF's dimensions from memory without decoding the image data. + if data.len < 8: + failInvalid() + + var + pos: int + isBigEndian: bool + + let signature = cast[array[4, uint8]](data.readUint32(0)) + if signature == tiffSignatures[0]: + isBigEndian = true + elif signature == tiffSignatures[1]: + discard + else: + failInvalid() + + pos = 4 + + let ifdOffset = data.readUint32(pos).maybeSwap(isBigEndian).int + pos = ifdOffset # Move to the first IFD offset. + + if pos + 2 > data.len: + failInvalid() + + let numEntries = data.readUint16(pos).maybeSwap(isBigEndian).int + pos += 2 + + for _ in 0 ..< numEntries: + if pos + 12 > data.len: + failInvalid() + + let + tag = data.readUint16(pos + 0).maybeSwap(isBigEndian) + fieldType = data.readUint16(pos + 2).maybeSwap(isBigEndian) + numValues = data.readUint32(pos + 4).maybeSwap(isBigEndian).int + valueOrOffset = pos + 8 + + pos += 12 + + if tag notin [ImageWidthTag, ImageLengthTag]: + continue + + if numValues != 1: + failInvalid() + + let bytesPerValue = + case fieldType: + of 1: + 1 + of 3: + 2 + of 4: + 4 + else: + raise newException(PixieError, "Unsupported field type " & $fieldType) + + let valueOffset = + if numValues * bytesPerValue <= 4: + valueOrOffset + else: + data.readUint32(valueOrOffset).maybeSwap(isBigEndian).int + + let value = + case fieldType: + of 1: + if valueOffset + 1 > data.len: + failInvalid() + data.readUint8(valueOffset).int + of 3: + if valueOffset + 2 > data.len: + failInvalid() + data.readUint16(valueOffset).maybeSwap(isBigEndian).int + of 4: + if valueOffset + 4 > data.len: + failInvalid() + data.readUint32(valueOffset).maybeSwap(isBigEndian).int + else: + raise newException(PixieError, "Unsupported field type " & $fieldType) + + case tag: + of ImageWidthTag: + result.width = value + of ImageLengthTag: + result.height = value + else: + discard -template failInvalid() = - raise newException(PixieError, "Invalid TIFF buffer, unable to load") + if result.width > 0 and result.height > 0: + return + + failInvalid() + +proc readTiffDimensions*(filePath: string): ImageDimensions = + ## Decodes a TIFF's dimensions from a file without decoding the image data. + try: + decodeTiffDimensions(readFile(filePath)) + except IOError as e: + raise newException(PixieError, e.msg, e) proc decodeTiff*(data: string): Tiff = if data.len < 8: @@ -35,10 +174,14 @@ proc decodeTiff*(data: string): Tiff = pos: int isBigEndian: bool bitsPerSample: seq[int] + sampleFormat = 1 # TIFF defaults to unsigned integer samples. compression: int photometricInterpretation: int stripOffsets, stripByteCounts: seq[int] rowsPerStrip: int + tileOffsets, tileByteCounts: seq[int] + tileWidth, tileLength: int + predictor = 1 colorMap: seq[ColorRGBA] let signature = cast[array[4, uint8]](data.readUint32(0)) @@ -52,7 +195,7 @@ proc decodeTiff*(data: string): Tiff = pos = 4 let ifdOffset = data.readUint32(pos).maybeSwap(isBigEndian).int - pos = ifdOffset # Move to the first IFD offset + pos = ifdOffset # Move to the first IFD offset. if pos + 2 > data.len: failInvalid() @@ -99,7 +242,7 @@ proc decodeTiff*(data: string): Tiff = of 1: if offset + 1 > data.len: failInvalid() - data.readUint8(offset).maybeSwap(isBigEndian).int + data.readUint8(offset).int of 3: if offset + 2 > data.len: failInvalid() @@ -112,39 +255,43 @@ proc decodeTiff*(data: string): Tiff = raise newException(PixieError, "Unsupported field type " & $fieldType) case tag: - of knownTags[0]: + of ImageWidthTag: if numValues != 1: failInvalid() result.width = readValue(valueOffset) - of knownTags[1]: + of ImageLengthTag: if numValues != 1: failInvalid() result.height = readValue(valueOffset) - of knownTags[2]: + of BitsPerSampleTag: for _ in 0 ..< numValues: bitsPerSample.add(readValue(valueOffset)) valueOffset += bytesPerValue - of knownTags[3]: + of CompressionTag: if numValues != 1: failInvalid() compression = readValue(valueOffset) - of knownTags[4]: + of PhotometricInterpretationTag: if numValues != 1: failInvalid() photometricInterpretation = readValue(valueOffset) - of knownTags[5]: + of StripOffsetsTag: for _ in 0 ..< numValues: stripOffsets.add(readValue(valueOffset)) valueOffset += bytesPerValue - of knownTags[6]: + of RowsPerStripTag: if numValues != 1: failInvalid() rowsPerStrip = readValue(valueOffset) - of knownTags[7]: + of StripByteCountsTag: for _ in 0 ..< numValues: stripByteCounts.add(readValue(valueOffset)) valueOffset += bytesPerValue - of knownTags[8]: + of PredictorTag: + if numValues != 1: + failInvalid("unsupported predictor count") + predictor = readValue(valueOffset) + of ColorMapTag: if fieldType != 3: failInvalid() var values: seq[int] @@ -159,83 +306,390 @@ proc decodeTiff*(data: string): Tiff = ((values[i + 2 * colorMap.len].float32 / 65535) * 255).uint8, 255 ) + of TileWidthTag: + if numValues != 1: + failInvalid("unsupported tile width count") + tileWidth = readValue(valueOffset) + of TileLengthTag: + if numValues != 1: + failInvalid("unsupported tile length count") + tileLength = readValue(valueOffset) + of TileOffsetsTag: + for _ in 0 ..< numValues: + tileOffsets.add(readValue(valueOffset)) + valueOffset += bytesPerValue + of TileByteCountsTag: + for _ in 0 ..< numValues: + tileByteCounts.add(readValue(valueOffset)) + valueOffset += bytesPerValue + of SampleFormatTag: + if fieldType != 3: + failInvalid() + sampleFormat = readValue(valueOffset) else: discard if result.width == 0 or result.height == 0: - failInvalid() + failInvalid("missing width/height") if stripOffsets.len != stripByteCounts.len: - failInvalid() + failInvalid("stripOffsets and stripByteCounts length mismatch") + if tileOffsets.len != tileByteCounts.len: + failInvalid("tileOffsets and tileByteCounts length mismatch") + if stripOffsets.len == 0 and tileOffsets.len == 0: + failInvalid("no strip or tile offsets found") if bitsPerSample.len == 0: - failInvalid() + failInvalid("missing bitsPerSample") - for i, bits in bitsPerSample: - if bits notin {8}: + for bits in bitsPerSample: + if bits notin {8, 16, 32}: raise newException( PixieError, "TIFF bits per sample of " & $bits & " not supported yet" ) - # Check the bits per sample are all equal + # Check the bits per sample are all equal. for i in 0 ..< bitsPerSample.len: for j in 0 ..< bitsPerSample.len: if bitsPerSample[i] != bitsPerSample[j]: - failInvalid() + failInvalid("mixed bitsPerSample values not supported") + + let + imageWidth = result.width + imageHeight = result.height + sampleBytes = bitsPerSample[0] div 8 + samplesPerPixel = bitsPerSample.len + bytesPerPixel = sampleBytes * samplesPerPixel + + const maxDecodedTiffBytes = 1024 * 1024 * 1024 + + proc checkedProduct(a, b: int, label: string): int = + if a < 0 or b < 0 or (a != 0 and b > high(int) div a): + failInvalid(label & " overflow") + result = a * b + + let + pixelCount = checkedProduct(imageWidth, imageHeight, "image size") + decodedLen = checkedProduct(pixelCount, bytesPerPixel, "decoded size") + + if decodedLen > maxDecodedTiffBytes: + failInvalid("decoded image too large") var decompressed: string - case compression: - of 1: # No compression - var stripDataLen: int - for byteCount in stripByteCounts: - stripDataLen += byteCount - decompressed.setLen(stripDataLen) + proc expectedRowsForStrip(stripIndex: int): int = + if rowsPerStrip <= 0: + return imageHeight + let rowsRemaining = imageHeight - stripIndex * rowsPerStrip + max(0, min(rowsPerStrip, rowsRemaining)) + + proc inflateData(offset, byteCount: int): string = + if offset + byteCount > data.len: + failInvalid("compressed data out of bounds") + + try: + if compression == 8: + result = uncompress( + cast[pointer](data[offset].unsafeAddr), + byteCount, + dfZlib + ) + else: + inflate( + result, + cast[ptr UncheckedArray[uint8]](data[offset].unsafeAddr), + byteCount, + 0 + ) + except CatchableError as e: + raise newException( + PixieError, + "Invalid TIFF buffer, unable to load: " & e.msg, + e + ) + + proc applyPredictor(buffer: var string, rowWidth, rows: int) = + if predictor == 1: + return + + let bytesPerPixelLocal = sampleBytes * samplesPerPixel + case predictor + of 2: + for row in 0 ..< rows: + let rowStart = row * rowWidth * bytesPerPixelLocal + for x in 1 ..< rowWidth: + let pixelStart = rowStart + x * bytesPerPixelLocal + let prevPixelStart = pixelStart - bytesPerPixelLocal + for b in 0 ..< bytesPerPixelLocal: + let reconstructed = ( + cast[uint8](buffer[pixelStart + b]) + + cast[uint8](buffer[prevPixelStart + b]) + ).uint8 + buffer[pixelStart + b] = cast[char](reconstructed) + of 3: + let rowBytes = rowWidth * bytesPerPixelLocal + var restored = newString(rowBytes) + for row in 0 ..< rows: + let rowStart = row * rowBytes + var cp = rowStart + var count = rowBytes + if samplesPerPixel == 1: + while count > 1: + let reconstructed = ( + cast[uint8](buffer[cp + 1]) + + cast[uint8](buffer[cp]) + ).uint8 + buffer[cp + 1] = cast[char](reconstructed) + inc cp + dec count + else: + while count > samplesPerPixel: + for _ in 0 ..< samplesPerPixel: + let reconstructed = ( + cast[uint8](buffer[cp + samplesPerPixel]) + + cast[uint8](buffer[cp]) + ).uint8 + buffer[cp + samplesPerPixel] = cast[char](reconstructed) + inc cp + dec count, samplesPerPixel + for x in 0 ..< rowWidth: + for plane in 0 ..< bytesPerPixelLocal: + let + sampleIndex = plane div sampleBytes + byteIndex = plane mod sampleBytes + mappedByteIndex = + if isBigEndian: + byteIndex + else: + sampleBytes - 1 - byteIndex + mappedPlane = sampleIndex * sampleBytes + mappedByteIndex + restored[x * bytesPerPixelLocal + plane] = + buffer[rowStart + mappedPlane * rowWidth + x] + if rowBytes > 0: + copyMem(buffer[rowStart].addr, restored[0].addr, rowBytes) + else: + failInvalid("unsupported TIFF predictor " & $predictor) - var at: int + proc readStrips() = + decompressed.setLen(decodedLen) + var dstOffset = 0 for i, offset in stripOffsets: - let byteCount = stripByteCounts[i] + let + byteCount = stripByteCounts[i] + rows = expectedRowsForStrip(i) + expectedLen = imageWidth * rows * bytesPerPixel + if offset + byteCount > data.len: - failInvalid() - copyMem(decompressed[at].addr, data[offset].unsafeAddr, byteCount) - at += byteCount + failInvalid("strip " & $i & " offset+byteCount out of bounds") + + var stripData = + if compression == 1: + data.substr(offset, offset + byteCount - 1) + else: + inflateData(offset, byteCount) + + if stripData.len < expectedLen: + failInvalid( + "strip " & $i & " decompressed too short, expected " & + $expectedLen & " got " & $stripData.len + ) + + applyPredictor(stripData, imageWidth, rows) + + if dstOffset + expectedLen > decompressed.len: + failInvalid("strip " & $i & " dstOffset overflow") - # of 5: # LZW + if expectedLen > 0: + copyMem(decompressed[dstOffset].addr, stripData[0].addr, expectedLen) + dstOffset += expectedLen + if dstOffset != decompressed.len: + failInvalid( + "inflated byte count mismatch, expected " & $decompressed.len & + " got " & $dstOffset + ) + + proc readTiles() = + if tileWidth <= 0 or tileLength <= 0: + failInvalid("missing tile dimensions") + + decompressed.setLen(decodedLen) + let tilesAcross = (imageWidth + tileWidth - 1) div tileWidth + for i, offset in tileOffsets: + let + byteCount = tileByteCounts[i] + tileX = i mod tilesAcross + tileY = i div tilesAcross + copyWidth = min(tileWidth, imageWidth - tileX * tileWidth) + copyHeight = min(tileLength, imageHeight - tileY * tileLength) + expectedLen = tileWidth * tileLength * bytesPerPixel + + if offset + byteCount > data.len: + failInvalid("tile " & $i & " offset+byteCount out of bounds") + + var tileData = + if compression == 1: + data.substr(offset, offset + byteCount - 1) + else: + inflateData(offset, byteCount) + + if tileData.len < expectedLen: + failInvalid( + "tile " & $i & " decompressed too short, expected " & + $expectedLen & " got " & $tileData.len + ) + + applyPredictor(tileData, tileWidth, tileLength) + + for row in 0 ..< copyHeight: + let + srcOffset = row * tileWidth * bytesPerPixel + dstOffset = ( + (tileY * tileLength + row) * imageWidth + tileX * tileWidth + ) * bytesPerPixel + copyMem( + decompressed[dstOffset].addr, + tileData[srcOffset].addr, + copyWidth * bytesPerPixel + ) + + case compression: + of 1: # No compression + if tileOffsets.len > 0: + readTiles() + else: + readStrips() + of 5: # LZW + raise newException(PixieError, "LZW TIFF not supported yet") + of 8, 32946: # Adobe Deflate / Deflate + if tileOffsets.len > 0: + readTiles() + else: + readStrips() else: raise newException( PixieError, "TIFF compression " & $compression & " not supported yet" ) - result.data.setLen(result.width * result.height) + proc readSample8(offset: int): uint8 = + decompressed[offset].uint8 + + proc readSample16(offset: int): uint16 = + decompressed.readUint16(offset).maybeSwap(isBigEndian) + + proc downsample16(value: uint16): uint8 = + (value div 257).uint8 + + result.data.setLen(pixelCount) case photometricInterpretation: + of 1: # BlackIsZero. For bilevel and grayscale images: 0 is imaged as black. + if bitsPerSample == @[8]: + result.dataFormat = tiffRgba + if decompressed.len != result.data.len: + failInvalid("grayscale decompressed length mismatch") + for i in 0 ..< result.data.len: + let gray = readSample8(i) + result.data[i] = rgba(gray, gray, gray, 255) + + elif bitsPerSample == @[16]: + result.data.setLen(0) + if sampleFormat == 1: + result.dataFormat = tiffGray16 + result.dataGray16.setLen(pixelCount) + for i in 0 ..< result.dataGray16.len: + result.dataGray16[i] = readSample16(i * 2) + elif sampleFormat == 2: + result.dataFormat = tiffGrayInt16 + result.dataGrayInt16.setLen(pixelCount) + for i in 0 ..< result.dataGrayInt16.len: + result.dataGrayInt16[i] = cast[int16](readSample16(i * 2)) + else: + raise newException( + PixieError, + "TIFF sample format " & $sampleFormat & " not supported yet" + ) + + elif bitsPerSample == @[32]: + if sampleFormat != 3: + raise newException( + PixieError, + "TIFF sample format " & $sampleFormat & " not supported yet" + ) + result.dataFormat = tiffFloat32 + result.data.setLen(0) + result.dataFloat32.setLen(pixelCount) + for i in 0 ..< result.dataFloat32.len: + let bits = decompressed.readUint32(i * 4).maybeSwap(isBigEndian) + result.dataFloat32[i] = cast[float32](bits) + + else: + raise newException( + PixieError, + "BlackIsZero TIFF only 8, 16 and 32 bits supported" + ) + of 2: # RGB - if bitsPerSample.len == 4: # 32 bit RGBA - raise newException(PixieError, "RGBA TIFF not supported yet") - elif bitsPerSample.len == 3: # 24 bit RGB + result.dataFormat = tiffRgba + if bitsPerSample == @[8, 8, 8]: if decompressed.len div 3 != result.data.len: - failInvalid() + failInvalid("RGB decompressed length mismatch") for i in 0 ..< result.data.len: let decompressedIdx = i * 3 result.data[i] = rgba( - decompressed[decompressedIdx + 0].uint8, - decompressed[decompressedIdx + 1].uint8, - decompressed[decompressedIdx + 2].uint8, + readSample8(decompressedIdx + 0), + readSample8(decompressedIdx + 1), + readSample8(decompressedIdx + 2), 255 ) + elif bitsPerSample == @[8, 8, 8, 8]: + if decompressed.len div 4 != result.data.len: + failInvalid("RGBA decompressed length mismatch") + for i in 0 ..< result.data.len: + let decompressedIdx = i * 4 + result.data[i] = rgba( + readSample8(decompressedIdx + 0), + readSample8(decompressedIdx + 1), + readSample8(decompressedIdx + 2), + readSample8(decompressedIdx + 3) + ) + elif bitsPerSample == @[16, 16, 16]: + if decompressed.len div 6 != result.data.len: + failInvalid("16-bit RGB decompressed length mismatch") + for i in 0 ..< result.data.len: + let decompressedIdx = i * 6 + result.data[i] = rgba( + downsample16(readSample16(decompressedIdx + 0)), + downsample16(readSample16(decompressedIdx + 2)), + downsample16(readSample16(decompressedIdx + 4)), + 255 + ) + elif bitsPerSample == @[16, 16, 16, 16]: + if decompressed.len div 8 != result.data.len: + failInvalid("16-bit RGBA decompressed length mismatch") + for i in 0 ..< result.data.len: + let decompressedIdx = i * 8 + result.data[i] = rgba( + downsample16(readSample16(decompressedIdx + 0)), + downsample16(readSample16(decompressedIdx + 2)), + downsample16(readSample16(decompressedIdx + 4)), + downsample16(readSample16(decompressedIdx + 6)) + ) else: - failInvalid() + failInvalid("RGB TIFF sample layout not supported") of 3: # Color Map + result.dataFormat = tiffRgba + if bitsPerSample != @[8]: + failInvalid("color map TIFF sample layout not supported") if decompressed.len != result.data.len: - failInvalid() + failInvalid("color map decompressed length mismatch") for i in 0 ..< result.data.len: let colorMapIndex = decompressed[i].int if colorMapIndex >= colorMap.len: - failInvalid() + failInvalid("color map index out of range") result.data[i] = colorMap[colorMapIndex] else: @@ -245,20 +699,52 @@ proc decodeTiff*(data: string): Tiff = " not supported yet" ) -proc newImage*(tiff: Tiff): Image = - result = newImage(tiff.width, tiff.height) - copyMem(result.data[0].addr, tiff.data[0].addr, tiff.data.len * 4) - result.data.toPremultipliedAlpha() - -proc convertToImage*(tiff: Tiff): Image {.raises: [].} = - ## Converts a PNG into an Image by moving the data. This is faster but can +proc convertToImage*(tiff: Tiff, min = 0.0f, max = 1.0f): Image {.raises: [].} = + ## Converts a TIFF into an Image by moving the data. This is faster but can ## only be done once. type Movable = ref object - width, height, channels: int + width, height: int + dataFormat: TiffDataFormat data: seq[ColorRGBX] result = Image() result.width = tiff.width result.height = tiff.height - result.data = move cast[Movable](tiff).data + + case tiff.dataFormat + of tiffRgba: + result.data = move cast[Movable](tiff).data + result.data.toPremultipliedAlpha() + of tiffGray16: + result.data.setLen(tiff.dataGray16.len) + for i, gray in tiff.dataGray16: + let value = (gray div 257).uint8 + result.data[i] = rgbx(value, value, value, 255) + of tiffGrayInt16: + result.data.setLen(tiff.dataGrayInt16.len) + for i, gray in tiff.dataGrayInt16: + let value = ((gray.int32 + 32768) div 257).uint8 + result.data[i] = rgbx(value, value, value, 255) + of tiffFloat32: + result.data.setLen(tiff.dataFloat32.len) + for i in 0 ..< tiff.dataFloat32.len: + var gray: float32 + if max > min: + gray = (tiff.dataFloat32[i] - min) / (max - min) + if gray < 0: + gray = 0 + elif gray > 1: + gray = 1 + let value = (gray * 255 + 0.5).uint8 + result.data[i] = rgbx(value, value, value, 255) + +proc newImage*(tiff: Tiff): Image = + if tiff.dataFormat != tiffRgba: + return convertToImage(tiff) + + result = newImage(tiff.width, tiff.height) + if tiff.data.len != result.data.len: + failInvalid("image data length mismatch") + if tiff.data.len > 0: + copyMem(result.data[0].addr, tiff.data[0].addr, tiff.data.len * 4) result.data.toPremultipliedAlpha() diff --git a/tests/imagetestsuite.nim b/tests/imagetestsuite.nim new file mode 100644 index 00000000..9b35e619 --- /dev/null +++ b/tests/imagetestsuite.nim @@ -0,0 +1,571 @@ +import algorithm, os, osproc, strformat, strutils +import pixie/common, pixie/fileformats/gif, pixie/fileformats/jpeg, + pixie/fileformats/png, pixie/fileformats/tiff + +when defined(writeImages): + import write_images + +const imageTestSuitePath = "../imagetestsuite" + +type + Expectation = enum + shouldDecode + shouldReject + shouldNotCrash + + RunStatus = enum + decoded + rejected + failed + + RunResult = object + status: RunStatus + message: string + + TestStats = object + total, passed: int + failures: seq[string] + +const gifExpectations = [ + # Based on the upstream GIFTestSuite notes. Most files are malformed on + # purpose and should be rejected cleanly instead of decoded. + ("0646caeb9b9161c777f117007921a687.gif", shouldReject), + ("243d9798466d64aba0acaa41f980bea6.gif", shouldReject), + ("2b5bc31d84703bfb9f371925f0e3e57d.gif", shouldReject), + ("55abb3cc464305dd554171c3d44cb61f.gif", shouldReject), + ("5f09a896c191db3fa7ea6bdd5ebe9485.gif", shouldReject), + ("6d939393058de0579fca1bbf10ecff25.gif", shouldReject), + ("7092f253998c1b6b869707ad7ae92854.gif", shouldReject), + ("9f8f6046eaf9ffa2d9c5d6db05c5f881.gif", shouldReject), + ("adaf0da1764aafb7039440dbe098569b.gif", shouldReject), + ("adf6f850b13dff73ebb22862c6ab028b.gif", shouldReject), + ("bc7af0616c4ae99144c8600e7b39beea.gif", shouldReject), + ("ce774930ac70449f38a18789c70095b8.gif", shouldReject), + ("d5a0175c07418852152ef33a886a5029.gif", shouldReject), + ("e34116d68f49c7852b362ec72a636df5.gif", shouldReject), + ("e6aa0c45a13dd7fc94f7b5451bd89bf4.gif", shouldDecode), + ("ea754e040929b7f9c157efc88c4d0eaf.gif", shouldReject), + ("ee6d1133f9264dc6467990e53d0bf104.gif", shouldReject), + # Pixie intentionally tolerates this extension-order issue. + ("f617c7af7f36296a37ddb419b828099c.gif", shouldDecode), + ("f88b6907ee086c4c8ac4b8c395748c49.gif", shouldReject), + ("fc3e2b992c559055267e26dc23e484c0.gif", shouldReject) +] + +const jpgExpectations = [ + # Curated against ImageMagick and Pixie's supported JPEG subset. Files Pixie + # should fully decode are shouldDecode, decodable-but-unsupported variants are + # shouldNotCrash, and malformed files that should still be rejected are + # shouldReject. + ("138d3b9e0d9fbf641b8135981e597c3a.jpg", shouldNotCrash), + ("194531363df5b73f59c4c0517422f917.jpg", shouldDecode), + ("1cbb1bb37d62c44f67374cd451643dc4.jpg", shouldNotCrash), + ("2183d39878e734cf79b62428b02fafb5.jpg", shouldReject), + ("21a84b8472f6d18f5bb5c0026e97cfaa.jpg", shouldReject), + ("21ad703b38e2c350215bb92a849486f3.jpg", shouldDecode), + ("255015e07b6f9137b53b0f97d67a8aef.jpg", shouldDecode), + ("28968137f4fc75fbf56f16d7a7a8551a.jpg", shouldReject), + ("28c74d9284d9836017fd519f6932efd8.jpg", shouldDecode), + ("2c9e7a1805f8b47630bbb83d21bf8222.jpg", shouldDecode), + ("316be81dfdeeb942e904feb3a77f4f83.jpg", shouldDecode), + ("32d08f4a5eb10332506ebedbb9bc7257.jpg", shouldDecode), + ("3976a754ef0aca80e84e2c403d714579.jpg", shouldDecode), + ("39f43f280b31152f1d27df3f9d189317.jpg", shouldNotCrash), + ("3ba6af611cc5467cfdbd5566561b8478.jpg", shouldReject), + ("3cc4a7fc6481ea3681138da4643f3d16.jpg", shouldNotCrash), + ("3ea649db8e81a46ca4f92fb3238f78ff.jpg", shouldDecode), + ("3ef05501315073d9d4e1c6b654d99ac0.jpg", shouldNotCrash), + ("4085c929e00c446d3fee18b5b20a27f9.jpg", shouldReject), + ("40bb78b1ac031125a6d8466b374962a8.jpg", shouldDecode), + ("46e5ac4a62d7a445a7c1fb704fafe05c.jpg", shouldReject), + ("46f5d9c1b0fe352353688f736e5617b6.jpg", shouldDecode), + ("4838ece0d3900220d33528ee027289bc.jpg", shouldDecode), + ("5315c35bbcc28d8eee419028ac9f38e0.jpg", shouldDecode), + ("5482a54657765056f1a94116a8dbffe7.jpg", shouldNotCrash), + ("551c2656a4f6f9f5ea7e9945b9081202.jpg", shouldDecode), + ("5633ed9d0eb700d0093bf85d86a95ebf.jpg", shouldReject), + ("56d4a1bb53241f7c5ed6ab531320a542.jpg", shouldDecode), + ("59d3b529c78ac722127c41ba75b3355b.jpg", shouldDecode), + ("5a43fa2cf9c1e47f0331ef71b928ee55.jpg", shouldReject), + ("5baad44ca4702949724234e35c5bb341.jpg", shouldDecode), + ("5bc61724b33e34a6188a817f9f2f8138.jpg", shouldDecode), + ("5c67195f6993c9f8d0d32d4ffe0d8e62.jpg", shouldDecode), + ("5dc71b1d868ef137394d3cc23abea65a.jpg", shouldReject), + ("627c0779eb46b98f751187c5c9f43aa3.jpg", shouldReject), + ("6903d4538fd33c8fd0ded32cb30d618e.jpg", shouldReject), + ("6de166ee2a3a60df9017650e2a808408.jpg", shouldReject), + ("72d091e08c93c9e590360130fa35221b.jpg", shouldDecode), + ("754664a12e36abff7950e796c906ae39.jpg", shouldReject), + ("75e4bd7544a85af6438497980b62fba5.jpg", shouldDecode), + ("786b67badc535fc95a4a76c29a0e0146.jpg", shouldDecode), + ("7997b6b229f25315d33f5c7085e37500.jpg", shouldDecode), + ("79f5fc6bca756e1f067c6fc83e18b32e.jpg", shouldDecode), + ("7acc832f70b2ca62e58a953f3b90fd82.jpg", shouldReject), + ("7dbf474f80e466e9e25ee46b84166420.jpg", shouldDecode), + ("7e7cdf7f4ee50b308531313bbf43e0c3.jpg", shouldReject), + ("8417a305e3b43d5b1bda4ff06a660c54.jpg", shouldReject), + ("8546907dbe574744d7fea6ca9de1de6b.jpg", shouldDecode), + ("865db3dd2d380626f16b6f9dc6d62dba.jpg", shouldDecode), + ("897b8b6d8feb466aa6cad5f512c3fce2.jpg", shouldReject), + ("8a9cc8eeed66aeb423a91c44111d9450.jpg", shouldDecode), + ("8e330afbd99ba01b66570ed62fcdc6ab.jpg", shouldReject), + ("8e5e74dbf9b68a322fbb9512db837329.jpg", shouldDecode), + ("90e46387f562ca8fa106b51dfcda1dc6.jpg", shouldReject), + ("96b3e939852157613fa2e48d58fe35fe.jpg", shouldDecode), + ("9efd60f04cd971daa83d3131e6d6f389.jpg", shouldDecode), + ("a17806f32b45d63eea5230e7893e1f15.jpg", shouldDecode), + ("a54f8c866cbef6e6cda858c85d72dfc8.jpg", shouldReject), + ("a7326ba8f3f4559991126474dd30083d.jpg", shouldNotCrash), + ("acb1fac4e618f636d415f62496e8b70e.jpg", shouldDecode), + ("acce3629083f0e348e94fb58f952d3de.jpg", shouldReject), + ("adcb34b94f4c839bdd29037419a0ee53.jpg", shouldReject), + ("b0b8914cc5f7a6eff409f16d8cc236c5.jpg", shouldReject), + ("b4103df93880fc5677c2a081e4bfc712.jpg", shouldDecode), + ("b5369bcbddca7135a5708c5237ad64e4.jpg", shouldDecode), + ("b55977028a3a574336966b6536640fc9.jpg", shouldDecode), + ("ba60305ac83fe3d8ef01da1d9a0ecc79.jpg", shouldDecode), + ("bd8cf05698aee36b82b4caf58edea442.jpg", shouldReject), + ("c1ca5583e4bfadc73e7fe9418b6e6bf4.jpg", shouldReject), + ("c3018ebe53d0046eecb58858ca869a99.jpg", shouldDecode), + ("c4ced510f44a9bfe85c696c05a7f791d.jpg", shouldReject), + ("c52ffdd6a0346c4d09271f8ccbdfd5a3.jpg", shouldReject), + ("c8bc97335529d069a753c67475b8c82c.jpg", shouldReject), + ("c8c1a5675f82021d92b928a10c597bad.jpg", shouldReject), + ("cc23dd79637b606cf5ba234a037e17ba.jpg", shouldReject), + ("cc4ee796d16c9fe68978166c7cd1ae1b.jpg", shouldReject), + ("ce380515a534e8226209daae00e7b4e8.jpg", shouldReject), + ("d085a42245996e5750a30ccb48791bcf.jpg", shouldReject), + ("d15b71b8cebe35a57cc6e996cc09218b.jpg", shouldDecode), + ("d22db5be7594c17a18a047ca9264ea0a.jpg", shouldDecode), + ("d3b044a94486cae0224c002800ddd642.jpg", shouldReject), + ("de4ae285a275bcfe2ac87c0126742552.jpg", shouldReject), + ("de5884cec093257d239f3b8be3e2f2e5.jpg", shouldDecode), + ("e18bb52107598f65b81b02be2c6c5124.jpg", shouldReject), + ("e6d9eca2c7405e13cfb850b7d0ef7476.jpg", shouldReject), + ("eddea4ef9629be031f750a8ff0b7497c.jpg", shouldReject), + ("eecb78b937a7c5f04aae2f5b0f5b5acc.jpg", shouldDecode), + ("ef1f8a057bb6056674fad92f6b8c0acd.jpg", shouldNotCrash), + ("ef724193653930f52acffa90e6426fd2.jpg", shouldReject), + ("f006e96f3b27fdfaa075322d759ea2e8.jpg", shouldDecode), + ("f012a4321f00f12af6b1eee7580ffb9c.jpg", shouldReject), + ("f1fad47f213bb64c99f714652f30e49e.jpg", shouldReject), + ("f6419b06a39ff09604343848658b1a41.jpg", shouldDecode), + ("f6b4389c3cf0f5997b2e5a4b905aea8d.jpg", shouldDecode), + ("f6d3f522dcb693d9e731d5a0fb4e1393.jpg", shouldDecode), + ("f8e19feecd246156b5d7e79efc455e99.jpg", shouldReject), + ("fd44dc63fa7bdd12ee34fc602231ef02.jpg", shouldDecode), + ("fddcfc778ada60229380c2493fc4c243.jpg", shouldReject) +] + +const pngExpectedFileCount = 254 + +# Based on the upstream PNGTestSuite notes and cross-checked with ImageMagick. +# Files listed in pngShouldDecode are valid PNGs in Pixie's supported subset. +# Files listed in pngShouldNotCrash are malformed files ImageMagick still +# decodes/identifies. Every other PNG in the current suite should reject. +const pngShouldDecode = """ +0839d93f8e77e21acd0ac40a80b14b7b.png +1ebd73c1d3fbc89782f29507364128fc.png +2d641a11233385bb37a524ff010a8531.png +66ac49ef3f48ac9482049e1ab57a53e9.png +affc57dfffa5ec448a0795738d456018.png +b59d7a023a8dcd112da2eb859004199a.png +ba2b2b6e72ca0e4683bb640e2d5572f8.png +c636287a4d7cb1a36362f7f236564cef.png +c-m1-66ac49ef3f48ac9482049e1ab57a53e9.png +c-m1-e0f25ec3373dfdca79ba7bcc3ad366f3.png +c-m3-66ac49ef3f48ac9482049e1ab57a53e9.png +c-m4-6bfb149151f58d124d6fa76eaad75520.png +d2e515cfdabae699301dcf290382474d.png +ebfb1cd42314a557e72d4da75c21fc1c.png +18f9baf3834980f4b80a3e82ad45be48.png +51a4d21670dc8dfa8ffc9e54afd62f5f.png +6c853ed9dacd5716bc54eb59cec30889.png +93e6127b9c4e7a99459c558b81d31bc5.png +ac6343a98f8edabfcc6e536dd75aacb0.png +e59ec0cfb8ab64558099543dc19f8378.png +""" + +const pngShouldNotCrash = """ +# Malformed PNGs that ImageMagick still decodes or identifies. +008b8bb75b8a487dc5aac86c9abb06fb.png +0132cfdbd8ca323574a2072e7ed5014c.png +0301fde58080883e938b604cab9768ea.png +073c98872b81d1004d750f18a4b5f732.png +0b7d50ac449fd59eb3de00647636d0c9.png +0d466db9067b719df0b06ef441bf1ee7.png +138331052d7c6e4acebfaa92af314e12.png +13f665c09e4b03cdbe2fff3015ec8aa7.png +18bd8bf75e7a9b40b961dd501654ce0e.png +1ae14e57b7062597279134ff2eeb39c0.png +1b9a48cf04466108f6f2d225d100edbf.png +1bcc34d49e56a2fba38490db206328b8.png +2a6ff5f8106894b22dad3ce99673481a.png +31e3bc3eb811cff582b5feee2494fed8.png +429104334d1fb6a58e17307883c17608.png +42ec8668adb5dbc6581393f463976510.png +4389427591c18bf36e748172640862c3.png +4c5b82ba0a9c12356007bd71e52185b2.png +4f14b7aab3a41855378c5517342598b9.png +579294d4d8110fc64980dd72a5066780.png +5b689479bd7e527c2385a40437272607.png +5beaadc10dfdbf61124e98fdf8a5c191.png +5e2b64196b9e014e0ed0a27873cafdb3.png +611b294df9cf794eeaa1ffcc620bf6a4.png +6399623892b45aa4901aa6e702c7a62d.png +64221ffc9050c92b8980326acc0e4194.png +71714b783e01aec455b5a4a760326ccc.png +7b9abb94ace0278f943a6df29d0ca652.png +829b05b759b2977bc3eb970ab256d867.png +8711007ea5e351755a80cba913d16a32.png +8905ba870cd5d3327a8310fa437aa076.png +9540743374e1fdb273b6a6ca625eb7a3.png +9bd8a9ed81c5a9190f74496197da7249.png +a1d54c960686558901e320a52a967158.png +a24a39e69554a701412b3ed0c009e7f6.png +b3ac9fdb7239f42c734921dfe790291b.png +bf203e765c98b12f6c2b2c33577c730d.png +c0a76d267196727887d45de4889bec33.png +c1a4baf5d7c68d366d4d4f948f7295be.png +c5c030bf52b9b2d8c45c88988fafff4f.png +c-5e2b64196b9e014e0ed0a27873cafdb3.png +d45b0dbbb808df6486f8a13ea44ea174.png +d92428f3fc9c806b0a4373b54e06785e.png +dd18aac055d531e0e4ff8979458dbaa3.png +e76546768d4a8f2f4c39339345c7614c.png +ed5f2464fcaadd4e0a5e905e3ac41ad5.png +edf5c1b0aa5b01eea5017290a286a173.png +f6266c0e9c2f7db9fab0f84562f63b6c.png +f757de9794666c3d14985210679bc98c.png +fa9f6aa9bcc679d20e171dbf07a628fd.png +m1-66ac49ef3f48ac9482049e1ab57a53e9.png +m1-e0f25ec3373dfdca79ba7bcc3ad366f3.png +m3-66ac49ef3f48ac9482049e1ab57a53e9.png +""" + +const tiffExpectedFileCount = 166 + +# Based on the upstream TIFFTestSuite notes and cross-checked with ImageMagick. +# Files listed in tiffShouldDecode are currently decoded by Pixie. Files listed +# in tiffShouldNotCrash are identified by ImageMagick but still outside Pixie's +# supported TIFF subset. +const tiffShouldDecode = """ +0c84d07e1b22b76f24cccc70d8788e4a.tif +551adc8ce6c3c9cc59040903b0428f47.tif +7324fcaff3aad96f27899da51c1bb5d9.tif +8d8582b004aa2560f5bccffbccf4f3d6.tif +e45931b568d12c9905b1db1775321972.tif +f505d8aaa96b8da1a1c92363b898f97a.tif +m2-108af7a96a2efa82a0cee0f200e6b9a2.tif +""" + +const tiffShouldNotCrash = """ +023c970a2a16794f9e51101f76d3bf4d.tif +034ed0549f9046b9c370ac26550a60da.tif +0ceffbda821c7564352b313bed43f7c7.tif +16f2a7e9adcda96170bc1fa873e275c1.tif +1af8e95246f4cfa4e5e58f67d6428ea3.tif +221209eb0a273029efa18f4c61f6628a.tif +27d40bc5f25d8382b890766accb28cf7.tif +2b27b742e68d313d5ea4abd7847cbff4.tif +356a619433db27fb412ec6fef583eded.tif +362323f81c0160afd677241cd5ce92e9.tif +401d27e0565674c24a017588b8cd61d2.tif +434cb1e9680e3b4eda7f4dd430bcd2bf.tif +48af30b09e42ec73f206ce1ac09a424b.tif +54743a2a36ef90c7ed8bb5da8b6ebaf4.tif +5dd2583cd54384e56a769f04ea05c999.tif +6453732434a8a2358a3c895d962bdce2.tif +84399cc32c29ac0cf33b96a0f654f379.tif +89b5888641d5910e92bf451b5e639ad0.tif +9b286add70871bbbef1601997b429344.tif +9bd49db0707bb5d7ddeca56b1de28ab8.tif +a516905c06cbc05e8adac7b0b7e4f514.tif +b05937c07e0f3ce1bfd2c8c71b0220ec.tif +b1247c37897d354610a07ddfe17eb669.tif +b52a2fceb34f9b31cb417379cf8c02ba.tif +b9bc00e0fb28f2a525c0acfe78251eda.tif +c0d253443d8d4241035fff993ea0581b.tif +c16f4894de3142b98e8f3d8fc6b3d23e.tif +ccd82bb72407d0ca03cac50eb42faf47.tif +ce50e53224a3b62bb821bc294bf6a588.tif +d664a73588796b59191ad8628065f04a.tif +e00804b3169008f1c00bf250edccdae4.tif +ebad222e9402d00e371c92ab5da86e6b.tif +ecf22eaecaf11fc75938973745138868.tif +efb780f70a7d94fa98adfc98d2aa8d50.tif +f15fca5bfdef640840d8ea30570c8cea.tif +f4bcc246f3471102a2dea2ee9153b372.tif +f8179f8f5e566349cf3583a1ff3ea95c.tif +fa0aa927763ee4c7ebf43a5d05dfebc7.tif +m10-42c19f8e79e582bef107f372f18a074b.tif +m11-42c19f8e79e582bef107f372f18a074b.tif +m12-42c19f8e79e582bef107f372f18a074b.tif +m1-62804e47400d6a0fd233c32ea8db4e48.tif +m1-68bc8a1966db7a1da2d3b5946c00d1af.tif +m17-42c19f8e79e582bef107f372f18a074b.tif +m1-76d5d8fd02d58b774f2bae6f7b763e3e.tif +m18-42c19f8e79e582bef107f372f18a074b.tif +m1-84da94dc7e5469f7849b0a7efdff5462.tif +m1-93456679a773921d30efafd08f3ad542.tif +m1-96292a1bd64fec83bb6cdd2480a755b6.tif +m1-b0d36ed02fc2624ac79d3144e8b1bda2.tif +m1-b127f0fb89daedea07abb50b9db2dfd9.tif +m1-d0f86ab189cbe900ec389ca6d7464713.tif +m1-f0d7bcb90496c323c880a9773dfe93ff.tif +m1-f228b1103fd7b629c08c1bbba94708e0.tif +m1-f4830e8d4fa458f78a09fcaaf260569c.tif +m2-b0d36ed02fc2624ac79d3144e8b1bda2.tif +m2-b1ab6f4b81e9b8020a90c8f2c9bcfedb.tif +m2-f0d7bcb90496c323c880a9773dfe93ff.tif +m2-f228b1103fd7b629c08c1bbba94708e0.tif +m2-f4830e8d4fa458f78a09fcaaf260569c.tif +m3-76d5d8fd02d58b774f2bae6f7b763e3e.tif +m3-f0d7bcb90496c323c880a9773dfe93ff.tif +m4-d0f86ab189cbe900ec389ca6d7464713.tif +m5-f0d7bcb90496c323c880a9773dfe93ff.tif +m6-f0d7bcb90496c323c880a9773dfe93ff.tif +m7-42c19f8e79e582bef107f372f18a074b.tif +m7-f0d7bcb90496c323c880a9773dfe93ff.tif +m8-42c19f8e79e582bef107f372f18a074b.tif +m8-76c43508fc007bcf5902b6a28e8055a5.tif +m8-f0d7bcb90496c323c880a9773dfe93ff.tif +m9-76c43508fc007bcf5902b6a28e8055a5.tif +m9-f0d7bcb90496c323c880a9773dfe93ff.tif +""" + +proc hasManifestFile(manifest, fileName: string): bool = + for line in manifest.splitLines: + let name = line.strip() + if name.len == 0 or name.startsWith("#"): + continue + if name == fileName: + return true + +proc expectedFor(kind, path: string): Expectation = + let fileName = path.splitPath.tail + + if kind == "gif": + for (name, expectation) in gifExpectations: + if fileName == name: + return expectation + raise newException(ValueError, "Missing GIF expectation for " & fileName) + + if kind == "jpg": + for (name, expectation) in jpgExpectations: + if fileName == name: + return expectation + raise newException(ValueError, "Missing JPG expectation for " & fileName) + + if kind == "png": + if pngShouldDecode.hasManifestFile(fileName): + return shouldDecode + if pngShouldNotCrash.hasManifestFile(fileName): + return shouldNotCrash + return shouldReject + + if kind == "tif": + if tiffShouldDecode.hasManifestFile(fileName): + return shouldDecode + if tiffShouldNotCrash.hasManifestFile(fileName): + return shouldNotCrash + return shouldReject + + shouldDecode + +proc sortedFiles(dir, pattern: string): seq[string] = + for path in walkFiles(dir / pattern): + result.add(path) + result.sort() + +proc checkDimensions( + stats: var TestStats, + path: string, + width, height: int, + dimensions: ImageDimensions +) = + if width != dimensions.width or height != dimensions.height: + stats.failures.add(&"{path}: decoded {width}x{height}, dimensions " & + &"{dimensions.width}x{dimensions.height}") + +template writeImageTestSuiteImage(kind, path: string, image: untyped) = + when defined(writeImages): + let dirName = + case kind + of "jpg": + "jpeg" + of "tif": + "tiff" + else: + kind + image.writeOutputImage(imageTestSuiteOutputDir / dirName, path) + +proc checkGif(stats: var TestStats, path: string) = + let data = readFile(path) + let + gif = decodeGif(data) + image = newImage(gif) + dimensions = decodeGifDimensions(data) + stats.checkDimensions(path, image.width, image.height, dimensions) + writeImageTestSuiteImage("gif", path, image) + +proc checkJpeg(stats: var TestStats, path: string) = + let data = readFile(path) + let + image = decodeJpeg(data) + dimensions = decodeJpegDimensions(data) + stats.checkDimensions(path, image.width, image.height, dimensions) + writeImageTestSuiteImage("jpg", path, image) + +proc checkPng(stats: var TestStats, path: string) = + let data = readFile(path) + let + dimensions = decodePngDimensions(data) + image = decodePng(data).convertToImage() + stats.checkDimensions(path, image.width, image.height, dimensions) + writeImageTestSuiteImage("png", path, image) + +proc checkTiff(stats: var TestStats, path: string) = + let data = readFile(path) + let + dimensions = decodeTiffDimensions(data) + image = decodeTiff(data).convertToImage() + stats.checkDimensions(path, image.width, image.height, dimensions) + writeImageTestSuiteImage("tif", path, image) + +proc checkOne(kind, path: string) = + var stats: TestStats + case kind + of "gif": + stats.checkGif(path) + of "jpg": + stats.checkJpeg(path) + of "png": + stats.checkPng(path) + of "tif": + stats.checkTiff(path) + else: + raise newException(ValueError, "Unknown image test kind " & kind) + + if stats.failures.len > 0: + raise newException(ValueError, stats.failures.join("; ")) + +proc lastNonEmptyLine(text: string): string = + for line in text.splitLines: + let line = line.strip() + if line.len > 0: + result = line + +proc runOne(kind, path: string): RunResult = + let + cmd = quoteShell(getAppFilename()) & " " & quoteShell(kind) & " " & + quoteShell(path) + (output, exitCode) = execCmdEx(cmd) + + let message = output.lastNonEmptyLine() + if exitCode == 0: + result.status = decoded + elif message.startsWith("PixieError:"): + result.status = rejected + result.message = message + else: + result.status = failed + if message.len > 0: + result.message = message + else: + result.message = "exit code " & $exitCode + +proc failedExpectation( + path: string, + expectation: Expectation, + runResult: RunResult +): string = + case expectation + of shouldDecode: + case runResult.status + of decoded: + discard + of rejected: + result = &"{path}: expected decode, rejected with {runResult.message}" + of failed: + result = &"{path}: expected decode, failed with {runResult.message}" + + of shouldReject: + case runResult.status + of decoded: + result = &"{path}: decoded, expected PixieError" + of rejected: + discard + of failed: + result = &"{path}: expected PixieError, failed with {runResult.message}" + + of shouldNotCrash: + if runResult.status == failed: + result = &"{path}: expected decode or PixieError, failed with " & + runResult.message + +proc checkFiles( + stats: var TestStats, + dir, pattern, label: string, + kind: string +) = + let files = sortedFiles(imageTestSuitePath / dir, pattern) + if kind == "png" and files.len != pngExpectedFileCount: + raise newException( + ValueError, + &"PNG expectation manifest covers {pngExpectedFileCount} files, " & + &"found {files.len}" + ) + if kind == "tif" and files.len != tiffExpectedFileCount: + raise newException( + ValueError, + &"TIFF expectation manifest covers {tiffExpectedFileCount} files, " & + &"found {files.len}" + ) + + let passedBefore = stats.passed + let failuresBefore = stats.failures.len + echo &"{label}: {files.len} files" + + for path in files: + inc stats.total + let + expectation = expectedFor(kind, path) + runResult = runOne(kind, path) + failure = failedExpectation(path, expectation, runResult) + if failure.len == 0: + inc stats.passed + else: + stats.failures.add(failure) + + echo &"{label}: {stats.passed - passedBefore}/{files.len} passed, " & + &"{stats.failures.len - failuresBefore} failed" + +if paramCount() == 2: + try: + checkOne(paramStr(1), paramStr(2)) + except Exception as exc: + echo &"{exc.name}: {exc.msg}" + quit(1) + quit(0) + +if not dirExists(imageTestSuitePath): + echo &"Skipping imagetestsuite: {imageTestSuitePath} was not found" +else: + when defined(writeImages): + resetOutputDir(imageTestSuiteOutputDir) + echo &"Writing decoded ImageTestSuite images to {imageTestSuiteOutputDir}" + + var stats: TestStats + stats.checkFiles("gif", "*.gif", "GIF", "gif") + stats.checkFiles("jpg", "*.jpg", "JPG", "jpg") + stats.checkFiles("png", "*.png", "PNG", "png") + stats.checkFiles("tif", "*.tif", "TIF", "tif") + + echo &"imagetestsuite: {stats.passed}/{stats.total} passed" + + if stats.failures.len > 0: + echo &"Failures ({stats.failures.len}):" + for failure in stats.failures: + echo " " & failure + quit(1) diff --git a/tests/pngsuite.nim b/tests/pngsuite.nim index 5aa40192..16165de8 100644 --- a/tests/pngsuite.nim +++ b/tests/pngsuite.nim @@ -7,89 +7,89 @@ const "basn0g02", # 2 bit (4 level) grayscale "basn0g04", # 4 bit (16 level) grayscale "basn0g08", # 8 bit (256 level) grayscale - # "basn0g16", # 16 bit (64k level) grayscale + "basn0g16", # 16 bit (64k level) grayscale "basn2c08", # 3x8 bits rgb color - # "basn2c16", # 3x16 bits rgb color + "basn2c16", # 3x16 bits rgb color "basn3p01", # 1 bit (2 color) paletted "basn3p02", # 2 bit (4 color) paletted "basn3p04", # 4 bit (16 color) paletted "basn3p08", # 8 bit (256 color) paletted "basn4a08", # 8 bit grayscale + 8 bit alpha-channel - # "basn4a16", # 16 bit grayscale + 16 bit alpha-channel + "basn4a16", # 16 bit grayscale + 16 bit alpha-channel "basn6a08", # 3x8 bits rgb color + 8 bit alpha-channel - # "basn6a16", # 3x16 bits rgb color + 16 bit alpha-channel + "basn6a16", # 3x16 bits rgb color + 16 bit alpha-channel # Interlaced - # "basi0g01", # black & white - # "basi0g02", # 2 bit (4 level) grayscale - # "basi0g04", # 4 bit (16 level) grayscale - # "basi0g08", # 8 bit (256 level) grayscale - # "basi0g16", # 16 bit (64k level) grayscale - # "basi2c08", # 3x8 bits rgb color - # "basi2c16", # 3x16 bits rgb color - # "basi3p01", # 1 bit (2 color) paletted - # "basi3p02", # 2 bit (4 color) paletted - # "basi3p04", # 4 bit (16 color) paletted - # "basi3p08", # 8 bit (256 color) paletted - # "basi4a08", # 8 bit grayscale + 8 bit alpha-channel - # "basi4a16", # 16 bit grayscale + 16 bit alpha-channel - # "basi6a08", # 3x8 bits rgb color + 8 bit alpha-channel - # "basi6a16", # 3x16 bits rgb color + 16 bit alpha-channel + "basi0g01", # black & white + "basi0g02", # 2 bit (4 level) grayscale + "basi0g04", # 4 bit (16 level) grayscale + "basi0g08", # 8 bit (256 level) grayscale + "basi0g16", # 16 bit (64k level) grayscale + "basi2c08", # 3x8 bits rgb color + "basi2c16", # 3x16 bits rgb color + "basi3p01", # 1 bit (2 color) paletted + "basi3p02", # 2 bit (4 color) paletted + "basi3p04", # 4 bit (16 color) paletted + "basi3p08", # 8 bit (256 color) paletted + "basi4a08", # 8 bit grayscale + 8 bit alpha-channel + "basi4a16", # 16 bit grayscale + 16 bit alpha-channel + "basi6a08", # 3x8 bits rgb color + 8 bit alpha-channel + "basi6a16", # 3x16 bits rgb color + 16 bit alpha-channel # Odd sizes - # "s01i3p01", # 1x1 paletted file, interlaced + "s01i3p01", # 1x1 paletted file, interlaced "s01n3p01", # 1x1 paletted file, no interlacing - # "s02i3p01", # 2x2 paletted file, interlaced + "s02i3p01", # 2x2 paletted file, interlaced "s02n3p01", # 2x2 paletted file, no interlacing - # "s03i3p01", # 3x3 paletted file, interlaced + "s03i3p01", # 3x3 paletted file, interlaced "s03n3p01", # 3x3 paletted file, no interlacing - # "s04i3p01", # 4x4 paletted file, interlaced + "s04i3p01", # 4x4 paletted file, interlaced "s04n3p01", # 4x4 paletted file, no interlacing - # "s05i3p02", # 5x5 paletted file, interlaced + "s05i3p02", # 5x5 paletted file, interlaced "s05n3p02", # 5x5 paletted file, no interlacing - # "s06i3p02", # 6x6 paletted file, interlaced + "s06i3p02", # 6x6 paletted file, interlaced "s06n3p02", # 6x6 paletted file, no interlacing - # "s07i3p02", # 7x7 paletted file, interlaced + "s07i3p02", # 7x7 paletted file, interlaced "s07n3p02", # 7x7 paletted file, no interlacing - # "s08i3p02", # 8x8 paletted file, interlaced + "s08i3p02", # 8x8 paletted file, interlaced "s08n3p02", # 8x8 paletted file, no interlacing - # "s09i3p02", # 9x9 paletted file, interlaced + "s09i3p02", # 9x9 paletted file, interlaced "s09n3p02", # 9x9 paletted file, no interlacing - # "s32i3p04", # 32x32 paletted file, interlaced + "s32i3p04", # 32x32 paletted file, interlaced "s32n3p04", # 32x32 paletted file, no interlacing - # "s33i3p04", # 33x33 paletted file, interlaced + "s33i3p04", # 33x33 paletted file, interlaced "s33n3p04", # 33x33 paletted file, no interlacing - # "s34i3p04", # 34x34 paletted file, interlaced + "s34i3p04", # 34x34 paletted file, interlaced "s34n3p04", # 34x34 paletted file, no interlacing - # "s35i3p04", # 35x35 paletted file, interlaced + "s35i3p04", # 35x35 paletted file, interlaced "s35n3p04", # 35x35 paletted file, no interlacing - # "s36i3p04", # 36x36 paletted file, interlaced + "s36i3p04", # 36x36 paletted file, interlaced "s36n3p04", # 36x36 paletted file, no interlacing - # "s37i3p04", # 37x37 paletted file, interlaced + "s37i3p04", # 37x37 paletted file, interlaced "s37n3p04", # 37x37 paletted file, no interlacing - # "s38i3p04", # 38x38 paletted file, interlaced + "s38i3p04", # 38x38 paletted file, interlaced "s38n3p04", # 38x38 paletted file, no interlacing - # "s39i3p04", # 39x39 paletted file, interlaced + "s39i3p04", # 39x39 paletted file, interlaced "s39n3p04", # 39x39 paletted file, no interlacing - # "s40i3p04", # 40x40 paletted file, interlaced + "s40i3p04", # 40x40 paletted file, interlaced "s40n3p04", # 40x40 paletted file, no interlacing - # "bgai4a08", # 8 bit grayscale, alpha, no background chunk, interlaced - # "bgai4a16", # 16 bit grayscale, alpha, no background chunk, interlaced + "bgai4a08", # 8 bit grayscale, alpha, no background chunk, interlaced + "bgai4a16", # 16 bit grayscale, alpha, no background chunk, interlaced "bgan6a08", # 3x8 bits rgb color, alpha, no background chunk - # "bgan6a16", # 3x16 bits rgb color, alpha, no background chunk + "bgan6a16", # 3x16 bits rgb color, alpha, no background chunk "bgbn4a08", # 8 bit grayscale, alpha, black background chunk - # "bggn4a16", # 16 bit grayscale, alpha, gray background chunk + "bggn4a16", # 16 bit grayscale, alpha, gray background chunk "bgwn6a08", # 3x8 bits rgb color, alpha, white background chunk - # "bgyn6a16", # 3x16 bits rgb color, alpha, yellow background chunk + "bgyn6a16", # 3x16 bits rgb color, alpha, yellow background chunk - # "tbbn0g04", # transparent, black background chunk - # # "tbbn2c16", # transparent, blue background chunk + "tbbn0g04", # transparent, black background chunk + "tbbn2c16", # transparent, blue background chunk "tbbn3p08", # transparent, black background chunk - # # "tbgn2c16", # transparent, green background chunk + "tbgn2c16", # transparent, green background chunk "tbgn3p08", # transparent, light-gray background chunk "tbrn2c08", # transparent, red background chunk - # # "tbwn0g16", # transparent, white background chunk + "tbwn0g16", # transparent, white background chunk "tbwn3p08", # transparent, white background chunk "tbyn3p08", # transparent, yellow background chunk "tp0n0g08", # not transparent for reference (logo on gray) @@ -98,22 +98,22 @@ const "tp1n3p08", # transparent, but no background chunk "tm3n3p02", # multiple levels of transparency, 3 entries - # "g03n0g16", # grayscale, file-gamma = 0.35 + "g03n0g16", # grayscale, file-gamma = 0.35 "g03n2c08", # color, file-gamma = 0.35 "g03n3p04", # paletted, file-gamma = 0.35 - # "g04n0g16", # grayscale, file-gamma = 0.45 + "g04n0g16", # grayscale, file-gamma = 0.45 "g04n2c08", # color, file-gamma = 0.45 "g04n3p04", # paletted, file-gamma = 0.45 - # "g05n0g16", # grayscale, file-gamma = 0.55 + "g05n0g16", # grayscale, file-gamma = 0.55 "g05n2c08", # color, file-gamma = 0.55 "g05n3p04", # paletted, file-gamma = 0.55 - # "g07n0g16", # grayscale, file-gamma = 0.70 + "g07n0g16", # grayscale, file-gamma = 0.70 "g07n2c08", # color, file-gamma = 0.70 "g07n3p04", # paletted, file-gamma = 0.70 - # "g10n0g16", # grayscale, file-gamma = 1.00 + "g10n0g16", # grayscale, file-gamma = 1.00 "g10n2c08", # color, file-gamma = 1.00 "g10n3p04", # paletted, file-gamma = 1.00 - # "g25n0g16", # grayscale, file-gamma = 2.50 + "g25n0g16", # grayscale, file-gamma = 2.50 "g25n2c08", # color, file-gamma = 2.50 "g25n3p04", # paletted, file-gamma = 2.50 @@ -129,12 +129,12 @@ const "f04n2c08", # color, no interlacing, filter-type 4 "f99n0g04", # bit-depth 4, filter changing per scanline - # "pp0n2c16", # six-cube palette-chunk in true-color image + "pp0n2c16", # six-cube palette-chunk in true-color image "pp0n6a08", # six-cube palette-chunk in true-color+alpha image "ps1n0g08", # six-cube suggested palette (1 byte) in grayscale image - # "ps1n2c16", # six-cube suggested palette (1 byte) in true-color image + "ps1n2c16", # six-cube suggested palette (1 byte) in true-color image "ps2n0g08", # six-cube suggested palette (2 bytes) in grayscale image - # "ps2n2c16", # six-cube suggested palette (2 bytes) in true-color image + "ps2n2c16", # six-cube suggested palette (2 bytes) in true-color image "ccwn2c08", # chroma chunk w:0.3127,0.3290 r:0.64,0.33 g:0.30,0.60 b:0.15,0.06 "ccwn3p08", # chroma chunk w:0.3127,0.3290 r:0.64,0.33 g:0.30,0.60 b:0.15,0.06 @@ -147,7 +147,7 @@ const "cm0n0g04", # modification time, 01-jan-2000 12:34:56 "cm7n0g04", # modification time, 01-jan-1970 00:00:00 "cm9n0g04", # modification time, 31-dec-1999 23:59:59 - # "cs3n2c16", # color, 13 significant bits + "cs3n2c16", # color, 13 significant bits "cs3n3p08", # paletted, 3 significant bits "cs5n2c08", # color, 5 significant bits "cs5n3p08", # paletted, 5 significant bits @@ -163,14 +163,16 @@ const "ctjn0g04", # international UTF-8, japanese "exif2c08", # chunk with jpeg exif data - # "oi1n0g16", # grayscale mother image with 1 idat-chunk - # "oi1n2c16", # color mother image with 1 idat-chunk - # "oi2n0g16", # grayscale image with 2 idat-chunks - # "oi2n2c16", # color image with 2 idat-chunks - # "oi4n0g16", # grayscale image with 4 unequal sized idat-chunks - # "oi4n2c16", # color image with 4 unequal sized idat-chunks - # "oi9n0g16", # grayscale image with all idat-chunks length one - # "oi9n2c16", # color image with all idat-chunks length one + "oi1n0g16", # grayscale mother image with 1 idat-chunk + "oi1n2c16", # color mother image with 1 idat-chunk + "oi2n0g16", # grayscale image with 2 idat-chunks + "oi2n2c16", # color image with 2 idat-chunks + "oi4n0g16", # grayscale image with 4 unequal sized idat-chunks + "oi4n2c16", # color image with 4 unequal sized idat-chunks + "oi9n0g16", # grayscale image with all idat-chunks length one + "oi9n2c16", # color image with all idat-chunks length one + + "PngSuite", "z00n2c08", # color, no interlacing, compression level 0 (none) "z03n2c08", # color, no interlacing, compression level 3 @@ -185,12 +187,12 @@ const "xs7n0g01", # 7th byte a space instead of control-Z "xcrn0g04", # added cr bytes "xlfn0g04", # added lf bytes - # "xhdn0g08", # incorrect IHDR checksum + "xhdn0g08", # incorrect IHDR checksum "xc1n0g08", # color type 1 "xc9n2c08", # color type 9 "xd0n2c08", # bit-depth 0 "xd3n2c08", # bit-depth 3 "xd9n2c08", # bit-depth 99 "xdtn0g01", # missing IDAT chunk - # "xcsn0g01" # incorrect IDAT checksum + "xcsn0g01" # incorrect IDAT checksum ] diff --git a/tests/test_gif.nim b/tests/test_gif.nim index 455ff25e..d4d4b0b7 100644 --- a/tests/test_gif.nim +++ b/tests/test_gif.nim @@ -36,6 +36,11 @@ block: doAssert animatedGif.frames.len == 36 doAssert animatedGif.intervals.len == animatedGif.frames.len +block: + var data = readFile("tests/fileformats/gif/3x5.gif") + data[12] = char(49) # Pixel aspect ratio is advisory and can be ignored. + discard decodeGif(data) + block: proc addLe16(data: var string, value: int) = data.add(char(value and 0xff)) diff --git a/tests/test_png.nim b/tests/test_png.nim index 28989afe..2138cd9a 100644 --- a/tests/test_png.nim +++ b/tests/test_png.nim @@ -1,10 +1,33 @@ import pixie, pixie/fileformats/png, pngsuite, strformat +when defined(writeImages): + import write_images + +when defined(writeImages): + resetOutputDir(pngSuiteOutputDir) + echo &"Writing decoded PNGSuite images to {pngSuiteOutputDir}" + for file in pngSuiteFiles: let original = readFile(&"tests/fileformats/png/pngsuite/{file}.png") decoded = decodePng(original) encoded = encodePng(decoded) + when defined(writeImages): + newImage(decoded).writeOutputImage(pngSuiteOutputDir, file) + +block: + let + png8 = decodePng(readFile("tests/fileformats/png/pngsuite/basn2c08.png")) + png16 = decodePng(readFile("tests/fileformats/png/pngsuite/basn2c16.png")) + image16 = png16.convertToImage() + doAssert png8.bitDepth == 8 + doAssert png8.data.len == png8.width * png8.height + doAssert png8.data16.len == 0 + doAssert png16.bitDepth == 16 + doAssert png16.data.len == 0 + doAssert png16.data16.len == png16.width * png16.height + doAssert image16.width == png16.width + doAssert image16.height == png16.height block: for channels in 1 .. 4: diff --git a/tests/test_qoi.nim b/tests/test_qoi.nim index 8949fd0b..8c2e51c1 100644 --- a/tests/test_qoi.nim +++ b/tests/test_qoi.nim @@ -1,17 +1,30 @@ -import pixie, pixie/fileformats/qoi +import math, pixie, pixie/fileformats/qoi const tests = ["testcard", "testcard_rgba"] +proc srgbToLinear(value: uint8): uint8 = + let c = value.float32 / 255 + let linear = + if c <= 0.04045: + c / 12.92 + else: + pow((c + 0.055) / 1.055, 2.4) + round(linear * 255).uint8 + +proc addBe32(data: var string, value: int) = + data.add(char((value shr 24) and 0xff)) + data.add(char((value shr 16) and 0xff)) + data.add(char((value shr 8) and 0xff)) + data.add(char(value and 0xff)) + for name in tests: let path = "tests/fileformats/qoi/" & name & ".qoi" input = readImage(path) - control = readImage("tests/fileformats/qoi/" & name & ".png") dimensions = decodeQoiDimensions(readFile(path)) - doAssert input.data == control.data, "input mismatch of " & name doAssert input.width == dimensions.width doAssert input.height == dimensions.height - discard encodeQoi(control) + discard encodeQoi(input) for name in tests: let @@ -19,4 +32,51 @@ for name in tests: input = decodeQoi(readFile(path)) output = decodeQoi(encodeQoi(input)) doAssert output.data.len == input.data.len - doAssert output.data == input.data + doAssert output.colorspace == Linear + if input.colorspace == Linear: + doAssert output.data == input.data + else: + for i, px in input.data: + doAssert output.data[i] == rgba( + px.r.srgbToLinear(), + px.g.srgbToLinear(), + px.b.srgbToLinear(), + px.a + ) + +block: + var data = "" + data.add(qoiSignature) + data.addBe32(1) + data.addBe32(1) + data.add(char(4)) # RGBA + data.add(char(sRBG.uint8)) + data.add(char(0xff)) # QOI_OP_RGBA + data.add(char(128)) + data.add(char(64)) + data.add(char(255)) + data.add(char(255)) + for _ in 0 .. 6: + data.add(char(0)) + data.add(char(1)) + + let + qoi = decodeQoi(data) + image = convertToImage(decodeQoi(data)) + encoded = decodeQoi(encodeQoi(qoi)) + + doAssert qoi.colorspace == sRBG + doAssert qoi.data[0] == rgba(128, 64, 255, 255) + doAssert image.data[0] == rgbx( + 128.uint8.srgbToLinear(), + 64.uint8.srgbToLinear(), + 255.uint8.srgbToLinear(), + 255 + ) + doAssert encoded.colorspace == Linear + doAssert encoded.data[0] == rgba( + 128.uint8.srgbToLinear(), + 64.uint8.srgbToLinear(), + 255.uint8.srgbToLinear(), + 255 + ) diff --git a/tests/test_tiff.nim b/tests/test_tiff.nim index 0c1789f8..aa01aa66 100644 --- a/tests/test_tiff.nim +++ b/tests/test_tiff.nim @@ -1,9 +1,55 @@ -import pixie, pixie/fileformats/tiff +import pixie/common, pixie/fileformats/tiff let - t = decodeTiff(readFile("tests/fileformats/tiff/pc260001.tif")) + data = readFile("tests/fileformats/tiff/pc260001.tif") + dimensions = decodeTiffDimensions(data) + t = decodeTiff(data) image = newImage(t) -# image.writeFile("tests/fileformats/tiff/pc260001.png") + +doAssert dimensions.width == t.width +doAssert dimensions.height == t.height +doAssert t.dataFormat == tiffRgba +doAssert t.data.len == t.width * t.height +doAssert t.dataFloat32.len == 0 +doAssert t.dataGray16.len == 0 +doAssert t.dataGrayInt16.len == 0 +doAssert image.width == t.width +doAssert image.height == t.height + +let + gray16Tiff = Tiff( + width: 1, + height: 1, + dataFormat: tiffGray16, + dataGray16: @[32768.uint16] + ) + gray16Image = convertToImage(gray16Tiff) + grayInt16Tiff = Tiff( + width: 3, + height: 1, + dataFormat: tiffGrayInt16, + dataGrayInt16: @[-32768.int16, 0.int16, 32767.int16] + ) + grayInt16Image = convertToImage(grayInt16Tiff) + floatTiff = Tiff( + width: 1, + height: 1, + dataFormat: tiffFloat32, + dataFloat32: @[0.5'f32] + ) + floatImage = convertToImage(floatTiff) + +doAssert gray16Image.width == 1 +doAssert gray16Image.height == 1 +doAssert gray16Image.data[0].r == 127 +doAssert grayInt16Image.width == 3 +doAssert grayInt16Image.height == 1 +doAssert grayInt16Image.data[0].r == 0 +doAssert grayInt16Image.data[1].r == 127 +doAssert grayInt16Image.data[2].r == 255 +doAssert floatImage.width == 1 +doAssert floatImage.height == 1 +doAssert floatImage.data[0].r == 128 block: proc addLe16(data: var string, value: int) = diff --git a/tests/write_images.nim b/tests/write_images.nim new file mode 100644 index 00000000..224b3b2a --- /dev/null +++ b/tests/write_images.nim @@ -0,0 +1,17 @@ +import os +import pixie/fileformats/png, pixie/images + +const + imageTestSuiteOutputDir* = "tmp" / "image test suite" + pngSuiteOutputDir* = "tmp" / "png suite" + +proc resetOutputDir*(dir: string) = + when defined(writeImages): + if dirExists(dir): + removeDir(dir) + createDir(dir) + +proc writeOutputImage*(image: Image, dir, sourcePath: string) = + when defined(writeImages): + createDir(dir) + writeFile(dir / (sourcePath.splitPath.tail & ".png"), image.encodePng())