Add a dedup option.

.circleci/config.yml

@@ -95,7 +95,7 @@ jobs:
     steps:
       - checkout
       - tox:
-          env: flake8
+          env: lint
   release:
     docker:
       - image: python:3.8

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Change Log
 
+## Version 1.5.0
+
+### Features
+- Add a deduplicate option to avoid writing all identical data blocks ([#92](../../pull/92))
+- Refactor how ifds-first option emits data so it is closer to the COGs standard ([#92](../../pull/92))
+
 ## Version 1.4.1
 
 ### Improvements

tests/test_write_tiff.py

@@ -1,4 +1,5 @@
 import copy
+import hashlib
 import logging
 import os
 
@@ -152,6 +153,30 @@ def test_write_bigtiff_with_repeated_offset_data(tmp_path):
     assert destinfo['bigtiff'] is False
 
 
+def test_write_with_dedup(tmp_path):
+    path = os.path.join(os.path.dirname(__file__), 'data', 'good_single.tif')
+    info = tifftools.read_tiff(path)
+    uniqueString = b'UNIQUESTRING'
+    info['ifds'][0]['tags'][23456] = {
+        'datatype': tifftools.Datatype.UNDEFINED,
+        'data': uniqueString
+    }
+    info['ifds'][0]['tags'][23457] = {
+        'datatype': tifftools.Datatype.UNDEFINED,
+        'data': uniqueString
+    }
+    destpath = tmp_path / 'sample.tiff'
+    tifftools.write_tiff(info, destpath)
+    assert len(open(destpath, 'rb').read().split(uniqueString)) == 3
+    dest2path = tmp_path / 'sample2.tiff'
+    tifftools.write_tiff(info, dest2path, dedup=True)
+    assert len(open(dest2path, 'rb').read().split(uniqueString)) == 2
+    info2 = tifftools.read_tiff(dest2path)
+    dest3path = tmp_path / 'sample3.tiff'
+    tifftools.write_tiff(info2, dest3path)
+    assert open(destpath, 'rb').read() == open(dest3path, 'rb').read()
+
+
 def test_write_bytecount_data(tmp_path):
     path = os.path.join(os.path.dirname(__file__), 'data', 'good_single.tif')
     info = tifftools.read_tiff(path)
@@ -227,5 +252,14 @@ def test_write_ifds_first(tmp_path):
     destpath = tmp_path / 'sample.tiff'
     tifftools.write_tiff(info, destpath, ifdsFirst=True)
     len = os.path.getsize(destpath)
-    tifftools.write_tiff(info, destpath, allowExisting=True)
-    assert len == os.path.getsize(destpath)
+    destpath2 = tmp_path / 'sample2.tiff'
+    tifftools.write_tiff(info, destpath2)
+    assert len == os.path.getsize(destpath2)
+
+    if hasattr(hashlib, 'file_digest'):
+        info = tifftools.read_tiff(destpath)
+        destpath3 = tmp_path / 'sample3.tiff'
+        tifftools.write_tiff(info, destpath3)
+        assert (
+            hashlib.file_digest(open(destpath2, 'rb'), 'sha512').hexdigest() ==
+            hashlib.file_digest(open(destpath3, 'rb'), 'sha512').hexdigest())

tifftools/commands.py

@@ -59,7 +59,9 @@ def tiff_concat(source, output, overwrite=False, **kwargs):
         nextInfo = read_tiff(path)
         ifds.extend(nextInfo['ifds'])
     _apply_flags_to_ifd(ifds, **kwargs)
-    write_tiff(ifds, output, allowExisting=overwrite, ifdsFirst=kwargs.get('ifdsfirst', False))
+    write_tiff(ifds, output, allowExisting=overwrite,
+               ifdsFirst=kwargs.get('ifdsfirst', False),
+               dedup=kwargs.get('dedup', False))
 
 
 def _tiff_dump_tag(tag, taginfo, linePrefix, max, dest=None, max_text=None, ifd=None):
@@ -360,7 +362,8 @@ def tiff_split(source, prefix=None, subifds=False, overwrite=False, **kwargs):
         logger.info('Writing %s', outputPath)
         _apply_flags_to_ifd(ifd, **kwargs)
         write_tiff(ifd, outputPath, allowExisting=overwrite,
-                   ifdsFirst=kwargs.get('ifdsfirst', False))
+                   ifdsFirst=kwargs.get('ifdsfirst', False),
+                   dedup=kwargs.get('dedup', False))
 
 
 def _value_to_types_numeric(results):
@@ -527,7 +530,9 @@ def _tiff_set(source, output=None, setlist=None, unset=None, setfrom=None,
             else:
                 ifd['tags'][int(tag)] = setinfo['ifds'][0]['tags'][int(tag)]
     _apply_flags_to_ifd(info, **kwargs)
-    write_tiff(info, output, allowExisting=overwrite, ifdsFirst=kwargs.get('ifdsfirst', False))
+    write_tiff(info, output, allowExisting=overwrite,
+               ifdsFirst=kwargs.get('ifdsfirst', False),
+               dedup=kwargs.get('dedup', False))
 
 
 def tiff_set(source, output=None, overwrite=False, setlist=None, unset=None,
@@ -599,6 +604,9 @@ def main(args=None):
     }, {
         'args': ('--ifdsfirst', '--ifds-first'),
         'kwargs': dict(action='store_true', help='Store IFDs before their related data.'),
+    }, {
+        'args': ('--dedup', '--deduplicate'),
+        'kwargs': dict(action='store_true', help='Do not repeat identical data.'),
     }, {
         'args': ('--stop-on-warning', '-X'),
         'kwargs': dict(

tifftools/constants.py

@@ -28,9 +28,10 @@ def __str__(self):
         return '%d (0x%X)' % (self.value, self.value)
 
     def __getitem__(self, key):
-        if hasattr(self, str(key)):
+        try:
             return getattr(self, str(key))
-        raise KeyError(key)
+        except AttributeError:
+            raise KeyError(key)
 
     def __int__(self):
         return self.value
@@ -61,9 +62,7 @@ def __hash__(self):
         return hash((type(self).__name__, self.value))
 
     def get(self, key, default=None):
-        if hasattr(self, str(key)):
-            return getattr(self, str(key))
-        return default
+        return getattr(self, str(key), default)
 
 
 class TiffTag(TiffConstant):