Merge pull request #160 from AllenInstitute/develop

Develop -> master

Author: RussTorres

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.3.0
+current_version = 2.4.0
 commit = True
 tag = True
 

.travis.yml

@@ -1,9 +1,7 @@
 language: python
 python:
   - "2.7"
-  - "3.5"
   - "3.6"
-  - "3.6-dev" # 3.6 development branch
   - "3.7-dev"
 services:
   - docker
@@ -35,11 +33,14 @@ before_install:
   - mkdir -p /tmp/example_1 && cp -R $RENDER_WS_JAVA_CLIENT_EXAMPLE_DATA/example_1 /tmp/.
   - cp -R $RENDER_APP_EXAMPLE_DATA/* /tmp/.
   - export RENDER_EXAMPLE_DATA=/tmp
+  - echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
   - docker-compose up -d
 env:
   global:
      - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts
-     - secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ="
+     - secure: "AZCuswnHLEvDkfcmijSb4sy1E0vCg3nHn8gk9EvO7z9JeJG4FtHYBW3bLiN4WSjxszzaCK16lku0RY4OL2gCv0C87iHaXkDqAWwN0YIFGOkzDhZJRD5CENTsx1w/JFySqTBNiO82gIAnNP+J19mFm1ArtNhDIYAiUPo+CseG8S0cpbQqQe3tBd+XB/u32PUS1zA1mIg6qDXKupoDCUuWBxCZ01Jdv/YDUH3zxz6K9SQmbzJ4m5B1TCw8E6r/dj6mBSjJSh2XmZlOU0Ha2U9nEkkkr7nbNU1GYQYRt9uiqDH4juvFw6WePP+MVzFJCq0Hqdyb9lk/7jh4BX5ffN2o/vs14FuXZ57wblcHZkkEgD0ViIlGjqgPn0WuTJstkVlpvvwuweVfXXxJrYAZwhf14goKiWneDZMJ6g82Z7r9GVQFwEwl/mu5fdLP/qKZ8i3AAd9SgnGdn5eQ/xD5JVJj9JZFqfLUJs7JQreGYfW2r5URIL3Kd2laRucUD4IPwnzHT+VjgNHFtv07bmNxRGPdwC4pbWOMiyq2BvG3JFw/y8j3bNvLxK6XLhDgHtV13NinZx8uZcocb8yCNyBpez+Pfo3RzTcHKnF5foKoEEZIU5OuyGGgeHUup/vg2qsKGn3Wl1nH5fWSbdR2pPHLSD+mpZdLAA9JjqEfVrhZtoXLUpA="
+     - secure: "tnw8YYrVoasmJ+3I3VjENfx6BIBG2qaJ4m0+rUR9dal7cmrGTrcyJ1d41Nm+K1XKRvLvBkEHKBzdEVlg2XaAOVBTam6u+zW/Dc16ZXgz7yK1qXtytwiXBe2P7c9HePXIDA/SCapKXmjF5/Zt0BdpXv4fPyK8kbdIjels9Xt8teDTfSxJ6AfXlfXXPiF6A1Lzgf+z9GRL9GXZrgic6itFdNfgErGThgefQSJKr3wKR/wRyx87QD5juzrZtcwtFzcvvO8X8n6D+oOdc9XfRsE1EgFO8RHla32c0BbcKmRP8vwwtfVUt1tqU3SQZP6+n9EUBiSUR9/B50hCOiX5O2RdtHIMncJT0n4ZQe2PmTmwpcY1bU1NJpUszxPXNZrEHRdGqbXfiQq8+sONAWZcrzbqLsP2cXpt+4FrHk+iepu0RBLpLq/UdcVLozELgkNxM1C9k3OF6OlAC//5jWi4hjeoc2L8LpR09Cxm7PCtBvqbCNO2mPhv+h4dsFzmlyN2JL34lQyfzD1v1Jqz4ifJHYupKKtIdA7UiU+RU40ylDzTKfvlI63MkUwxS2u4BY0Gc5WWy/NoJuF6F39c+peoLgcoHiKnXoIvTyWCNSfieXCnuiHrU7nLnijM9qzSZxM6a43KiSGRZAQInQUXzKX5a7cGHaQIFskd3TK71emnhdewM40="
+     - secure: "UucVIVQCg9T2d9NiSdMirj37nlwWEyZi2mybzNNVJ8TMfC05Vz+E/b/1ByxOLpUYmXZTzU22Xx316ueIaIk05L+Q8q4Dy1S/spaKBNcJQ8kDxlcD7y0YBPX+QqyOpXCnB1Kt1oawy92g2qALFDMBoupr7xT8aTvCwDigrJYghMiVM1RVlQQAJBaGkB32qs934uQeknvLqSZV35BTFGHTaqJoWV8OyxNwOZFtf5Z8gdWxrAYHOQg9iW+5CP+zLhybROLrMuI5vhj/yvP2I07Scx+jMKOgGw7snPygHpwR0dNw/NvNdy+XIx3zqUlts5Bb1nqCeaNzNouXd1D5NHoqsw/0VLt/HcbCxUKXuaViJy5KfN/qi9U7lSJ0KC2qazwsQXviFYTrT2q1U2n6zlV/CC9e1sLjQmIF3GAysFD3Fcg88Qz/3aZKl6nkqP/T2Y1t/FOPjBSD5ytlpr8xbs6rpxZMU/k91agD5UeP4/lwbncfsJswPNajdTz5scWjHPACDLIxYTwEry64z808lHcc23c9MIeqrkXMI4nPYnLSPQyhXBOJncEugKernBh053Pu5+hDrg9bsoIjtsGmh+p9GnIjrtPAkc8g4tyU+v0PKMVUkqw+mvkVEgzRlKUzURuRTVy5lTHisu0mo6Xmtgw0qfVkTOa2Gnr9gsfmaTToyXY="
 # command to run tests
 script:
   - python setup.py test # or py.test for Python versions 3.5 and below
@@ -49,4 +50,4 @@ cache:
     - $HOME/.m2
 after_success:
   - codecov -t 3f12d985-af62-455d-a11d-9669c039640d
-  - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=fcollman/render-python .travis/merge_script.sh"
+  - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=AllenInstitute/render-python .travis/merge_script.sh"

README.rst

@@ -1,11 +1,11 @@
 .. image:: https://readthedocs.org/projects/render-python/badge/
    :target: http://render-python.readthedocs.io/en/latest/
    :alt: Documentation Status 
-.. image:: https://travis-ci.org/fcollman/render-python.svg?branch=master
-   :target: https://travis-ci.org/fcollman/render-python
+.. image:: https://travis-ci.com/AllenInstitute/render-python.svg?branch=master
+   :target: https://travis-ci.com/AllenInstitute/render-python
    :alt: Build Status
-.. image:: https://codecov.io/gh/fcollman/render-python/branch/master/graph/badge.svg
-  :target: https://codecov.io/gh/fcollman/render-python
+.. image:: https://codecov.io/gh/AllenInstitute/render-python/branch/master/graph/badge.svg
+  :target: https://codecov.io/gh/AllenInstitute/render-python
 
 render-python
 #############

integration_tests/test_client_integrated.py

@@ -185,7 +185,6 @@ def teststack2(render, render_example_tilespec_and_transforms):
     renderapi.stack.delete_stack(stack, render=render)
 
 
-
 def test_tile_pair_client(render, teststack, **kwargs):
     zvalues = np.array(renderapi.stack.get_z_values_for_stack(
         teststack, render=render))
@@ -196,6 +195,21 @@ def test_tile_pair_client(render, teststack, **kwargs):
     assert isinstance(tilepairjson, dict)
     assert len(tilepairjson['neighborPairs']) > 3
 
+    multitpjson_tmp, multitpfiles_tmp = renderapi.client.tilePairClient(
+        teststack, np.min(zvalues), np.max(zvalues), outjson=None,
+        render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs)
+
+    with tempfile.NamedTemporaryFile(
+            suffix=".json", mode='r', delete=False) as f:
+        outjson_specified = f.name
+
+    multitpjson, multitpfiles = renderapi.client.tilePairClient(
+        teststack, np.min(zvalues), np.max(zvalues), outjson=outjson_specified,
+        render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs)
+
+    assert len(multitpjson["neighborPairs"]) == len(tilepairjson["neighborPairs"]) == len(multitpjson_tmp["neighborPairs"])
+    assert len(multitpfiles) == len(multitpfiles_tmp) == len(tilepairjson["neighborPairs"]) // 2
+
 
 @pytest.mark.parametrize("bounds,raises", [
     ({}, True),

renderapi/__init__.py

@@ -13,7 +13,7 @@
 from .render import connect
 from .render import Render
 
-__version__ = "2.3.0"
+__version__ = "2.4.0"
 __all__ = ['render', 'client', 'tilespec', 'errors',
            'stack', 'image', 'pointmatch', 'coordinate',
            'connect', 'transform', 'resolvedtiles', 'Render']

renderapi/client/client_calls.py

@@ -1,6 +1,7 @@
 import logging
 import json
 import os
+import re
 import subprocess
 import tempfile
 
@@ -135,6 +136,11 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False,
                    excludeSameSectionNeighbors=None,
                    excludePairsInMatchCollection=None,
                    minx=None, maxx=None, miny=None, maxy=None,
+                   useRowColPositions=None, existingMatchOwner=None,
+                   minExistingMatchCount=None, onlyIncludeTilesFromStack=None,
+                   onlyIncludeTilesNearTileIdsJson=None, maxPairsPerFile=None,
+                   return_jsondata=True,
+                   return_jsonfiles=False,
                    subprocess_mode=None,
                    host=None, port=None, owner=None, project=None,
                    client_script=None, memGB=None,
@@ -191,11 +197,29 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False,
         minimum y bound from which tile 'p' is selected
     maxy : float
         maximum y bound from wich tile 'p' is selected
+    useRowColPositions : bool
+        whether to use raster positions for neighbor analysis rather
+        than radial cutoff
+    existingMatchOwner : str
+        owner for the excludePairsInMatchCollection collection
+    minExistingMatchCount : int
+        minimum match threshold to exclude pairs
+        present in excludePairsInMatchCollection collection
+    onlyIncludeTilesFromStack : str
+        only include tiles which exist in this stack
+    onlyIncludeTilesNearTileIdsJson : str
+        path to json listing tileIds which should be paired
+    maxPairsPerFile : int
+        maximum neighborPairs per file.  Generating more pairs than
+        this number (default 100000) will generate additional json
+        files with a p[0-9]+ suffix on the root.
 
     Returns
     -------
-    :obj:`list` of :obj:`dict`
+    :obj:`list` of :obj:`dict`, optional
         list of tilepairs
+    :obj:`list` of string, optional
+        list of json files containing tilepairs
     """
     if outjson is None:
         with tempfile.NamedTemporaryFile(
@@ -219,6 +243,18 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False,
                        '--excludeSameSectionNeighbors') +
              get_param(excludePairsInMatchCollection,
                        '--excludePairsInMatchCollection') +
+             get_param(useRowColPositions,
+                       '--useRowColPositions') +
+             get_param(existingMatchOwner,
+                       '--existingMatchOwner') +
+             get_param(minExistingMatchCount,
+                       '--minExistingMatchCount') +
+             get_param(onlyIncludeTilesFromStack,
+                       "--onlyIncludeTilesFromStack") +
+             get_param(onlyIncludeTilesNearTileIdsJson,
+                       '--onlyIncludeTilesNearTileIdsJson') +
+             get_param(maxPairsPerFile,
+                       '--maxPairsPerFile') +
              ['--toJson', outjson] +
              get_param(minx, '--minX') + get_param(maxx, '--maxX') +
              get_param(miny, '--minY') + get_param(maxy, '--maxY'))
@@ -228,12 +264,45 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False,
                        subprocess_mode=subprocess_mode,
                        add_args=argvs, **kwargs)
 
-    with open(outjson, 'r') as f:
-        jsondata = json.load(f)
+    # We create the jsonfile, so if it is empty it could be multiple
+    if not os.path.isfile(outjson) or os.stat(outjson).st_size == 0:
+        if os.path.isfile(outjson):
+            # outjson we created is empty, so remove it
+            os.remove(outjson)
+
+        outbn_root, outbn_ext = os.path.splitext(os.path.basename(outjson))
+        outbn_pattern = "{root}_p[0-9]+{ext}".format(
+            root=outbn_root, ext=outbn_ext)
+        jsonfiles = [
+            os.path.join(os.path.dirname(outjson), bn)
+            for bn in os.listdir(os.path.dirname(outjson))
+            if re.match(outbn_pattern, bn)]
+    else:
+        jsonfiles = [outjson]
+
+    if return_jsondata:
+        jsondata_list = []
+        for jsonfile in jsonfiles:
+            with open(jsonfile, 'r') as f:
+                jsondata_list.append(json.load(f))
+        if len({d["renderParametersUrlTemplate"] for d in jsondata_list}) != 1:  # pragma: no cover
+            raise ValueError(
+                "Found tilepair files with disparate "
+                "renderParametersUrlTemplate values. "
+                "Maybe there are additional files "
+                "matching the outjson pattern?")
+        pairdata = [i for l in (d["neighborPairs"] for d in jsondata_list)
+                    for i in l]
+        jsondata = dict(jsondata_list[0], **{"neighborPairs": pairdata})
 
     if delete_json:
-        os.remove(outjson)
-    return jsondata
+        for jsonfile in jsonfiles:
+            os.remove(jsonfile)
+
+    return ((jsondata, jsonfiles) if return_jsonfiles and return_jsondata
+            else jsondata if return_jsondata
+            else jsonfiles if return_jsonfiles
+            else None)
 
 
 @renderclientaccess
@@ -346,10 +415,11 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None,
                         maxIntensity=None, minIntensity=None, bounds=None,
                         format=None, channel=None, customOutputFolder=None,
                         customSubFolder=None, padFileNamesWithZeros=None,
-                        doFilter=None, fillWithNoise=None, imageType=None,
-                        subprocess_mode=None, host=None, port=None, owner=None,
-                        project=None, client_script=None, memGB=None,
-                        render=None, **kwargs):
+                        resolutionUnit=None, doFilter=None, fillWithNoise=None,
+                        imageType=None, subprocess_mode=None, host=None,
+                        port=None, owner=None, project=None,
+                        client_script=None, memGB=None, render=None,
+                        **kwargs):
     """run RenderSectionClient.java
 
     Parameters
@@ -379,6 +449,9 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None,
         folder to save all images in under outputFolder (overrides default of none)
     padFileNamesWithZeros: bool
         whether to pad file names with zeros to make sortable
+    resolutionUnit: str
+        if format is tiff and unit is specified (e.g. as 'nm'), include resolution data 
+        in rendered tiff headers.
     imageType: int
         8,16,24 to specify what kind of image type to save
     doFilter : str
@@ -418,9 +491,11 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None,
              get_param(customOutputFolder, '--customOutputFolder') +
              get_param(imageType, '--imageType') +
              get_param(channel, '--channels') +
-             get_param(customSubFolder, '--customSubFolder') +
+             get_param(customSubFolder, '--customSubFolder') +             
              get_param(padFileNamesWithZeros, '--padFileNamesWithZeros') +
+             get_param(resolutionUnit, '--resolutionUnit') +
              bound_param + zs)
+    
     call_run_ws_client('org.janelia.render.client.RenderSectionClient',
                        memGB=memGB, client_script=client_script,
                        subprocess_mode=subprocess_mode, add_args=argvs,

renderapi/pointmatch.py

@@ -11,6 +11,83 @@
 logger.addHandler(NullHandler())
 
 
+def copy_match_explicit(match):
+    """create independent match dictionary equivalent to the input match
+      significantly faster than e.g. copy.deepcopy or json serialization
+
+    Parameters
+    ----------
+    match : dict
+      match dictionary
+
+    Returns
+    -------
+    new_match : dict
+      match dictionary equivalent to input match
+
+    """
+    # explicitly copy match dictionary since it contains lists
+    new_matchd = {
+        "p": [i[:] for i in match["matches"]["p"]],
+        "q": [i[:] for i in match["matches"]["q"]],
+        "w": match["matches"]["w"][:]
+    }
+
+    new_match = {k: (match[k] if k != "matches" else new_matchd)
+                 for k in match.keys()}
+    return new_match
+
+
+def copy_matches_explicit(matches):
+    """create independent match dictionaries equivalent to the input matches
+      significantly faster than e.g. copy.deepcopy or json serialization
+
+    Parameters
+    ----------
+    matches :  list[dict]
+      list of match dictionaries to copy
+
+    Returns
+    -------
+    new_matches : list[dict]
+      list of match dictionaries equivalent to input matches
+
+    """
+    return [copy_match_explicit(match) for match in matches]
+
+
+def swap_matchpair(match, copy=True):
+    """
+
+    Parameters
+    ----------
+    match : dict
+        match dictionary to swap p->q,q->p
+    copy : bool
+        whether to return a copy which, when modified, does not change original
+
+    Returns
+    -------
+    new_match : dict
+        match dictionary with "p" and "q" swapped
+    """
+    updated_d = {
+        "pId": match["qId"],
+        "qId": match["pId"],
+        "qGroupId": match["pGroupId"],
+        "pGroupId": match["qGroupId"],
+        "matches": {
+            "p": match["matches"]["q"],
+            "q": match["matches"]["p"],
+            "w": match["matches"]["w"]  # include weight because shallow swap
+            }
+    }
+
+    new_match = {k: updated_d.get(k, v) for k, v in match.items()}
+
+    return (copy_match_explicit(new_match) if copy else new_match)
+
+
 @renderaccess
 def get_matchcollection_owners(host=None, port=None,
                                session=requests.session(),

renderapi/resolvedtiles.py

@@ -53,6 +53,32 @@ def from_dict(self, d):
     """  # noqa: E501
 
 
+def combine_resolvedtiles(rts_l):
+    """combine multiple ResolvedTiles objects into a single ResolvedTiles
+
+    Like render, this has an implicit expectation that transformIds and tileIds
+      are unique among all input objects
+
+    Parameters
+    ----------
+    rts_l : list[renderapi.resolvedtiles.ResolvedTiles]
+        list of ResolvedTiles objects to combine
+
+    Returns
+    -------
+    rts : renderapi.resolvedtiles.ResolvedTiles
+        combined ResolvedTiles object
+    """
+    tforms = list({tform.transformId: tform for l in (
+        rts.transforms for rts in rts_l)
+                   for tform in l}.values())
+    tspecs = list({ts.tileId: ts for l in (
+        rts.tilespecs for rts in rts_l)
+                   for ts in l}.values())
+
+    return ResolvedTiles(transformList=tforms, tilespecs=tspecs)
+
+
 @renderaccess
 def put_tilespecs(stack, resolved_tiles=None, deriveData=True,
                   tilespecs=None, shared_transforms=None,

renderapi/stack.py

@@ -50,7 +50,7 @@ def __init__(self, cycleNumber=None, cycleStepNumber=None,
         self.stackResolutionZ = stackResolutionZ
         self.mipmapPathBuilder = mipmapPathBuilder
         self.materializedBoxRootPath = materializedBoxRootPath
-        self.createTimestamp = (strftime('%Y-%M-%dT%H:%M:%S.00Z') if
+        self.createTimestamp = (strftime('%Y-%m-%dT%H:%M:%S.00Z') if
                                 createTimestamp is None else createTimestamp)
         self.versionNotes = versionNotes