Skip to content

Commit 2c8f5ae

Browse files
bsmartradiobsmartradio
committed
Make trailedAssociatorTask
Make trailedAssociatorTask which filters out trails whose lengths are above 0.416 arcseconds/second times the exposure time in length.
1 parent 6758148 commit 2c8f5ae

File tree

7 files changed

+265
-14
lines changed

7 files changed

+265
-14
lines changed

‎python/lsst/ap/association/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
# along with this program. If not, see <https://www.gnu.org/licenses/>.
2121

2222
from .version import *
23+
from .trailedSourceFilter import *
2324
from .association import *
2425
from .diaForcedSource import *
2526
from .loadDiaCatalogs import *

‎python/lsst/ap/association/association.py‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import lsst.pex.config as pexConfig
3333
import lsst.pipe.base as pipeBase
3434
from lsst.utils.timer import timeMethod
35+
from .trailedSourceFilter import TrailedSourceFilterTask
3536

3637
# Enforce an error for unsafe column/array value setting in pandas.
3738
pd.options.mode.chained_assignment = 'raise'
@@ -47,6 +48,18 @@ class AssociationConfig(pexConfig.Config):
4748
default=1.0,
4849
)
4950

51+
trailedSourceFilter = pexConfig.ConfigurableField(
52+
target=TrailedSourceFilterTask,
53+
doc="Subtask to filter artifact candidates based on morphological "
54+
"criteria, i.g. those that appear to be streaks.",
55+
)
56+
57+
doTrailedSourceFilter = pexConfig.Field(
58+
doc="Set flag for trailed source filter subtask to run.",
59+
dtype=bool,
60+
default=True,
61+
)
62+
5063

5164
class AssociationTask(pipeBase.Task):
5265
"""Associate DIAOSources into existing DIAObjects.
@@ -60,10 +73,16 @@ class AssociationTask(pipeBase.Task):
6073
ConfigClass = AssociationConfig
6174
_DefaultName = "association"
6275

76+
def __init__(self, *args, **kwargs):
77+
super().__init__(*args, **kwargs)
78+
if self.config.doTrailedSourceFilter:
79+
self.makeSubtask("trailedSourceFilter")
80+
6381
@timeMethod
6482
def run(self,
6583
diaSources,
66-
diaObjects):
84+
diaObjects,
85+
diffIm=None):
6786
"""Associate the new DiaSources with existing DiaObjects.
6887
6988
Parameters
@@ -72,6 +91,8 @@ def run(self,
7291
New DIASources to be associated with existing DIAObjects.
7392
diaObjects : `pandas.DataFrame`
7493
Existing diaObjects from the Apdb.
94+
diffIm : `pandas.DataFrame` optional
95+
Difference image on which the DiaSources were detected.
7596
7697
Returns
7798
-------
@@ -98,7 +119,13 @@ def run(self,
98119
nUpdatedDiaObjects=0,
99120
nUnassociatedDiaObjects=0)
100121

101-
matchResult = self.associate_sources(diaObjects, diaSources)
122+
if self.config.doTrailedSourceFilter:
123+
diaTrailedResult = self.trailedSourceFilter.run(diaSources, diffIm)
124+
125+
matchResult = self.associate_sources(diaObjects, diaTrailedResult.diaSources)
126+
127+
else:
128+
matchResult = self.associate_sources(diaObjects, diaSources)
102129

103130
mask = matchResult.diaSources["diaObjectId"] != 0
104131

‎python/lsst/ap/association/diaPipe.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@
2828
Currently loads directly from the Apdb rather than pre-loading.
2929
"""
3030

31+
__all__ = ("DiaPipelineConfig",
32+
"DiaPipelineTask",
33+
"DiaPipelineConnections")
34+
3135
import pandas as pd
3236

3337
import lsst.dax.apdb as daxApdb
@@ -44,10 +48,6 @@
4448
PackageAlertsTask)
4549
from lsst.ap.association.ssoAssociation import SolarSystemAssociationTask
4650

47-
__all__ = ("DiaPipelineConfig",
48-
"DiaPipelineTask",
49-
"DiaPipelineConnections")
50-
5151

5252
class DiaPipelineConnections(
5353
pipeBase.PipelineTaskConnections,
@@ -368,7 +368,7 @@ def run(self,
368368

369369
# Associate new DiaSources with existing DiaObjects.
370370
assocResults = self.associator.run(diaSourceTable,
371-
loaderResult.diaObjects)
371+
loaderResult.diaObjects, exposure=diffIm)
372372
if self.config.doSolarSystemAssociation:
373373
ssoAssocResult = self.solarSystemAssociator.run(
374374
assocResults.unAssocDiaSources,

‎python/lsst/ap/association/trailedSourceFilter.py‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# This file is part of ap_association.
2+
#
3+
# Developed for the LSST Data Management System.
4+
# This product includes software developed by the LSST Project
5+
# (https://www.lsst.org).
6+
# See the COPYRIGHT file at the top-level directory of this distribution
7+
# for details of code ownership.
8+
#
9+
# This program is free software: you can redistribute it and/or modify
10+
# it under the terms of the GNU General Public License as published by
11+
# the Free Software Foundation, either version 3 of the License, or
12+
# (at your option) any later version.
13+
#
14+
# This program is distributed in the hope that it will be useful,
15+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
# GNU General Public License for more details.
18+
#
19+
# You should have received a copy of the GNU General Public License
20+
# along with this program. If not, see <https://www.gnu.org/licenses/>.
21+
22+
__all__ = ("TrailedSourceFilterTask", "TrailedSourceFilterConfig")
23+
24+
import lsst.pex.config as pexConfig
25+
import lsst.pipe.base as pipeBase
26+
from lsst.utils.timer import timeMethod
27+
28+
29+
class TrailedSourceFilterConfig(pexConfig.Config):
30+
"""Config class for TrailedSourceFilterTask.
31+
"""
32+
maxTrailLength = pexConfig.Field(
33+
dtype=float,
34+
doc='Maximum trail length permitted is less than 10 degrees/day. '
35+
'This is a rate of 0.416 arcseconds per second. As trail length'
36+
' is measured in arcseconds, it is dependant on the length of '
37+
'the exposure.',
38+
default=0.416,
39+
)
40+
41+
42+
class TrailedSourceFilterTask(pipeBase.Task):
43+
"""Find trailed sources in DIASources. This task checks the length of
44+
trailLength in the DIASource catalog against using a given
45+
arcsecond/second rate from maxTrailLength and the exposure time
46+
to calculate the maximum allowed length before filtering.
47+
48+
"""
49+
50+
ConfigClass = TrailedSourceFilterConfig
51+
_DefaultName = "trailedSourceFilter"
52+
53+
@timeMethod
54+
def run(self, dia_sources, diffIm):
55+
"""Find trailed sources which do not meet requirements and
56+
will not be included in the diaSource catalog.
57+
58+
Parameters
59+
----------
60+
dia_sources : `pandas.DataFrame`
61+
New DIASources to be checked for trailed sources.
62+
diffIm : `lsst.afw.image.ExposureF`
63+
Difference image on which the DIASources were detected.
64+
65+
Returns
66+
-------
67+
result : `lsst.pipe.base.Struct`
68+
Results struct with components.
69+
70+
- ``dia_sources`` : DIASource table that is free from unwanted
71+
trailed sources. (`pandas.DataFrame`)
72+
73+
- ``trailed_dia_sources`` : DIASources that have trailed more
74+
than 0.416 arcseconds/second*exposure_time. (`pandas.DataFrame`)
75+
"""
76+
77+
trail_mask = self.check_dia_source_trail(dia_sources, diffIm)
78+
79+
return pipeBase.Struct(
80+
diaSources=dia_sources[~trail_mask].reset_index(drop=True),
81+
trailedDiaSources=dia_sources[trail_mask].reset_index(drop=True))
82+
83+
def check_dia_source_trail(self, dia_sources, diffIm):
84+
"""Find DiaSources that have trails.
85+
86+
Creates a mask for sources with lengths greater than 0.416
87+
arcseconds/second multiplied by the exposure time.
88+
89+
Parameters
90+
----------
91+
dia_sources : `pandas.DataFrame`
92+
Input DIASources to check for trail lengths.
93+
diffIm : `pandas.DataFrame`
94+
Difference image on which the DIASources were detected.
95+
96+
Returns
97+
-------
98+
trail_mask : `pandas.DataFrame`
99+
Boolean mask for DIASources which are greater than the
100+
cutoff length.
101+
"""
102+
diffIm_time = diffIm.getInfo().getVisitInfo().getExposureTime()
103+
trail_mask = (dia_sources.loc[:, "trailLength"].values[:] >= self.config.maxTrailLength*diffIm_time)
104+
105+
return trail_mask

‎tests/test_association_task.py‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import numpy as np
2323
import pandas as pd
2424
import unittest
25-
25+
import lsst.afw.image as afwImage
2626
import lsst.geom as geom
2727
import lsst.utils.tests
2828

@@ -46,20 +46,30 @@ def setUp(self):
4646
self.diaSources = pd.DataFrame(data=[
4747
{"ra": 0.04*idx + scatter*rng.uniform(-1, 1),
4848
"dec": 0.04*idx + scatter*rng.uniform(-1, 1),
49-
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0}
49+
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0, "trailLength": 5.5*idx}
5050
for idx in range(self.nSources)])
5151
self.diaSourceZeroScatter = pd.DataFrame(data=[
5252
{"ra": 0.04*idx,
5353
"dec": 0.04*idx,
54-
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0}
54+
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0, "trailLength": 5.5*idx}
5555
for idx in range(self.nSources)])
56+
visitInfo = afwImage.VisitInfo(exposureTime=30.0,
57+
)
58+
exposureInfo = afwImage.ExposureInfo()
59+
exposureInfo.setVisitInfo(visitInfo)
60+
maskedImage = afwImage.MaskedImageF(lsst.geom.Extent2I(64, 64))
61+
self.exposure = afwImage.ExposureF(maskedImage, exposureInfo)
5662

5763
def test_run(self):
5864
"""Test the full task by associating a set of diaSources to
5965
existing diaObjects.
6066
"""
61-
assocTask = AssociationTask()
62-
results = assocTask.run(self.diaSources, self.diaObjects)
67+
68+
config = AssociationTask.ConfigClass()
69+
config.doTrailedSourceFilter = False
70+
assocTask = AssociationTask(config=config)
71+
72+
results = assocTask.run(self.diaSources, self.diaObjects, self.exposure)
6373

6474
self.assertEqual(results.nUpdatedDiaObjects, len(self.diaObjects) - 1)
6575
self.assertEqual(results.nUnassociatedDiaObjects, 1)
@@ -75,13 +85,36 @@ def test_run(self):
7585
[0]):
7686
self.assertEqual(test_obj_id, expected_obj_id)
7787

88+
def test_run_trailed_sources(self):
89+
"""Test the full task by associating a set of diaSources to
90+
existing diaObjects when trailed sources are filtered.
91+
"""
92+
93+
assocTask = AssociationTask()
94+
95+
results = assocTask.run(self.diaSources, self.diaObjects, self.exposure)
96+
97+
self.assertEqual(results.nUpdatedDiaObjects, len(self.diaObjects) - 3)
98+
self.assertEqual(results.nUnassociatedDiaObjects, 3)
99+
self.assertEqual(len(results.matchedDiaSources),
100+
len(self.diaObjects) - 3)
101+
self.assertEqual(len(results.unAssocDiaSources), 1)
102+
for test_obj_id, expected_obj_id in zip(
103+
results.matchedDiaSources["diaObjectId"].to_numpy(),
104+
[1, 2, 3, 4]):
105+
self.assertEqual(test_obj_id, expected_obj_id)
106+
for test_obj_id, expected_obj_id in zip(
107+
results.unAssocDiaSources["diaObjectId"].to_numpy(),
108+
[0]):
109+
self.assertEqual(test_obj_id, expected_obj_id)
110+
78111
def test_run_no_existing_objects(self):
79112
"""Test the run method with a completely empty database.
80113
"""
81114
assocTask = AssociationTask()
82115
results = assocTask.run(
83116
self.diaSources,
84-
pd.DataFrame(columns=["ra", "dec", "diaObjectId"]))
117+
pd.DataFrame(columns=["ra", "dec", "diaObjectId", "trailLength"]), self.exposure)
85118
self.assertEqual(results.nUpdatedDiaObjects, 0)
86119
self.assertEqual(results.nUnassociatedDiaObjects, 0)
87120
self.assertEqual(len(results.matchedDiaSources), 0)

‎tests/test_diaPipe.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ def solarSystemAssociator_run(self, unAssocDiaSources, solarSystemObjectTable, d
121121
unAssocDiaSources=MagicMock(spec=pd.DataFrame()))
122122

123123
@lsst.utils.timer.timeMethod
124-
def associator_run(self, table, diaObjects):
124+
def associator_run(self, table, diaObjects, exposure):
125125
return lsst.pipe.base.Struct(nUpdatedDiaObjects=2, nUnassociatedDiaObjects=3,
126126
matchedDiaSources=MagicMock(spec=pd.DataFrame()),
127127
unAssocDiaSources=MagicMock(spec=pd.DataFrame()))

‎tests/test_trailedSourceFilter.py‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import unittest
2+
from lsst.ap.association import TrailedSourceFilterTask
3+
import numpy as np
4+
import pandas as pd
5+
import lsst.afw.image as afwImage
6+
import lsst.utils.tests
7+
8+
9+
class TestTrailedSourceFilterTask(unittest.TestCase):
10+
11+
def setUp(self):
12+
"""Create sets of diaSources and diaObjects.
13+
"""
14+
rng = np.random.default_rng(1234)
15+
scatter = 0.1 / 3600
16+
self.nSources = 5
17+
self.diaSources = pd.DataFrame(data=[
18+
{"ra": 0.04*idx + scatter*rng.uniform(-1, 1),
19+
"dec": 0.04*idx + scatter*rng.uniform(-1, 1),
20+
"diaSourceId": idx, "diaObjectId": 0, "trailLength": 5.5*idx}
21+
for idx in range(self.nSources)])
22+
visitInfo = afwImage.VisitInfo(exposureTime=30.0,
23+
)
24+
exposureInfo = afwImage.ExposureInfo()
25+
exposureInfo.setVisitInfo(visitInfo)
26+
maskedImage = afwImage.MaskedImageF(lsst.geom.Extent2I(64, 64))
27+
self.exposure = afwImage.ExposureF(maskedImage, exposureInfo)
28+
29+
def test_run(self):
30+
# Run with default trail length
31+
32+
trailedSourceFilterTask = TrailedSourceFilterTask()
33+
34+
results = trailedSourceFilterTask.run(self.diaSources, self.exposure)
35+
36+
self.assertEqual(len(results.diaSources),
37+
3)
38+
39+
def test_run_short_max_trail(self):
40+
# Run with low trail rate cutoff to test config settings.
41+
42+
config = TrailedSourceFilterTask.ConfigClass()
43+
config.maxTrailLength = 0.01
44+
trailedSourceFilterTask = TrailedSourceFilterTask(config=config)
45+
46+
results = trailedSourceFilterTask.run(self.diaSources, self.exposure)
47+
48+
self.assertEqual(len(results.diaSources),
49+
1)
50+
51+
def test_run_no_trails(self):
52+
# Run with long trail length rate so that no sources are filtered.
53+
54+
config = TrailedSourceFilterTask.ConfigClass()
55+
config.maxTrailLength = 10.00
56+
trailedSourceFilterTask = TrailedSourceFilterTask(config=config)
57+
58+
results = trailedSourceFilterTask.run(self.diaSources, self.exposure)
59+
60+
self.assertEqual(len(results.diaSources),
61+
5)
62+
self.assertEqual(len(results.trailedDiaSources),
63+
0)
64+
65+
def test_check_dia_source_trail(self):
66+
# Check that the task outputs the correct masks.
67+
68+
trailedSourceFilterTask = TrailedSourceFilterTask()
69+
mask = trailedSourceFilterTask.check_dia_source_trail(self.diaSources, self.exposure)
70+
71+
comparison = mask == [False, False, False, True, True]
72+
self.assertTrue(comparison.all())
73+
74+
75+
class MemoryTester(lsst.utils.tests.MemoryTestCase):
76+
pass
77+
78+
79+
def setup_module(module):
80+
lsst.utils.tests.init()
81+
82+
83+
if __name__ == "__main__":
84+
lsst.utils.tests.init()
85+
unittest.main()

0 commit comments

Comments
 (0)