Skip to content

Commit 40927bb

Browse files
author
Valeriy Fedyunin
authoredDec 21, 2021
Add CL1LossLayer (#511)
* Add CL1LossLayer Signed-off-by: Valeriy Fedyunin <[email protected]> * Register L1Loss in Python wrapper Signed-off-by: Valeriy Fedyunin <[email protected]>
·
NeoML-neoml-2.0.212-no-jit_2.0.212.1NeoML-MKL2022_2.0.34.1
1 parent a6f5dc7 commit 40927bb

File tree

17 files changed

+273
-2
lines changed

17 files changed

+273
-2
lines changed
 

‎NeoML/Python/neoml/Dnn/Loss.py

+44
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,50 @@ def __init__(self, input_layers, loss_weight=1.0, name=None):
266266
# ----------------------------------------------------------------------------------------------------------------------
267267

268268

269+
class L1Loss(Loss):
270+
"""The layer that calculates the loss function equal to L1 distance
271+
between the objects in the network response and the correct objects.
272+
273+
:param input_layers: The input layers to be connected.
274+
The integer in each tuple specifies the number of the output.
275+
If not set, the first output will be used.
276+
:type input_layers: list of object, tuple(object, int)
277+
:param loss_weight: The multiplier for the loss function value during training.
278+
:type loss_weight: float, default=1.0
279+
:param name: The layer name.
280+
:type name: str, default=None
281+
282+
.. rubric:: Layer inputs:
283+
284+
(1) the network response for which you are calculating the loss.
285+
286+
(2) the correct objects.
287+
288+
(3) (optional): the objects' weights.
289+
290+
The dimensions of all inputs are the same:
291+
292+
- **BatchLength** * **BatchWidth** * **ListSize** - the number of objects
293+
- **Height** * **Width** * **Depth** * **Channels** - the object size
294+
295+
.. rubric:: Layer outputs:
296+
297+
The layer has no output.
298+
"""
299+
def __init__(self, input_layers, loss_weight=1.0, name=None):
300+
301+
if type(input_layers) is PythonWrapper.L1Loss:
302+
super().__init__(input_layers)
303+
return
304+
305+
layers, outputs = check_input_layers(input_layers, (2, 3))
306+
307+
internal = PythonWrapper.L1Loss(str(name), layers, outputs, float(loss_weight))
308+
super().__init__(internal)
309+
310+
# ----------------------------------------------------------------------------------------------------------------------
311+
312+
269313
class HingeLoss(Loss):
270314
"""The layer that calculates hinge loss function for binary classification:
271315
:math:`f(x) = \max(0, 1 - x * y)`, where

‎NeoML/Python/neoml/Dnn/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from .ImageConversion import ImageResize, PixelToImage, ImageToPixel
2727
from .IndRnn import IndRnn
2828
from .Irnn import Irnn
29-
from .Loss import CrossEntropyLoss, BinaryCrossEntropyLoss, EuclideanLoss, HingeLoss, SquaredHingeLoss, FocalLoss, BinaryFocalLoss, CenterLoss, MultiHingeLoss, MultiSquaredHingeLoss, CustomLoss, CustomLossCalculatorBase, call_loss_calculator
29+
from .Loss import CrossEntropyLoss, BinaryCrossEntropyLoss, EuclideanLoss, L1Loss, HingeLoss, SquaredHingeLoss, FocalLoss, BinaryFocalLoss, CenterLoss, MultiHingeLoss, MultiSquaredHingeLoss, CustomLoss, CustomLossCalculatorBase, call_loss_calculator
3030
from .Lrn import Lrn
3131
from .Lstm import Lstm
3232
from .MatrixMultiplication import MatrixMultiplication

‎NeoML/Python/src/PyDnn.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ REGISTER_NEOML_PYLAYER( "FullyConnectedSource", "FmlCnnFullyConnectedSourceLayer
100100
REGISTER_NEOML_PYLAYER_EX( "Loss", "CrossEntropyLoss", "FmlCnnCrossEntropyLossLayer" )
101101
REGISTER_NEOML_PYLAYER_EX( "Loss", "BinaryCrossEntropyLoss", "FmlCnnBinaryCrossEntropyLossLayer" )
102102
REGISTER_NEOML_PYLAYER_EX( "Loss", "EuclideanLoss", "FmlCnnEuclideanLossLayer" )
103+
REGISTER_NEOML_PYLAYER_EX( "Loss", "L1Loss", "NeoMLDnnL1LossLayer" )
103104
REGISTER_NEOML_PYLAYER_EX( "Loss", "HingeLoss", "FmlCnnHingeLossLayer" )
104105
REGISTER_NEOML_PYLAYER_EX( "Loss", "SquaredHingeLoss", "FmlCnnSquaredHingeLossLayer" )
105106
REGISTER_NEOML_PYLAYER_EX( "Loss", "CustomLoss", "NeoMLCustomLossLayer" )

‎NeoML/Python/src/PyLossLayer.cpp

+43-1
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,21 @@ class CPyEuclideanLossLayer : public CPyLossLayer {
7070

7171
//------------------------------------------------------------------------------------------------------------
7272

73+
class CPyL1LossLayer : public CPyLossLayer {
74+
public:
75+
explicit CPyL1LossLayer( CL1LossLayer& layer, CPyMathEngineOwner& mathEngineOwner ) :
76+
CPyLossLayer( layer, mathEngineOwner ) {}
77+
78+
py::object CreatePythonObject() const
79+
{
80+
py::object pyModule = py::module::import( "neoml.Dnn" );
81+
py::object pyConstructor = pyModule.attr( "L1Loss" );
82+
return pyConstructor( py::cast(this) );
83+
}
84+
};
85+
86+
//------------------------------------------------------------------------------------------------------------
87+
7388
class CPyHingeLossLayer : public CPyLossLayer {
7489
public:
7590
explicit CPyHingeLossLayer( CHingeLossLayer& layer, CPyMathEngineOwner& mathEngineOwner ) :
@@ -288,6 +303,33 @@ void InitializeLossLayer( py::module& m )
288303
}) )
289304
;
290305

306+
//------------------------------------------------------------------------------------------------------------
307+
308+
py::class_<CPyL1LossLayer, CPyLossLayer>(m, "L1Loss")
309+
.def( py::init([]( const CPyLayer& layer )
310+
{
311+
return CPyL1LossLayer( *layer.Layer<CL1LossLayer>(), layer.MathEngineOwner() );
312+
}) )
313+
.def( py::init([]( const std::string& name, const py::list& layers, const py::list& outputs, float lossWeight )
314+
{
315+
py::gil_scoped_release release;
316+
CDnn& dnn = layers[0].cast<CPyLayer>().Dnn();
317+
IMathEngine& mathEngine = dnn.GetMathEngine();
318+
319+
CPtr<CL1LossLayer> loss = new CL1LossLayer( mathEngine );
320+
loss->SetLossWeight( lossWeight );
321+
loss->SetName( FindFreeLayerName( dnn, "L1Loss", name ).c_str() );
322+
dnn.AddLayer( *loss );
323+
loss->Connect( 0, layers[0].cast<CPyLayer>().BaseLayer(), outputs[0].cast<int>() );
324+
loss->Connect( 1, layers[1].cast<CPyLayer>().BaseLayer(), outputs[1].cast<int>() );
325+
if( layers.size() == 3 ) {
326+
loss->Connect( 2, layers[2].cast<CPyLayer>().BaseLayer(), outputs[2].cast<int>() );
327+
}
328+
329+
return CPyL1LossLayer( *loss, layers[0].cast<CPyLayer>().MathEngineOwner() );
330+
}) )
331+
;
332+
291333
//------------------------------------------------------------------------------------------------------------
292334

293335
py::class_<CPyHingeLossLayer, CPyLossLayer>(m, "HingeLoss")
@@ -303,7 +345,7 @@ void InitializeLossLayer( py::module& m )
303345

304346
CPtr<CHingeLossLayer> loss = new CHingeLossLayer( mathEngine );
305347
loss->SetLossWeight( lossWeight );
306-
loss->SetName( FindFreeLayerName( dnn, "EuclideanLoss", name ).c_str() );
348+
loss->SetName( FindFreeLayerName( dnn, "HingeLoss", name ).c_str() );
307349
dnn.AddLayer( *loss );
308350
loss->Connect( 0, layers[0].cast<CPyLayer>().BaseLayer(), outputs[0].cast<int>() );
309351
loss->Connect( 1, layers[1].cast<CPyLayer>().BaseLayer(), outputs[1].cast<int>() );

‎NeoML/Python/tests.py

+3
Original file line numberDiff line numberDiff line change
@@ -2293,6 +2293,9 @@ def test_binary_cross_entropy_loss(self):
22932293
def test_euclidean_loss(self):
22942294
self._test_loss('EuclideanLoss', dict(loss_weight=7.7), last_loss=0.)
22952295

2296+
def test_l1_loss(self):
2297+
self._test_loss('L1Loss', dict(loss_weight=7.7), last_loss=0.)
2298+
22962299
def test_hinge_loss(self):
22972300
self._test_loss('HingeLoss', dict(loss_weight=7.7), last_loss=0.)
22982301

‎NeoML/docs/en/API/NN/LossLayers/L1LossLayer.md

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# CL1LossLayer Class
2+
3+
<!-- TOC -->
4+
5+
- [CL1LossLayer Class](#cl1losslayer-class)
6+
- [Settings](#settings)
7+
- [Trainable parameters](#trainable-parameters)
8+
- [Inputs](#inputs)
9+
- [Outputs](#outputs)
10+
- [Getting the value of the loss function](#getting-the-value-of-the-loss-function)
11+
12+
<!-- /TOC -->
13+
14+
This class implements a layer that calculates a loss function equal to the l1 distance between the objects from the network response and the objects of the correct answer.
15+
16+
## Settings
17+
18+
The layer has no settings.
19+
20+
## Trainable parameters
21+
22+
The layer has no trainable parameters.
23+
24+
## Inputs
25+
26+
The layer may have 2 to 3 inputs:
27+
28+
1. The network output for which you are calculating the loss function. It contains `BatchLength * BatchWidth * ListSize` objects, each of `Height * Width * Depth * Channels` size.
29+
2. A blob of the same size as the first input, containing the correct class objects. The loss function will calculate the L1 distance between the first and the second input.
30+
3. *[Optional]* The objects' weights. This blob should have the same dimensions as the first input.
31+
32+
## Outputs
33+
34+
This layer has no output.
35+
36+
### Getting the value of the loss function
37+
38+
```c++
39+
float GetLastLoss() const;
40+
```
41+
42+
Use this method to get the value of the loss function calculated on the network's last run.

‎NeoML/docs/en/API/NN/LossLayers/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@
1212
- [CFocalLossLayer](FocalLossLayer.md) - focal loss (modified cross-entropy)
1313
- For regression:
1414
- [CEuclideanLossLayer](EuclideanLossLayer.md) - Euclidean distance
15+
- [CL1LossLayer](L1LossLayer.md) - L1 distance
1516
- Additionally:
1617
- [CCenterLossLayer](CenterLossLayer.md) - the auxiliary *center loss* function that penalizes large variance inside a class

‎NeoML/docs/en/API/NN/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ delete gpuMathEngine;
297297
- [CFocalLossLayer](LossLayers/FocalLossLayer.md) - focal loss function (modified cross-entropy)
298298
- For regression:
299299
- [CEuclideanLossLayer](LossLayers/EuclideanLossLayer.md) - Euclidean distance
300+
- [CL1LossLayer](LossLayers/L1LossLayer.md) - L1 distance
300301
- Additionally:
301302
- [CCenterLossLayer](LossLayers/CenterLossLayer.md) - the auxiliary *center loss* function that penalizes large variance inside a class
302303
- Working with discrete features:

‎NeoML/docs/ru/API/NN/LossLayers/L1LossLayer.md

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Класс CL1LossLayer
2+
3+
<!-- TOC -->
4+
5+
- [Класс CL1LossLayer](#класс-cl1losslayer)
6+
- [Настройки](#настройки)
7+
- [Обучаемые параметры](#обучаемые-параметры)
8+
- [Входы](#входы)
9+
- [Выходы](#выходы)
10+
- [Значение функции потерь](#значение-функции-потерь)
11+
12+
<!-- /TOC -->
13+
14+
Класс реализует слой, вычисляющий функцию потерь, равную L1 расстоянию между объектами из ответа сети и объектами из правильных ответов.
15+
16+
## Настройки
17+
18+
Слой не имеет настроек.
19+
20+
## Обучаемые параметры
21+
22+
Слой не имеет обучаемых параметров.
23+
24+
## Входы
25+
26+
Слой имеет 2 или 3 входа:
27+
28+
1. Ответы сети, на которых необходимо вычислить функцию потерь. Содержит `BatchLength * BatchWidth * ListSize` объектов размера `Height * Width * Depth * Channels`.
29+
2. Правильные ответы в виде блоба того же размера, что и блоб у первого входа. Функция потерь равняется расстояниям между объектами из первого входа и второго.
30+
3. *[Опционально]* Веса объектов. Блоб этого входа должен иметь те же `BatchLength`, `BatchWidth` и `ListSize`, что и у первого входа. `Height`, `Width`, `Depth` и `Channels` должны быть равны `1`.
31+
32+
## Выходы
33+
34+
Слой не имеет выходов.
35+
36+
### Значение функции потерь
37+
38+
```c++
39+
float GetLastLoss() const;
40+
```
41+
42+
Получение значения функции потерь на последнем запуске сети.

‎NeoML/docs/ru/API/NN/LossLayers/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@
1212
- [CFocalLossLayer](FocalLossLayer.md) - функция `Focal` (модифицированная кросс-энтропия);
1313
- Регрессия:
1414
- [CEuclideanLossLayer](EuclideanLossLayer.md) - евклидово расстояние;
15+
- [CL1LossLayer](L1LossLayer.md) - L1 расстояние;
1516
- Дополнительно:
1617
- [CCenterLossLayer](CenterLossLayer.md) - вспомогательная функция `Center`, штрафующая дисперсию внутри классов.

‎NeoML/docs/ru/API/NN/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,7 @@ delete gpuMathEngine;
296296
- [CFocalLossLayer](LossLayers/FocalLossLayer.md) - функция `Focal` (модифицированная кросс-энтропия)
297297
- Регрессия:
298298
- [CEuclideanLossLayer](LossLayers/EuclideanLossLayer.md) - евклидово расстояние
299+
- [CL1LossLayer](LossLayers/L1ossLayer.md) - L1 расстояние
299300
- Дополнительно:
300301
- [CCenterLossLayer](LossLayers/CenterLossLayer.md) - вспомогательная функция `Center`, штрафующая дисперсию внутри классов
301302
- Работа с дискретными признаками:

‎NeoML/include/NeoML/Dnn/Layers/LossLayer.h

+19
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,25 @@ NEOML_API CLayerWrapper<CEuclideanLossLayer> EuclideanLoss( float lossWeight = 1
219219

220220
///////////////////////////////////////////////////////////////////////////////////
221221

222+
// CL1LossLayer implements a layer that estimates the loss value as abs(result - standard)
223+
// The layer has two inputs: #0 - result, #1 - standard
224+
class NEOML_API CL1LossLayer : public CLossLayer {
225+
NEOML_DNN_LAYER( CL1LossLayer )
226+
public:
227+
explicit CL1LossLayer( IMathEngine& mathEngine ) : CLossLayer( mathEngine, "CL1LossLayer" ) {}
228+
229+
void Serialize( CArchive& archive ) override;
230+
231+
protected:
232+
void Reshape() override;
233+
void BatchCalculateLossAndGradient( int batchSize, CConstFloatHandle data, int vectorSize, CConstFloatHandle label,
234+
int labelSize, CFloatHandle lossValue, CFloatHandle lossGradient ) override;
235+
};
236+
237+
NEOML_API CLayerWrapper<CL1LossLayer> L1Loss( float lossWeight = 1.0f );
238+
239+
///////////////////////////////////////////////////////////////////////////////////
240+
222241
// CHingeLossLayer implements a layer that estimates the loss value as max(0, 1 - result * standard)
223242
// The layer has two inputs: #0 - result, #1 - standard
224243
// The standard contains the data in the format: 1 for objects that belong to the class, -1 for the rest

‎NeoML/src/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ set(NeoML_SOURCES
6363
Dnn/Layers/ImageResizeLayer.cpp
6464
Dnn/Layers/IndRnnLayer.cpp
6565
Dnn/Layers/IrnnLayer.cpp
66+
Dnn/Layers/L1LossLayer.cpp
6667
Dnn/Layers/LrnLayer.cpp
6768
Dnn/Layers/LstmLayer.cpp
6869
Dnn/Layers/MatrixMultiplicationLayer.cpp

‎NeoML/src/Dnn/Dnn.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,7 @@ REGISTER_NEOML_LAYER( CCastLayer, "NeoMLDnnCastLayer" )
340340
REGISTER_NEOML_LAYER( CDataLayer, "NeoMLDnnDataLayer" )
341341
REGISTER_NEOML_LAYER( CTransformerEncoderLayer, "NeoMLDnnTransformerEncoderLayer" )
342342
REGISTER_NEOML_LAYER( CBertConvLayer, "NeoMLDnnBertConvLayer" )
343+
REGISTER_NEOML_LAYER( CL1LossLayer, "NeoMLDnnL1LossLayer" )
343344

344345
}
345346

‎NeoML/src/Dnn/Layers/L1LossLayer.cpp

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/* Copyright © 2017-2021 ABBYY Production LLC
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
--------------------------------------------------------------------------------------------------------------*/
15+
16+
#include <common.h>
17+
#pragma hdrstop
18+
19+
#include <NeoML/Dnn/Layers/LossLayer.h>
20+
#include <NeoMathEngine/NeoMathEngine.h>
21+
22+
namespace NeoML {
23+
24+
///////////////////////////////////////////////////////////////////////////////////
25+
// CL1LossLayer
26+
27+
void CL1LossLayer::Reshape()
28+
{
29+
CLossLayer::Reshape();
30+
CheckArchitecture( inputDescs[1].GetDataType() == CT_Float, GetName(), "labels must be CT_Float" );
31+
CheckArchitecture( inputDescs[0].ObjectSize() == inputDescs[1].ObjectSize(), GetName(),
32+
"the labels dimensions should be equal to the first input dimensions" );
33+
}
34+
35+
void CL1LossLayer::BatchCalculateLossAndGradient( int batchSize, CConstFloatHandle data, int vectorSize,
36+
CConstFloatHandle label, int /* labelSize */, CFloatHandle lossValue, CFloatHandle lossGradient )
37+
{
38+
int totalSize = batchSize * vectorSize;
39+
40+
CFloatHandleStackVar temp( MathEngine(), totalSize );
41+
42+
MathEngine().VectorSub( data, label, temp, totalSize );
43+
44+
if( !lossGradient.IsNull() ) {
45+
CFloatHandleStackVar ones( MathEngine(), totalSize );
46+
MathEngine().VectorFill( ones.GetHandle(), 1.f, totalSize );
47+
48+
MathEngine().VectorAbsDiff( temp, ones, lossGradient, totalSize );
49+
}
50+
51+
MathEngine().VectorAbs( temp, temp, totalSize );
52+
MathEngine().SumMatrixColumns( lossValue, temp, batchSize, vectorSize );
53+
}
54+
55+
static const int L1LossLayerVersion = 0;
56+
57+
void CL1LossLayer::Serialize( CArchive& archive )
58+
{
59+
archive.SerializeVersion( L1LossLayerVersion );
60+
CLossLayer::Serialize( archive );
61+
}
62+
63+
CLayerWrapper<CL1LossLayer> L1Loss( float lossWeight )
64+
{
65+
return CLayerWrapper<CL1LossLayer>( "L1Loss", [=]( CL1LossLayer* result ) {
66+
result->SetLossWeight( lossWeight );
67+
} );
68+
}
69+
70+
} // namespace NeoML

‎NeoML/test/data/LayersSerializationTestData/NeoMLDnnL1LossLayer.arch

76 Bytes
Binary file not shown.

‎NeoML/test/src/DnnLayersSerializationTest.cpp

+2
Original file line numberDiff line numberDiff line change
@@ -683,6 +683,7 @@ GTEST_TEST( SerializeToFile, LossLayerSerialization )
683683
serializeToFile<CCrfInternalLossLayer>( "FmlCnnCrfInternalLossLayer" );
684684
serializeToFile<CMultiHingeLossLayer>( "FmlCnnMultyHingeLossLayer" );
685685
serializeToFile<CMultiSquaredHingeLossLayer>( "FmlCnnMultySquaredHingeLossLayer" );
686+
serializeToFile<CL1LossLayer>( "NeoMLDnnL1LossLayer" );
686687
}
687688

688689
#endif
@@ -701,6 +702,7 @@ GTEST_TEST( SerializeFromFile, LossLayerSerialization )
701702
checkSerializeLayer<CLossLayer>( "FmlCnnCrfInternalLossLayer" );
702703
checkSerializeLayer<CLossLayer>( "FmlCnnMultyHingeLossLayer" );
703704
checkSerializeLayer<CLossLayer>( "FmlCnnMultySquaredHingeLossLayer" );
705+
checkSerializeLayer<CLossLayer>( "NeoMLDnnL1LossLayer" );
704706
}
705707

706708
// ====================================================================================================================

0 commit comments

Comments
 (0)
Please sign in to comment.