@@ -45,20 +45,14 @@ class EvalCreateResponse < OpenAI::Internal::Type::BaseModel
45
45
# @return [Symbol, :eval]
46
46
required :object , const : :eval
47
47
48
- # @!attribute share_with_openai
49
- # Indicates whether the evaluation is shared with OpenAI.
50
- #
51
- # @return [Boolean]
52
- required :share_with_openai , OpenAI ::Internal ::Type ::Boolean
53
-
54
48
# @!attribute testing_criteria
55
49
# A list of testing criteria.
56
50
#
57
- # @return [Array<OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader>]
51
+ # @return [Array<OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::Python, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel >]
58
52
required :testing_criteria ,
59
53
-> { OpenAI ::Internal ::Type ::ArrayOf [ union : OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ] }
60
54
61
- # @!method initialize(id:, created_at:, data_source_config:, metadata:, name:, share_with_openai:, testing_criteria:, object: :eval)
55
+ # @!method initialize(id:, created_at:, data_source_config:, metadata:, name:, testing_criteria:, object: :eval)
62
56
# An Eval object with a data source config and testing criteria. An Eval
63
57
# represents a task to be done for your LLM integration. Like:
64
58
#
@@ -71,8 +65,7 @@ class EvalCreateResponse < OpenAI::Internal::Type::BaseModel
71
65
# @param data_source_config [OpenAI::Models::EvalCustomDataSourceConfig, OpenAI::Models::EvalStoredCompletionsDataSourceConfig]
72
66
# @param metadata [Hash{Symbol=>String}, nil]
73
67
# @param name [String]
74
- # @param share_with_openai [Boolean]
75
- # @param testing_criteria [Array<OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader>]
68
+ # @param testing_criteria [Array<OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::Python, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel>]
76
69
# @param object [Symbol, :eval]
77
70
78
71
# Configuration of data sources used in runs of the evaluation.
@@ -116,8 +109,211 @@ module TestingCriterion
116
109
# A TextSimilarityGrader object which grades text based on similarity metrics.
117
110
variant :text_similarity , -> { OpenAI ::Models ::EvalTextSimilarityGrader }
118
111
112
+ # A PythonGrader object that runs a python script on the input.
113
+ variant :python , -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::Python }
114
+
115
+ # A ScoreModelGrader object that uses a model to assign a score to the input.
116
+ variant :score_model , -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel }
117
+
118
+ class Python < OpenAI ::Internal ::Type ::BaseModel
119
+ # @!attribute name
120
+ # The name of the grader.
121
+ #
122
+ # @return [String]
123
+ required :name , String
124
+
125
+ # @!attribute source
126
+ # The source code of the python script.
127
+ #
128
+ # @return [String]
129
+ required :source , String
130
+
131
+ # @!attribute type
132
+ # The object type, which is always `python`.
133
+ #
134
+ # @return [Symbol, :python]
135
+ required :type , const : :python
136
+
137
+ # @!attribute image_tag
138
+ # The image tag to use for the python script.
139
+ #
140
+ # @return [String, nil]
141
+ optional :image_tag , String
142
+
143
+ # @!attribute pass_threshold
144
+ # The threshold for the score.
145
+ #
146
+ # @return [Float, nil]
147
+ optional :pass_threshold , Float
148
+
149
+ # @!method initialize(name:, source:, image_tag: nil, pass_threshold: nil, type: :python)
150
+ # A PythonGrader object that runs a python script on the input.
151
+ #
152
+ # @param name [String]
153
+ # @param source [String]
154
+ # @param image_tag [String]
155
+ # @param pass_threshold [Float]
156
+ # @param type [Symbol, :python]
157
+ end
158
+
159
+ class ScoreModel < OpenAI ::Internal ::Type ::BaseModel
160
+ # @!attribute input
161
+ # The input text. This may include template strings.
162
+ #
163
+ # @return [Array<OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input>]
164
+ required :input ,
165
+ -> { OpenAI ::Internal ::Type ::ArrayOf [ OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ] }
166
+
167
+ # @!attribute model
168
+ # The model to use for the evaluation.
169
+ #
170
+ # @return [String]
171
+ required :model , String
172
+
173
+ # @!attribute name
174
+ # The name of the grader.
175
+ #
176
+ # @return [String]
177
+ required :name , String
178
+
179
+ # @!attribute type
180
+ # The object type, which is always `score_model`.
181
+ #
182
+ # @return [Symbol, :score_model]
183
+ required :type , const : :score_model
184
+
185
+ # @!attribute pass_threshold
186
+ # The threshold for the score.
187
+ #
188
+ # @return [Float, nil]
189
+ optional :pass_threshold , Float
190
+
191
+ # @!attribute range
192
+ # The range of the score. Defaults to `[0, 1]`.
193
+ #
194
+ # @return [Array<Float>, nil]
195
+ optional :range , OpenAI ::Internal ::Type ::ArrayOf [ Float ]
196
+
197
+ # @!attribute sampling_params
198
+ # The sampling parameters for the model.
199
+ #
200
+ # @return [Object, nil]
201
+ optional :sampling_params , OpenAI ::Internal ::Type ::Unknown
202
+
203
+ # @!method initialize(input:, model:, name:, pass_threshold: nil, range: nil, sampling_params: nil, type: :score_model)
204
+ # A ScoreModelGrader object that uses a model to assign a score to the input.
205
+ #
206
+ # @param input [Array<OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input>]
207
+ # @param model [String]
208
+ # @param name [String]
209
+ # @param pass_threshold [Float]
210
+ # @param range [Array<Float>]
211
+ # @param sampling_params [Object]
212
+ # @param type [Symbol, :score_model]
213
+
214
+ class Input < OpenAI ::Internal ::Type ::BaseModel
215
+ # @!attribute content
216
+ # Text inputs to the model - can contain template strings.
217
+ #
218
+ # @return [String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Content::OutputText]
219
+ required :content ,
220
+ union : -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ::Content }
221
+
222
+ # @!attribute role
223
+ # The role of the message input. One of `user`, `assistant`, `system`, or
224
+ # `developer`.
225
+ #
226
+ # @return [Symbol, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Role]
227
+ required :role , enum : -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ::Role }
228
+
229
+ # @!attribute type
230
+ # The type of the message input. Always `message`.
231
+ #
232
+ # @return [Symbol, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Type, nil]
233
+ optional :type , enum : -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ::Type }
234
+
235
+ # @!method initialize(content:, role:, type: nil)
236
+ # A message input to the model with a role indicating instruction following
237
+ # hierarchy. Instructions given with the `developer` or `system` role take
238
+ # precedence over instructions given with the `user` role. Messages with the
239
+ # `assistant` role are presumed to have been generated by the model in previous
240
+ # interactions.
241
+ #
242
+ # @param content [String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Content::OutputText]
243
+ # @param role [Symbol, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Role]
244
+ # @param type [Symbol, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Type]
245
+
246
+ # Text inputs to the model - can contain template strings.
247
+ #
248
+ # @see OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input#content
249
+ module Content
250
+ extend OpenAI ::Internal ::Type ::Union
251
+
252
+ # A text input to the model.
253
+ variant String
254
+
255
+ # A text input to the model.
256
+ variant -> { OpenAI ::Models ::Responses ::ResponseInputText }
257
+
258
+ # A text output from the model.
259
+ variant -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ::Content ::OutputText }
260
+
261
+ class OutputText < OpenAI ::Internal ::Type ::BaseModel
262
+ # @!attribute text
263
+ # The text output from the model.
264
+ #
265
+ # @return [String]
266
+ required :text , String
267
+
268
+ # @!attribute type
269
+ # The type of the output text. Always `output_text`.
270
+ #
271
+ # @return [Symbol, :output_text]
272
+ required :type , const : :output_text
273
+
274
+ # @!method initialize(text:, type: :output_text)
275
+ # A text output from the model.
276
+ #
277
+ # @param text [String]
278
+ # @param type [Symbol, :output_text]
279
+ end
280
+
281
+ # @!method self.variants
282
+ # @return [Array(String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Content::OutputText)]
283
+ end
284
+
285
+ # The role of the message input. One of `user`, `assistant`, `system`, or
286
+ # `developer`.
287
+ #
288
+ # @see OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input#role
289
+ module Role
290
+ extend OpenAI ::Internal ::Type ::Enum
291
+
292
+ USER = :user
293
+ ASSISTANT = :assistant
294
+ SYSTEM = :system
295
+ DEVELOPER = :developer
296
+
297
+ # @!method self.values
298
+ # @return [Array<Symbol>]
299
+ end
300
+
301
+ # The type of the message input. Always `message`.
302
+ #
303
+ # @see OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input#type
304
+ module Type
305
+ extend OpenAI ::Internal ::Type ::Enum
306
+
307
+ MESSAGE = :message
308
+
309
+ # @!method self.values
310
+ # @return [Array<Symbol>]
311
+ end
312
+ end
313
+ end
314
+
119
315
# @!method self.variants
120
- # @return [Array(OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader)]
316
+ # @return [Array(OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::Python, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel )]
121
317
end
122
318
end
123
319
end
0 commit comments