Skip to content

Commit 725e85e

Browse files
dskloetcapp-token-issuer-data-feeds[bot]
dskloetc
and
app-token-issuer-data-feeds[bot]
authored
OPDATA-3889: FTSE sFTP EA (#4036)
* Chray's changes * David's changes * Remove from readme blacklist * Use options to restrict instrument parameter * Remove SftpTransport.endpointName * Add config SFTP_READY_TIMEOUT_MS --------- Co-authored-by: app-token-issuer-data-feeds[bot] <134377064+app-token-issuer-data-feeds[bot]@users.noreply.github.com>
1 parent 58e6876 commit 725e85e

File tree

12 files changed

+635
-32
lines changed

12 files changed

+635
-32
lines changed

‎.changeset/warm-hornets-compete.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@chainlink/ftse-sftp-adapter': major
3+
---
4+
5+
Adding Downloading and parsing logic for russell and ftse csv files from ftse sftp server

‎packages/scripts/src/generate-readme/readmeBlacklist.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@
6464
"xsushi-price",
6565
"coinbase-prime",
6666
"harris-and-trotter",
67-
"nav-consulting",
68-
"ftse-sftp"
67+
"nav-consulting"
6968
]
7069
}

‎packages/sources/ftse-sftp/src/config/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ export const config = new AdapterConfig({
2424
sensitive: true,
2525
required: true,
2626
},
27+
SFTP_READY_TIMEOUT_MS: {
28+
description: 'How long (in milliseconds) to wait for the SSH handshake to complete',
29+
type: 'number',
30+
default: 30000,
31+
},
2732
BACKGROUND_EXECUTE_MS: {
2833
description:
2934
'The amount of time the background execute should sleep before performing the next request',

‎packages/sources/ftse-sftp/src/endpoint/index.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1 @@
1-
import * as sftp from './sftp'
2-
3-
export { sftp }
1+
export { endpoint as sftp } from './sftp'

‎packages/sources/ftse-sftp/src/endpoint/sftp.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ export const inputParameters = new InputParameters(
1111
required: true,
1212
type: 'string',
1313
description: 'Abstract identifier of the index to fetch the data for',
14+
options: ['FTSE100INDEX', 'Russell1000INDEX', 'Russell2000INDEX', 'Russell3000INDEX'],
1415
},
1516
},
1617
[
@@ -29,7 +30,7 @@ export const inputParameters = new InputParameters(
2930
],
3031
)
3132

32-
export type TInputParameters = typeof inputParameters.definition
33+
export type Instrument = (typeof inputParameters.validated)['instrument']
3334

3435
/**
3536
* Union type for all possible response data structures
@@ -41,14 +42,15 @@ export type BaseEndpointTypes = {
4142
Response: {
4243
Result: number
4344
Data: {
45+
filename: string
4446
result: IndexResponseData
4547
}
4648
}
4749
Settings: typeof config.settings
4850
}
4951

5052
export const endpoint = new AdapterEndpoint({
51-
name: 'ftse_sftp',
53+
name: 'sftp',
5254
transport: sftpTransport,
5355
inputParameters,
5456
})

‎packages/sources/ftse-sftp/src/index.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,13 @@
1-
export * from './parsing'
1+
import { expose, ServerInstance } from '@chainlink/external-adapter-framework'
2+
import { Adapter } from '@chainlink/external-adapter-framework/adapter'
3+
import { config } from './config'
4+
import { sftp } from './endpoint'
5+
6+
export const adapter = new Adapter({
7+
defaultEndpoint: sftp.name,
8+
name: 'FTSE_SFTP',
9+
config,
10+
endpoints: [sftp],
11+
})
12+
13+
export const server = (): Promise<ServerInstance | undefined> => expose(adapter)

‎packages/sources/ftse-sftp/src/transport/constants.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import { Instrument } from '../endpoint/sftp'
2+
3+
export const FTSE100INDEX: Instrument = 'FTSE100INDEX'
4+
export const RUSSELL_1000_INDEX: Instrument = 'Russell1000INDEX'
5+
export const RUSSELL_2000_INDEX: Instrument = 'Russell2000INDEX'
6+
export const RUSSELL_3000_INDEX: Instrument = 'Russell3000INDEX'
7+
8+
export const instrumentToDirectoryMap: Record<Instrument, string> = {
9+
[FTSE100INDEX]: '/data/valuation/uk_all_share/',
10+
[RUSSELL_1000_INDEX]:
11+
'/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/',
12+
[RUSSELL_2000_INDEX]:
13+
'/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/',
14+
[RUSSELL_3000_INDEX]:
15+
'/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/',
16+
}
17+
18+
export const instrumentToFileRegexMap: Record<Instrument, RegExp> = {
19+
[FTSE100INDEX]: /^ukallv\d{4}\.csv$/,
20+
[RUSSELL_1000_INDEX]: /^daily_values_russell_\d{6}\.CSV$/,
21+
[RUSSELL_2000_INDEX]: /^daily_values_russell_\d{6}\.CSV$/,
22+
[RUSSELL_3000_INDEX]: /^daily_values_russell_\d{6}\.CSV$/,
23+
}

‎packages/sources/ftse-sftp/src/transport/sftp.ts‎

Lines changed: 102 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,23 @@
11
import { EndpointContext } from '@chainlink/external-adapter-framework/adapter'
22
import { TransportDependencies } from '@chainlink/external-adapter-framework/transports'
33
import { SubscriptionTransport } from '@chainlink/external-adapter-framework/transports/abstract/subscription'
4-
import { sleep } from '@chainlink/external-adapter-framework/util'
5-
import SftpClient from 'ssh2-sftp-client'
6-
import { BaseEndpointTypes } from '../endpoint/sftp'
4+
import { AdapterResponse, makeLogger, sleep } from '@chainlink/external-adapter-framework/util'
5+
import { AdapterInputError } from '@chainlink/external-adapter-framework/validation/error'
6+
import { ConnectOptions } from 'ssh2-sftp-client'
7+
import { BaseEndpointTypes, IndexResponseData, inputParameters, Instrument } from '../endpoint/sftp'
8+
import { CSVParserFactory } from '../parsing/factory'
9+
import { instrumentToDirectoryMap, instrumentToFileRegexMap } from './constants'
10+
import { getFileContentsFromFileRegex } from './utils'
11+
12+
const logger = makeLogger('FTSE SFTP Adapter')
13+
14+
type RequestParams = typeof inputParameters.validated
715

816
export class SftpTransport extends SubscriptionTransport<BaseEndpointTypes> {
917
config!: BaseEndpointTypes['Settings']
10-
endpointName!: string
11-
sftpClient: SftpClient
1218

1319
constructor() {
1420
super()
15-
this.sftpClient = new SftpClient()
1621
}
1722

1823
async initialize(
@@ -23,13 +28,102 @@ export class SftpTransport extends SubscriptionTransport<BaseEndpointTypes> {
2328
): Promise<void> {
2429
await super.initialize(dependencies, adapterSettings, endpointName, transportName)
2530
this.config = adapterSettings
26-
this.endpointName = endpointName
2731
}
2832

29-
async backgroundHandler(context: EndpointContext<BaseEndpointTypes>): Promise<void> {
33+
async backgroundHandler(
34+
context: EndpointContext<BaseEndpointTypes>,
35+
entries: RequestParams[],
36+
): Promise<void> {
37+
await Promise.all(entries.map(async (param) => this.handleRequest(param)))
3038
await sleep(context.adapterSettings.BACKGROUND_EXECUTE_MS)
3139
}
3240

41+
async handleRequest(param: RequestParams) {
42+
let response: AdapterResponse<BaseEndpointTypes['Response']>
43+
try {
44+
response = await this._handleRequest(param)
45+
} catch (e) {
46+
const errorMessage = e instanceof Error ? e.message : 'Unknown error occurred'
47+
response = {
48+
statusCode: 502,
49+
errorMessage,
50+
timestamps: {
51+
providerDataRequestedUnixMs: 0,
52+
providerDataReceivedUnixMs: 0,
53+
providerIndicatedTimeUnixMs: undefined,
54+
},
55+
}
56+
}
57+
58+
await this.responseCache.write(this.name, [{ params: param, response }])
59+
}
60+
61+
async _handleRequest(
62+
param: RequestParams,
63+
): Promise<AdapterResponse<BaseEndpointTypes['Response']>> {
64+
const providerDataRequestedUnixMs = Date.now()
65+
66+
const { filename, result, parsedData } = await this.tryDownloadAndParseFile(param.instrument)
67+
68+
logger.debug(`Successfully processed data for instrument: ${param.instrument}`)
69+
return {
70+
data: {
71+
filename,
72+
result: parsedData,
73+
},
74+
statusCode: 200,
75+
result,
76+
timestamps: {
77+
providerDataRequestedUnixMs,
78+
providerDataReceivedUnixMs: Date.now(),
79+
providerIndicatedTimeUnixMs: undefined,
80+
},
81+
}
82+
}
83+
84+
private async tryDownloadAndParseFile(instrument: Instrument): Promise<{
85+
filename: string
86+
result: number
87+
parsedData: IndexResponseData
88+
}> {
89+
const connectOptions: ConnectOptions = {
90+
host: this.config.SFTP_HOST,
91+
port: this.config.SFTP_PORT,
92+
username: this.config.SFTP_USERNAME,
93+
password: this.config.SFTP_PASSWORD,
94+
readyTimeout: this.config.SFTP_READY_TIMEOUT_MS,
95+
}
96+
97+
const directory = instrumentToDirectoryMap[instrument]
98+
const filenameRegex = instrumentToFileRegexMap[instrument]
99+
100+
const { filename, fileContent } = await getFileContentsFromFileRegex({
101+
connectOptions,
102+
directory,
103+
filenameRegex,
104+
})
105+
106+
// we need latin1 here because the file contains special characters like "®"
107+
const csvContent = fileContent.toString('latin1')
108+
109+
const parser = CSVParserFactory.detectParserByInstrument(instrument)
110+
111+
if (!parser) {
112+
throw new AdapterInputError({
113+
statusCode: 500,
114+
message: `Parser initialization failed for instrument: ${instrument}`,
115+
})
116+
}
117+
118+
const { result, parsedData } = await parser.parse(csvContent)
119+
120+
return {
121+
filename,
122+
result,
123+
parsedData: parsedData as IndexResponseData,
124+
}
125+
}
126+
33127
getSubscriptionTtlFromConfig(adapterSettings: BaseEndpointTypes['Settings']): number {
34128
return adapterSettings.BACKGROUND_EXECUTE_MS || 60000
35129
}

‎packages/sources/ftse-sftp/test/fixtures/index.ts‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import fs from 'fs'
22
import path from 'path'
3+
import { FileInfo } from 'ssh2-sftp-client'
34

45
// Helper function to read fixture files
56
export function readFixtureFile(filename: string): string {
@@ -24,23 +25,26 @@ export const expectedRussellData = {
2425
close: 3547.4,
2526
}
2627

27-
// Test data generation helpers
28-
export const createFTSETestData = (dataRows: string[]): string => {
29-
const header = `02/09/2025 (C) FTSE International Limited 2025. All Rights Reserved
30-
FTSE UK All-Share Indices Valuation Service
28+
export const ftseFilename = 'ukallv0209.csv'
29+
export const russellFilename = 'daily_values_russell_250827.CSV'
3130

32-
Index Code,Index/Sector Name,Number of Constituents,Index Base Currency,USD Index,GBP Index,EUR Index,JPY Index,AUD Index,CNY Index,HKD Index,CAD Index,LOC Index,Base Currency (GBP) Index,USD TRI,GBP TRI,EUR TRI,JPY TRI,AUD TRI,CNY TRI,HKD TRI,CAD TRI,LOC TRI,Base Currency (GBP) TRI,Mkt Cap (USD),Mkt Cap (GBP),Mkt Cap (EUR),Mkt Cap (JPY),Mkt Cap (AUD),Mkt Cap (CNY),Mkt Cap (HKD),Mkt Cap (CAD),Mkt Cap (LOC),Mkt Cap Base Currency (GBP),XD Adjustment (YTD),Dividend Yield`
31+
export const ftseDirectory = '/data/valuation/uk_all_share/'
32+
export const russellDirectory =
33+
'/data/Returns_and_Values/Russell_US_Indexes_Daily_Index_Values_Real_Time_TXT/'
3334

34-
return header + '\n' + dataRows.join('\n')
35+
export const fileContents: Record<string, string> = {
36+
[path.join(ftseDirectory, ftseFilename)]: ftseCsvFixture,
37+
[path.join(russellDirectory, russellFilename)]: russellCsvFixture,
3538
}
3639

37-
export const createRussellTestData = (dataRows: string[]): string => {
38-
const header = `"Daily Values",,,,,,,,,,,,,,
39-
,,,,,,,,,,,,,,
40-
,,,,,,,,,,,,,,
41-
"As of August 27, 2025",,,,,,,,"Last 5 Trading Days",,,,"1 Year Ending",,
42-
,,,,,,,,"Closing Values",,,,"Closing Values",,
43-
,"Open","High","Low","Close","Net Chg","% Chg","High","Low","Net Chg","% Chg","High","Low","Net Chg","% Chg"`
44-
45-
return header + '\n' + dataRows.join('\n')
46-
}
40+
export const directoryListings = {
41+
[ftseDirectory]: [
42+
'vall_icb2302.csv',
43+
'vall1809.csv',
44+
'valllst.csv',
45+
ftseFilename,
46+
'ukallvlst.csv',
47+
'vall_icb2302_v1.csv',
48+
].map((name) => ({ name })),
49+
[russellDirectory]: ['history', russellFilename].map((name) => ({ name })),
50+
} as unknown as Record<string, FileInfo[]>

‎packages/sources/ftse-sftp/test/integration/__snapshots__/adapter.test.ts.snap‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Jest Snapshot v1, https://goo.gl/fbAQLP
2+
3+
exports[`execute sftp endpoint should return error for unknown instrument 1`] = `
4+
{
5+
"error": {
6+
"message": "[Param: instrument] input is not one of valid options (FTSE100INDEX,Russell1000INDEX,Russell2000INDEX,Russell3000INDEX)",
7+
"name": "AdapterError",
8+
},
9+
"status": "errored",
10+
"statusCode": 400,
11+
}
12+
`;
13+
14+
exports[`execute sftp endpoint should return success for FTSE100INDEX 1`] = `
15+
{
16+
"data": {
17+
"filename": "ukallv0209.csv",
18+
"result": {
19+
"gbpIndex": 9116.68749114,
20+
"indexBaseCurrency": "GBP",
21+
"indexCode": "UKX",
22+
"indexSectorName": "FTSE 100 Index",
23+
"numberOfConstituents": 100,
24+
},
25+
},
26+
"result": 9116.68749114,
27+
"statusCode": 200,
28+
"timestamps": {
29+
"providerDataReceivedUnixMs": 978347471111,
30+
"providerDataRequestedUnixMs": 978347471111,
31+
},
32+
}
33+
`;
34+
35+
exports[`execute sftp endpoint should return success for Russell1000INDEX 1`] = `
36+
{
37+
"data": {
38+
"filename": "daily_values_russell_250827.CSV",
39+
"result": {
40+
"close": 3547.4,
41+
"indexName": "Russell 1000® Index",
42+
},
43+
},
44+
"result": 3547.4,
45+
"statusCode": 200,
46+
"timestamps": {
47+
"providerDataReceivedUnixMs": 978347471111,
48+
"providerDataRequestedUnixMs": 978347471111,
49+
},
50+
}
51+
`;
52+
53+
exports[`execute sftp endpoint should return success for Russell2000INDEX 1`] = `
54+
{
55+
"data": {
56+
"filename": "daily_values_russell_250827.CSV",
57+
"result": {
58+
"close": 2373.8,
59+
"indexName": "Russell 2000® Index",
60+
},
61+
},
62+
"result": 2373.8,
63+
"statusCode": 200,
64+
"timestamps": {
65+
"providerDataReceivedUnixMs": 978347471111,
66+
"providerDataRequestedUnixMs": 978347471111,
67+
},
68+
}
69+
`;
70+
71+
exports[`execute sftp endpoint should return success for Russell3000INDEX 1`] = `
72+
{
73+
"data": {
74+
"filename": "daily_values_russell_250827.CSV",
75+
"result": {
76+
"close": 3690.93,
77+
"indexName": "Russell 3000® Index",
78+
},
79+
},
80+
"result": 3690.93,
81+
"statusCode": 200,
82+
"timestamps": {
83+
"providerDataReceivedUnixMs": 978347471111,
84+
"providerDataRequestedUnixMs": 978347471111,
85+
},
86+
}
87+
`;

0 commit comments

Comments
 (0)