Skip to content

Commit 0382372

Browse files
caparkercaparker
committed
Fixed syntax errors
1 parent c01a7e9 commit 0382372

File tree

12 files changed

+476
-150
lines changed

12 files changed

+476
-150
lines changed

‎fetcher/index.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ async function handler(event) {
2626
if (!source) throw new Error(`Unable to find ${source_name} in sources.`);
2727

2828
const log = await providers.processor(source);
29-
await providers.publish(log, 'fetcher/success');
29+
await providers.publish(log, 'fetcher/success');
3030
return log;
3131
} catch (err) {
32-
providers.publish(err, 'fetcher/error');
32+
providers.publish(err, 'fetcher/error');
3333
process.exit(1);
3434
}
3535
}

‎fetcher/lib/measure.js

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ class Measures {
77
constructor(type) {
88
this.headers = [];
99
this.measures = [];
10-
this.from = null;
11-
this.to = null;
10+
this.from = null;
11+
this.to = null;
1212

1313
if (type === FixedMeasure) {
1414
this.headers = ['sensor_id', 'measure', 'timestamp'];
@@ -18,12 +18,12 @@ class Measures {
1818
}
1919

2020
push(measure) {
21-
if(!this.to || measure.timestamp > this.to) {
22-
this.to = measure.timestamp;
23-
}
24-
if(!this.from || measure.timestamp < this.from) {
25-
this.from = measure.timestamp;
26-
}
21+
if (!this.to || measure.timestamp > this.to) {
22+
this.to = measure.timestamp;
23+
}
24+
if (!this.from || measure.timestamp < this.from) {
25+
this.from = measure.timestamp;
26+
}
2727
this.measures.push(measure);
2828
}
2929

‎fetcher/lib/meta.js

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
const { VERBOSE } = require('./utils');
2-
const { S3Client, GetObjectCommand, PutObjectCommand } = require("@aws-sdk/client-s3");
32

43
const {
5-
getObject,
6-
putObject,
4+
getObject,
5+
putObject
76
} = require('./utils');
87

98
/**
@@ -35,9 +34,9 @@ class MetaDetails {
3534

3635
save(body) {
3736
return putObject(
38-
JSON.stringify(body),
39-
this.props.Bucket,
40-
this.props.Key,
37+
JSON.stringify(body),
38+
this.props.Bucket,
39+
this.props.Key,
4140
);
4241
}
4342
}

‎fetcher/lib/providers.js

Lines changed: 31 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,14 @@
11
const fs = require('fs');
22
const path = require('path');
3-
//const AWS = require('aws-sdk');
4-
const { SNSClient, PublishCommand } = require("@aws-sdk/client-sns");
3+
const { SNSClient, PublishCommand } = require('@aws-sdk/client-sns');
54

65
const {
76
VERBOSE,
87
DRYRUN,
98
gzip,
10-
unzip,
11-
fetchSecret,
12-
getObject,
13-
putObject,
9+
fetchSecret,
10+
getObject,
11+
putObject,
1412
prettyPrintStation
1513
} = require('./utils');
1614

@@ -32,23 +30,22 @@ class Providers {
3230
/**
3331
* Given a source config file, choose the corresponding provider script to run
3432
*
35-
* @param {String} source_name
3633
* @param {Object} source
3734
*/
3835
async processor(source) {
39-
if(VERBOSE) console.debug('Processing', source.provider);
36+
if (VERBOSE) console.debug('Processing', source.provider);
4037
if (!this[source.provider]) throw new Error(`${source.provider} is not a supported provider`);
41-
// fetch any secrets we may be storing for the provider
42-
if(VERBOSE) console.debug('Fetching secret: ', source.secretKey);
43-
const config = await fetchSecret(source);
44-
// and combine them with the source config for more generic access
45-
if(VERBOSE) console.log('Starting processor', { ...source, ...config });
38+
// fetch any secrets we may be storing for the provider
39+
if (VERBOSE) console.debug('Fetching secret: ', source.secretKey);
40+
const config = await fetchSecret(source);
41+
// and combine them with the source config for more generic access
42+
if (VERBOSE) console.log('Starting processor', { ...source, ...config });
4643
const log = await this[source.provider].processor({ ...source, ...config });
47-
// source_name is more consistent with our db schema
48-
if(typeof(log) == 'object' && !Array.isArray(log) && !log.source_name) {
49-
log.source_name = source.provider;
50-
}
51-
return(log);
44+
// source_name is more consistent with our db schema
45+
if (typeof(log) == 'object' && !Array.isArray(log) && !log.source_name) {
46+
log.source_name = source.provider;
47+
}
48+
return (log);
5249
}
5350

5451
/**
@@ -57,20 +54,20 @@ class Providers {
5754
* @param {Object} message
5855
* @param {String} subject
5956
*/
60-
async publish(message, subject) {
61-
console.log('Publishing:', subject, message);
62-
if(process.env.TOPIC_ARN) {
63-
const cmd = new PublishCommand({
64-
TopicArn: process.env.TOPIC_ARN,
65-
Subject: subject,
66-
Message: JSON.stringify(message),
67-
});
68-
return await sns.send(cmd);
69-
} else {
70-
console.log('No publish topic', subject, message);
71-
return {};
72-
}
73-
}
57+
async publish(message, subject) {
58+
console.log('Publishing:', subject, message);
59+
if (process.env.TOPIC_ARN) {
60+
const cmd = new PublishCommand({
61+
TopicArn: process.env.TOPIC_ARN,
62+
Subject: subject,
63+
Message: JSON.stringify(message)
64+
});
65+
return await sns.send(cmd);
66+
} else {
67+
console.log('No publish topic', subject, message);
68+
return {};
69+
}
70+
}
7471

7572
/**
7673
* Push an array of stations to S3
@@ -112,9 +109,7 @@ class Providers {
112109
prettyPrintStation(newData);
113110
console.log('-----------------> from');
114111
prettyPrintStation(currentData);
115-
} //else {
116-
// console.log(`Updating the station file: ${providerStation}`);
117-
// }
112+
}
118113
} catch (err) {
119114
if (err.statusCode !== 404) throw err;
120115
}
@@ -157,7 +152,7 @@ class Providers {
157152
}
158153
if (VERBOSE) console.debug(`Saving measurements to ${Bucket}/${Key}`);
159154

160-
return s3.putObject({
155+
return putObject({
161156
Bucket,
162157
Key,
163158
Body: compressedString,

‎fetcher/lib/utils.js

Lines changed: 50 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ const zlib = require('zlib');
22
const { promisify } = require('util');
33
const request = promisify(require('request'));
44

5-
const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager");
6-
const { S3Client, GetObjectCommand, PutObjectCommand } = require("@aws-sdk/client-s3");
5+
const { SecretsManagerClient, GetSecretValueCommand } = require('@aws-sdk/client-secrets-manager');
6+
const { S3Client, GetObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3');
77

88
const VERBOSE = !!process.env.VERBOSE;
99
const DRYRUN = !!process.env.DRYRUN;
@@ -12,56 +12,55 @@ const s3 = new S3Client({
1212
maxRetries: 10
1313
});
1414

15-
15+
const gzip = promisify(zlib.gzip);
16+
const unzip = promisify(zlib.unzip);
1617

1718
async function getObject(Bucket, Key) {
18-
const cmd = new GetObjectCommand({
19-
Bucket,
20-
Key,
21-
});
22-
var stream = null;
23-
const resp = await s3.send(cmd);
24-
let currentData = null;
25-
if(resp && resp.ContentEncoding == 'gzip') {
26-
const ba = await resp.Body.transformToByteArray();
27-
currentData = (await unzip(Buffer.from(ba))).toString('utf-8');
28-
} else if(resp && resp.Body) {
29-
currentData = await resp.Body.transformToString();
30-
}
31-
return currentData;
19+
const cmd = new GetObjectCommand({
20+
Bucket,
21+
Key
22+
});
23+
const resp = await s3.send(cmd);
24+
let currentData = null;
25+
if (resp && resp.ContentEncoding === 'gzip') {
26+
const ba = await resp.Body.transformToByteArray();
27+
currentData = (await unzip(Buffer.from(ba))).toString('utf-8');
28+
} else if (resp && resp.Body) {
29+
currentData = await resp.Body.transformToString();
30+
}
31+
return currentData;
3232
}
3333

34-
async function putObject(text, Bucket, Key, gzip=true, ContentType='application/json') {
35-
let ContentEncoding = null;
36-
if(gzip) {
37-
text = compressedString = await gzip(text);
38-
ContentEncoding = 'gzip';
39-
}
40-
const cmd = new PutObjectCommand({
41-
Bucket,
42-
Key,
34+
async function putObject(text, Bucket, Key, gzip = true, ContentType = 'application/json') {
35+
let ContentEncoding = null;
36+
if (gzip) {
37+
text = await gzip(text);
38+
ContentEncoding = 'gzip';
39+
}
40+
const cmd = new PutObjectCommand({
41+
Bucket,
42+
Key,
4343
Body: text,
4444
ContentType,
45-
ContentEncoding,
46-
});
47-
return await s3.send(cmd);
45+
ContentEncoding
46+
});
47+
return await s3.send(cmd);
4848
}
4949

5050
/**
5151
* Retrieve secret from AWS Secrets Manager
52-
* @param {string} source_name The source for which we are fetching a secret.
52+
* @param {string} source The source object for which we are fetching a secret.
5353
*
5454
* @returns {object}
5555
*/
5656
async function fetchSecret(source) {
57-
const key = source.secretKey;
58-
if(!key) {
59-
return {};
60-
}
57+
const key = source.secretKey;
58+
if (!key) {
59+
return {};
60+
}
6161
const secretsManager = new SecretsManagerClient({
6262
region: process.env.AWS_DEFAULT_REGION || 'us-east-1',
63-
maxAttemps: 1,
64-
63+
maxAttemps: 1
6564
});
6665

6766
if (!process.env.STACK) throw new Error('STACK Env Var Required');
@@ -72,17 +71,18 @@ async function fetchSecret(source) {
7271

7372
if (VERBOSE) console.debug(`Fetching ${SecretId} secret...`);
7473

75-
const cmd = new GetSecretValueCommand({
76-
SecretId
77-
});
74+
const cmd = new GetSecretValueCommand({
75+
SecretId
76+
});
77+
7878
const resp = await secretsManager
79-
.send(cmd)
80-
.catch(err => console.error(`Missing ${key} secret`));
81-
if(resp && resp.SecretString) {
82-
return JSON.parse(resp.SecretString);
83-
} else {
84-
return {};
85-
}
79+
.send(cmd)
80+
.catch((err) => console.error(`Missing ${key} secret: ${err}`));
81+
if (resp && resp.SecretString) {
82+
return JSON.parse(resp.SecretString);
83+
} else {
84+
return {};
85+
}
8686
}
8787

8888
/**
@@ -138,6 +138,8 @@ function prettyPrintStation(station) {
138138
* @param {array} data
139139
* @param {timestamp} start_timestamp
140140
* @param {timestamp} end_timestamp
141+
*
142+
* @returns {array}
141143
*/
142144
function checkResponseData(data, start_timestamp, end_timestamp) {
143145
const n = data && data.length;
@@ -170,12 +172,6 @@ function checkResponseData(data, start_timestamp, end_timestamp) {
170172
return fdata;
171173
}
172174

173-
function publisher(subject, message, topic) {
174-
175-
}
176-
177-
const gzip = promisify(zlib.gzip);
178-
const unzip = promisify(zlib.unzip);
179175

180176
module.exports = {
181177
fetchSecret,
@@ -185,8 +181,8 @@ module.exports = {
185181
unzip,
186182
VERBOSE,
187183
DRYRUN,
188-
getObject,
189-
putObject,
184+
getObject,
185+
putObject,
190186
prettyPrintStation,
191187
checkResponseData
192188
};

0 commit comments

Comments
 (0)