Skip to content

Commit 3c6d23d

Browse files
saurvssaurvs
committed
Add non-sha1 metadata processing. Add fetching catalog statistics and properties.
1 parent d8ee421 commit 3c6d23d

File tree

11 files changed

+215
-27
lines changed

11 files changed

+215
-27
lines changed

‎cvmfs/fetcher.js

Lines changed: 38 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,12 @@ cvmfs.fetcher.downloadCertificate = function(data_url, hash) {
2626
return this.downloadChunk(data_url, hash, 'X');
2727
};
2828

29+
cvmfs.fetcher.downloadCatalog = function(data_url, hash) {
30+
return this.downloadChunk(data_url, hash, 'C');
31+
};
32+
2933
cvmfs.fetcher.parseManifest = function(data, repo_name) {
3034
const manifest = {};
31-
const metadata_digest = new KJUR.crypto.MessageDigest({alg: 'sha1', prov: 'cryptojs'});
3235

3336
const lines = data.split('\n');
3437
for (const i in lines) {
@@ -44,7 +47,7 @@ cvmfs.fetcher.parseManifest = function(data, repo_name) {
4447
manifest.catalog_size = parseInt(tail);
4548
break;
4649
case 'C':
47-
manifest.catalog_hash = tail;
50+
manifest.catalog_hash = new cvmfs.util.hash(tail);
4851
break;
4952
case 'D':
5053
manifest.ttl = parseInt(tail);
@@ -53,10 +56,10 @@ cvmfs.fetcher.parseManifest = function(data, repo_name) {
5356
manifest.garbage_collectable = (tail === 'yes');
5457
break;
5558
case 'H':
56-
manifest.history_hash = tail;
59+
manifest.history_hash = new cvmfs.util.hash(tail);
5760
break;
5861
case 'M':
59-
manifest.json_hash = tail;
62+
manifest.json_hash = new cvmfs.util.hash(tail);
6063
break;
6164
case 'N':
6265
if (tail !== repo_name) return undefined;
@@ -72,27 +75,28 @@ cvmfs.fetcher.parseManifest = function(data, repo_name) {
7275
manifest.published_timestamp = parseInt(tail);
7376
break;
7477
case 'X':
75-
manifest.certificate_hash = tail;
78+
manifest.certificate_hash = new cvmfs.util.hash(tail);
7679
break;
7780
}
7881

7982
if (head === '-') {
8083
const j = (parseInt(i) + 1).toString();
81-
manifest.metadata_hash = lines[j];
84+
manifest.metadata_hash = new cvmfs.util.hash(lines[j]);
8285
break;
8386
}
84-
85-
metadata_digest.updateString(line + '\n')
8687
}
8788

8889
if (manifest.catalog_hash === undefined ||
8990
manifest.root_hash === undefined ||
9091
manifest.ttl === undefined ||
9192
manifest.revision === undefined) return undefined;
9293

93-
if (manifest.metadata_hash !== metadata_digest.digest()) return undefined;
94+
const metadata = data.substring(0, data.search('--'));
95+
const computed_metadata_hash = cvmfs.util.digestString(metadata, manifest.metadata_hash.alg);
96+
if (manifest.metadata_hash.hex !== computed_metadata_hash) return undefined;
9497

95-
const signature = data.substr(data.search('-') + 3 /*(--\n)*/ + 40 /*(SHA1 hex len)*/ + 1 /*(\n)*/);
98+
var signature = data.substr(data.search('--') + 3 /*(--\n)*/);
99+
signature = signature.substr(signature.search('\n') + 1 /*\n*/);
96100
manifest.signature_hex = cvmfs.util.stringToHex(signature);
97101

98102
return manifest;
@@ -104,9 +108,13 @@ cvmfs.fetcher.fetchManifest = function(repo_url, repo_name) {
104108
};
105109

106110
cvmfs.fetcher.parseWhitelist = function(data, repo_name) {
107-
const metadata = data.substr(0, data.search('-'));
108-
const metadata_hash = data.substr(metadata.length + 3 /*(--\n)*/, 40 /*(SHA1 hex len)*/);
109-
if (metadata_hash !== KJUR.crypto.Util.sha1(metadata)) return undefined;
111+
const metadata = data.substr(0, data.search('--'));
112+
var metadata_hash_str = data.substr(metadata.length + 3 /*(--\n)*/);
113+
metadata_hash_str = metadata_hash_str.substr(0, metadata_hash_str.search('\n'));
114+
115+
const metadata_hash = new cvmfs.util.hash(metadata_hash_str);
116+
const computed_metadata_hash = cvmfs.util.digestString(metadata, metadata_hash.alg);
117+
if (metadata_hash.hex !== computed_metadata_hash) return undefined;
110118

111119
const whitelist = { metadata_hash: metadata_hash };
112120
const lines = metadata.split('\n');
@@ -122,9 +130,10 @@ cvmfs.fetcher.parseWhitelist = function(data, repo_name) {
122130
parseInt(expiry_line.substr(9, 2))
123131
);
124132

125-
whitelist.certificate_fingerprint = lines[3].replace(/\:/g, '').toLowerCase();
133+
whitelist.certificate_fingerprint = new cvmfs.util.hash(lines[3].replace(/\:/g, '').toLowerCase());
126134

127-
const signature = data.substr(metadata.length + 3 /*(--\n)*/ + 40 /*(SHA1 hex len)*/ + 1 /*(\n)*/);
135+
var signature = data.substr(metadata.length + 3 /*(--\n)*/);
136+
signature = signature.substr(signature.search('\n') + 1 /*(\n)*/);
128137
whitelist.signature_hex = cvmfs.util.stringToHex(signature);
129138

130139
return whitelist;
@@ -136,11 +145,11 @@ cvmfs.fetcher.fetchWhitelist = function(repo_url, repo_name) {
136145
};
137146

138147
cvmfs.fetcher.fetchCertificate = function(data_url, cert_hash) {
139-
const data = cvmfs.fetcher.downloadCertificate(data_url, cert_hash);
148+
const data = cvmfs.fetcher.downloadCertificate(data_url, cert_hash.download_handle);
140149

141150
const data_hex = cvmfs.util.stringToHex(data);
142-
const data_hash = KJUR.crypto.Util.hashHex(data_hex, 'sha1');
143-
if (data_hash !== cert_hash) return undefined;
151+
const data_hash = cvmfs.util.digestHex(data_hex, cert_hash.alg);
152+
if (data_hash !== cert_hash.hex) return undefined;
144153

145154
const data_deflated = pako.inflate(data);
146155
const decoder = new TextDecoder("utf-8");
@@ -149,4 +158,15 @@ cvmfs.fetcher.fetchCertificate = function(data_url, cert_hash) {
149158
const certificate = new X509();
150159
certificate.readCertPEM(pem);
151160
return certificate;
161+
};
162+
163+
cvmfs.fetcher.fetchCatalog = function(data_url, catalog_hash) {
164+
const data = cvmfs.fetcher.downloadCatalog(data_url, catalog_hash.download_handle);
165+
166+
const data_hex = cvmfs.util.stringToHex(data);
167+
const data_hash = cvmfs.util.digestHex(data_hex, catalog_hash.alg);
168+
if (data_hash !== catalog_hash.hex) return undefined;
169+
170+
const db_data = pako.inflate(data);
171+
return new SQL.Database(db_data);
152172
};

‎cvmfs/repo.js

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ cvmfs.repo = function(base_url, repo_name) {
1111
const master_keys = cvmfs.getMasterKeys();
1212
for (const key of master_keys) {
1313
whitelist_verified = key.verifyRawWithMessageHex(
14-
cvmfs.util.stringToHex(this._whitelist.metadata_hash),
14+
cvmfs.util.stringToHex(this._whitelist.metadata_hash.download_handle),
1515
this._whitelist.signature_hex
1616
);
1717
if (whitelist_verified) break;
@@ -21,17 +21,61 @@ cvmfs.repo = function(base_url, repo_name) {
2121
/* verify certificate fingerprint */
2222
const now = new Date();
2323
if (now >= this._whitelist.expiry_date) return undefined;
24-
const fingerprint = KJUR.crypto.Util.hashHex(this._certificate.hex, 'sha1');
25-
if (fingerprint !== this._whitelist.certificate_fingerprint) return undefined;
24+
const fingerprint = cvmfs.util.digestHex(this._certificate.hex, this._whitelist.certificate_fingerprint.alg);
25+
if (fingerprint !== this._whitelist.certificate_fingerprint.hex) return undefined;
2626

2727
/* verify manifest signature */
2828
const signature = new KJUR.crypto.Signature({alg: 'SHA1withRSA'});
2929
signature.init(this._certificate.getPublicKey());
30-
signature.updateString(this._manifest.metadata_hash);
30+
signature.updateString(this._manifest.metadata_hash.download_handle);
3131
if (!signature.verify(this._manifest.signature_hex)) return undefined;
3232
};
3333

3434
cvmfs.repo.prototype = {
35+
_catalog: null,
36+
_catalog_stats: null,
37+
_catalog_properties: null,
38+
_getCatalog: function() {
39+
if (this._catalog === null) {
40+
this._catalog = cvmfs.fetcher.fetchCatalog(this._data_url, this._manifest.catalog_hash);
41+
}
42+
return this._catalog;
43+
},
44+
getCatalogStats: function() {
45+
if (this._catalog_stats === null) {
46+
const catalog = this._getCatalog();
47+
const result = catalog.exec("SELECT * FROM STATISTICS")[0].values;
48+
49+
this._catalog_stats = {};
50+
for (const row of result) {
51+
const counter = row[0];
52+
const value = row[1];
53+
this._catalog_stats[counter] = value;
54+
}
55+
}
56+
return this._catalog_stats;
57+
},
58+
getCatalogProperties: function() {
59+
if (this._catalog_properties === null) {
60+
const catalog = this._getCatalog();
61+
const result = catalog.exec("SELECT * FROM PROPERTIES")[0].values;
62+
63+
this._catalog_properties = {};
64+
for (const row of result) {
65+
const counter = row[0];
66+
67+
var value = row[1];
68+
if (/^\d+$/.test(value)) { // if value is base-10 integer
69+
value = parseInt(value);
70+
} else if (/^\d+\.?\d*$/.test(value)) { // if value is float
71+
value = parseFloat(value);
72+
}
73+
74+
this._catalog_properties[counter] = value;
75+
}
76+
}
77+
return this._catalog_properties;
78+
},
3579
getManifest: function() { return this._manifest; },
3680
getWhitelist: function() { return this._whitelist; },
3781
getCertificate: function() { return this._certificate; }

‎cvmfs/util.js

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,45 @@ cvmfs.util.stringToHex = function(str) {
1414
hex[i] = ('0' + byte.toString(16)).slice(-2);
1515
}
1616
return hex.join('');
17+
}
18+
19+
cvmfs.util.hash = function(download_handle) {
20+
this.download_handle = download_handle;
21+
22+
const hash_len = download_handle.search('-');
23+
if (hash_len === -1) {
24+
this.hex = download_handle;
25+
this.alg = 'sha1';
26+
} else {
27+
this.hex = download_handle.substring(0, hash_len);
28+
this.alg = download_handle.substring(hash_len + 1);
29+
}
30+
}
31+
32+
cvmfs.util.hash.prototype = {
33+
hex: null,
34+
alg: null,
35+
download_handle: null
36+
};
37+
38+
cvmfs.util.digestString = function(str, alg) {
39+
if (alg === undefined) return undefined;
40+
else if (alg === 'shake128') {
41+
const shake128 = new jsSHA("SHAKE128", "TEXT");
42+
shake128.update(str);
43+
return shake128.getHash("HEX", {shakeLen: 160});
44+
} else {
45+
return KJUR.crypto.Util.hashString(str, alg);
46+
}
47+
}
48+
49+
cvmfs.util.digestHex = function(hex, alg) {
50+
if (alg === undefined) return undefined;
51+
else if (alg === 'shake128') {
52+
const shake128 = new jsSHA("SHAKE128", "HEX");
53+
shake128.update(hex);
54+
return shake128.getHash("HEX", {shakeLen: 160});
55+
} else {
56+
return KJUR.crypto.Util.hashHex(hex, alg);
57+
}
1758
}

‎emcc-cvmfs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ THIRD_PARTY_DIR=$SRC_DIR/third_party
88
emcc $* \
99
-s WASM=0 \
1010
--pre-js $THIRD_PARTY_DIR/jsrsasign-all-min.js \
11+
--pre-js $THIRD_PARTY_DIR/sha3.js \
1112
--pre-js $THIRD_PARTY_DIR/pako_inflate.min.js \
13+
--pre-js $THIRD_PARTY_DIR/sql.js \
1214
--pre-js $CVMFS_DIR/cvmfs.js \
1315
--pre-js $CVMFS_DIR/fetcher.js \
1416
--pre-js $CVMFS_DIR/master_keys.js \

‎fs/library_cvmfs.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ mergeInto(LibraryManager.library, {
1414
console.log(whitelist);
1515
console.log(certificate);
1616

17+
console.log(repo.getCatalogStats());
18+
console.log(repo.getCatalogProperties());
19+
1720
return CVMFS.createNode(null, '/', {{{ cDefine('S_IFDIR') }}} | 0777);
1821
},
1922
createNode: function(parent, name, mode) {

‎tests/data/cvmfspublished-shake128

595 Bytes
Binary file not shown.

‎tests/data/cvmfswhitelist-shake128

430 Bytes
Binary file not shown.

‎tests/js/manifest.js

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,29 @@ tests.runTest(function test_parseManifest() {
1313
const manifest = cvmfs.fetcher.parseManifest(data, repo_name);
1414
chai.assert.strictEqual(manifest.has_alt_catalog_path, false);
1515
chai.assert.strictEqual(manifest.catalog_size, 18432);
16-
chai.assert.strictEqual(manifest.catalog_hash, '34ec515941acb52652ac2c448407caebca2bfe49');
16+
chai.assert.strictEqual(manifest.catalog_hash.hex, '34ec515941acb52652ac2c448407caebca2bfe49');
1717
chai.assert.strictEqual(manifest.ttl, 240);
1818
chai.assert.strictEqual(manifest.garbage_collectable, false);
19-
chai.assert.strictEqual(manifest.history_hash, 'e615b5a83d3a20120199ba05afe4ffcbef07714a');
20-
chai.assert.strictEqual(manifest.json_hash, 'decf358874358c4f313db997d36a3b8e3d62622e');
19+
chai.assert.strictEqual(manifest.history_hash.hex, 'e615b5a83d3a20120199ba05afe4ffcbef07714a');
20+
chai.assert.strictEqual(manifest.json_hash.hex, 'decf358874358c4f313db997d36a3b8e3d62622e');
2121
chai.assert.strictEqual(manifest.repository_name, repo_name);
2222
chai.assert.strictEqual(manifest.root_hash, 'd41d8cd98f00b204e9800998ecf8427e');
2323
chai.assert.strictEqual(manifest.revision, 10);
2424
chai.assert.strictEqual(manifest.published_timestamp, 1525779612);
25-
chai.assert.strictEqual(manifest.certificate_hash, 'f03e97f4586869a417dadd56ff206c8ba9566284');
26-
chai.assert.strictEqual(manifest.metadata_hash, '92db4f64d030df3a32bb18233b4b933511c20d9c');
25+
chai.assert.strictEqual(manifest.certificate_hash.hex, 'f03e97f4586869a417dadd56ff206c8ba9566284');
26+
chai.assert.strictEqual(manifest.metadata_hash.hex, '92db4f64d030df3a32bb18233b4b933511c20d9c');
2727
chai.assert.strictEqual(manifest.signature_hex, 'b8f34fa30053d1ea3a440ad6c8a7d0b684182fbccabca531674059fcf1f83a5e5535e428983ede29d38c8c87741c1bd700ec8ee1315bea6d9b6aca5159460c7e889a62c137d90ce4ea6d0d8413ec86dd521b57544f456411df49e9791a9c3a8955a14e8bddd1a5e52cd8dc9d7c28bb6cf76e9b1dc11b8009dde2a78b340303e9913dbc36dc4520438bf2a402cc1efca1e938082a7235136238d880d1c8fca4add6755ae53887237841f46a7aa72b0d98c43f890dd724cdcd7a1cacd7f6aaf95c9f7f26c2b224e811ec0ed1734246bb745e0410086938b6acafc43d00496de1091a8eb3e42a801cd0a07db89283d362972f702d8048d1d7ad9d994693053a80d7');
28+
});
29+
30+
tests.runTest(function test_parseManifestShake128() {
31+
const data = tests.readFile('data/cvmfspublished-shake128');
32+
const repo_name = 'emscripten.cvmfs.io';
33+
34+
const manifest = cvmfs.fetcher.parseManifest(data, repo_name);
35+
chai.assert.strictEqual(manifest.catalog_hash.hex, '08b158e37829afbb15a0d8f78885b91941b16df2');
36+
chai.assert.strictEqual(manifest.history_hash.hex, 'c0bb8dd94b8b2ccdf7df816e6d1dfd2af07c3ea9');
37+
chai.assert.strictEqual(manifest.json_hash.hex, 'decf358874358c4f313db997d36a3b8e3d62622e');
38+
chai.assert.strictEqual(manifest.certificate_hash.hex, 'f223b5c0af62e4e494f6c1cd629ef39163b1a32d');
39+
chai.assert.strictEqual(manifest.metadata_hash.hex, '984a2eb4ac636675caceec3df39e53012dffa6e4');
40+
chai.assert.strictEqual(manifest.signature_hex, 'a065622a4619cce396670f6688970765d1ef82453a9f9a8a4c13fe84f13df2facb2b284bab09ccf3e0a9f5f6ee4a381047cc446a3c45fb2326ee842f6a71140d3036d96ce74a295457df48a517a4242d7bdb21baf931c1666d272e204df24d6852619695b359f83dbefd105743fb0bd65a679910430a4f1a63f4d0aa534543c661066e2794a898e161f27da53161362936d63fc83015dd8ae64c953089d0223310ef732452c41d6f4a8f519df01d5795fa9008049524b85d728c3bf3a752d4f4c8e378f0e8121f264c80f1ec3e7d70238676dd70a75ad84960f4d5af970c63c0166983ca40d90ea84dc3517de71100a791b28db58c914c63c0241a231a659e2a');
2841
});

‎tests/js/whitelist.js

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,14 @@ tests.runTest(function parseWhitelist() {
1616
chai.assert.strictEqual(whitelist.certificate_fingerprint, 'fc97541ac39e72cac9bfd7b27b3cf54c3c2a35b9');
1717
chai.assert.strictEqual(whitelist.metadata_hash, 'b04ddf54b894e03d7b6fe2ab9e26f0161229dc7d');
1818
chai.assert.strictEqual(whitelist.signature_hex, 'b0d8d8f9b7d6a1dfceed6e5995c6f56b716a5e54fc4121da1c74da0ff4bc97805001ef2ec502a10e52921f72d0e81efa3a83541a0d59c99c29cf4e627ea9f233debde50ffc3ecacc354a345745cf3d487ac6ee3b550facaab50380eea992bbaeb7681a39bc48c5b0b9ac6f9a613b9d15c3198168621c9a75d832d04ccaab203617858e5985169b0a18130bcdc0c113b5f92611b289c4eb53f74e7813584c9b02620fa2113a5e7328d3fb65fb43c66d3edaffc02005c47326719005035c98f53555156b444f0cf3a3e56a7efed1c1ebce740bf13de7e8040f72e187b7ed642ab0402b35c7f07222b673f72cd5cdf81c70b7a8ce139c2eddd445c88f830fb6297f');
19+
});
20+
21+
tests.runTest(function parseWhitelistShake128() {
22+
const data = tests.readFile('data/cvmfswhitelist-shake128');
23+
const repo_name = 'emscripten.cvmfs.io';
24+
25+
const whitelist = cvmfs.fetcher.parseWhitelist(data, repo_name);
26+
chai.assert.strictEqual(whitelist.certificate_fingerprint.hex, '2d63ea98499ed1041fca8359c7989dc28c171edf');
27+
chai.assert.strictEqual(whitelist.metadata_hash.hex, 'c6e2ed2dd9e768601f5621ebcc5b6b058a624282');
28+
chai.assert.strictEqual(whitelist.signature_hex, '49a9d8bf0520f84b8040347b8170b005f5235cb509e80e4f5631866b56e510562809369a5c664edfb402e9a772b471266062506e7890c778bed41abf2a71f9577d4182b3cc2257480787c985f489a26f590b41926d59145fac4b68b8509670385a2522c59016e128bd12165b7ea2a7ac4812a9653637773672fc149962fec4d1abf4df9b68dabca54ed46c55bdfa906155a16ea5872446c14ec82dbdd706d6c2efc7d6e1f4abae75be87b84cecfd898fbdcc81843043252edc8a0cad0027ad7e525db1adaf7830ba316d6b96695d3a2e194e6b857aa80cd694501831305699339ffa6847571662606b76b91b6cb9874298a90bfc338c98a4327d453b4ea76ddf');
1929
});

0 commit comments

Comments
 (0)