From a61431bdd7adbcd706bd0f3a9b71c63e0176f022 Mon Sep 17 00:00:00 2001
From: lyn1dev <78355114+lyn1dev@users.noreply.github.com>
Date: Sat, 7 Nov 2026 18:14:57 +0700
Subject: [PATCH 1/2] sync game data automatically
the static data was a one-off snapshot from august and the scripts that built
it weren't in the repo, so it never updated. this adds a sync that rebuilds it
every 6 hours from NotEnoughUpdates-REPO, SkyShards-Parser and the hypixel api.
broken data stops the sync, suspicious changes wait for a review, and everything
else goes live.
the site now shows when data last synced, and warns if syncing stops or the
bazaar lists shards it doesn't know about.
fixes found along the way:
- every craft was labelled common, which also emptied the rarity filters
- slayer levels used the zombie xp table for every slayer
- essence recipes with an unpriced material would have counted it as free
- esc() didn't escape quotes
- bazaar tax is now a setting, since it depends on the community shop upgrade
---
.github/workflows/sync-data.yml | 61 ++++
README.md | 31 +-
THIRD_PARTY_NOTICES.md | 60 ++++
index.html | 216 ++++++++-----
scripts/publish-sync.sh | 61 ++++
scripts/sync-data.mjs | 518 ++++++++++++++++++++++++++++++++
scripts/sync-data.test.mjs | 243 +++++++++++++++
7 files changed, 1117 insertions(+), 73 deletions(-)
create mode 100644 .github/workflows/sync-data.yml
create mode 100644 THIRD_PARTY_NOTICES.md
create mode 100644 scripts/publish-sync.sh
create mode 100644 scripts/sync-data.mjs
create mode 100644 scripts/sync-data.test.mjs
diff --git a/.github/workflows/sync-data.yml b/.github/workflows/sync-data.yml
new file mode 100644
index 0000000..31aec1a
--- /dev/null
+++ b/.github/workflows/sync-data.yml
@@ -0,0 +1,61 @@
+name: Sync game data
+
+on:
+ schedule:
+ - cron: '17 */6 * * *'
+ workflow_dispatch:
+ # Check changes to the sync itself without publishing anything.
+ pull_request:
+ paths:
+ - 'scripts/**'
+ - '.github/workflows/sync-data.yml'
+
+permissions:
+ contents: write
+ pull-requests: write
+ issues: write
+
+concurrency:
+ group: sync-data
+
+jobs:
+ sync:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 22
+
+ - name: Test the sync script
+ run: node --test scripts/sync-data.test.mjs
+
+ - name: Dry run
+ if: github.event_name == 'pull_request'
+ run: node scripts/sync-data.mjs --dry-run
+
+ - name: Build and check data
+ id: sync
+ if: github.event_name != 'pull_request'
+ run: node scripts/sync-data.mjs --report "$RUNNER_TEMP/sync-report.md"
+
+ - name: Publish
+ if: steps.sync.outputs.status == 'publish' || steps.sync.outputs.status == 'review'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ STATUS: ${{ steps.sync.outputs.status }}
+ run: bash scripts/publish-sync.sh "$STATUS" "$RUNNER_TEMP/sync-report.md"
+
+ - name: Report failure
+ if: failure() && github.event_name != 'pull_request'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ run: |
+ title='Data sync is failing'
+ existing=$(gh issue list --state open --search "\"$title\" in:title" --json number --jq '.[0].number // empty')
+ if [ -z "$existing" ]; then
+ gh issue create --title "$title" --body "The live site still has the last good data. Details: $RUN_URL"
+ fi
diff --git a/README.md b/README.md
index fabbd3a..8c77e8f 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ hypixel skyblock bazaar flipping calculation suite. includes recursive shard fus
## modules
-- **โก shard flipper**: 321 shards, 257k recipes. calculates optimal fusion paths (e.g. when fusing galaxy fish / sun fish is cheaper than buying them raw), accounts for minimum recipe yields, order book depth, fulfillment timers, and total fusion machine cycles.
+- **โก shard flipper**: every attribute shard and fusion recipe. calculates optimal fusion paths (e.g. when fusing galaxy fish / sun fish is cheaper than buying them raw), accounts for minimum recipe yields, order book depth, fulfillment timers, and total fusion machine cycles.
- **๐ book flipper**: 155 enchanted book types. calculates optimal anvil combination paths (e.g. if buying and combining 2x level 2 books to make a level 3 book is cheaper than buying 4x level 1 books, it does that). reverse-engineers combinable limits from enchants.json and table limits, tracks dedication 3 caps, base level 3/4/2 enchants, shortened arrays, and counts total anvil combines.
- **๐ผ portfolio flipper**: multi-order capital allocation optimizer (`bazaar_gui.py` / `bazaar_flipper.py`). allocates your total budget (e.g. 220m) across up to 12-24 bazaar slots, enforces in-game hard caps (128 / 71,680), penalizes queue friction, and calculates exact check-back fill timers. optional goldilocks anti-bot volume filter.
- **๐จ craft flipper**: slayer and collection craft optimizer (`craft_flipper.py`). calculates exact profits for revenant viscera, tarantula silk, golden tooth, null ovoid, super compactor 3000, enchanted lava / magma / plasma buckets, hot potato books, refined titanium/mithril, and fermento.
@@ -12,7 +12,7 @@ hypixel skyblock bazaar flipping calculation suite. includes recursive shard fus
## how to run
-literally just host `index.html`, `fusion-data.json`, and `enchants.json` with any static server:
+literally just host the repo folder with any static server:
```bash
python -m http.server 8000
@@ -25,6 +25,33 @@ open `http://localhost:8000` in your browser.
bazaar prices work without a key. you only need a free hypixel api key from https://developer.hypixel.net to look up player profiles in the craft flipper.
the first time you do a lookup it asks for the key once, saves it in your browser localstorage (not shared anywhere), and leaves you alone. you can add or change it any time with the `๐ API KEY` button.
+## data sources
+
+prices are live. everything else updates by itself: a github action runs `scripts/sync-data.mjs` every 6 hours and commits new data to `main`, which deploys the site.
+
+| data | file | source |
+|---|---|---|
+| fusion recipes, shards | `fusion-data.json` | [SkyShards-Parser](https://github.com/Campionnn/SkyShards-Parser) |
+| shard icons | `shardIcons/` | [SkyShards](https://github.com/Campionnn/SkyShards) |
+| craft recipes | `craft-recipes.json` | [NotEnoughUpdates-REPO](https://github.com/NotEnoughUpdates/NotEnoughUpdates-REPO) |
+| craft rarity, collection unlocks | `craft-recipes.json` | hypixel api |
+| enchants | `enchants.json` | NotEnoughUpdates-REPO |
+| essence shops, slayer xp | `game-data.json` | NotEnoughUpdates-REPO |
+| item names | `game-data.json` | hypixel api, then NotEnoughUpdates-REPO |
+
+the sync checks the data before it goes live:
+
+- **broken data** (bad format, missing values, html in names) stops the sync and opens an issue. the site keeps the last good data.
+- **suspicious changes** (over 10% of fusions gone, shards removed, over 5% of crafts gone or changed) go to a `data-sync-review` branch and a pr or issue instead of the live site.
+
+to run it yourself: `node scripts/sync-data.mjs --dry-run` shows what would change without writing anything. drop `--dry-run` to write the files. no dependencies, just node 22 and `git`. tests: `node --test scripts/sync-data.test.mjs`.
+
+**this can still be wrong.** hypixel doesn't publish recipes or fusion rules, so they come from community projects that can lag behind a game update. the site shows when data last synced and warns if syncing stops or the bazaar lists shards it doesn't know, but check anything expensive in-game before buying.
+
+these are still maintained by hand in `index.html`, because no source publishes them: anvil combine limits (`EXPLICIT_LIMITS`, `EXPLICIT_BASE_LEVELS`) and bazaar order caps (`ITEM_LIMITS`). the same goes for the slayer unlocks in `SLAYER_FALLBACKS` in the sync script.
+
+data licenses and credits are in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).
+
## star this thing
if this saved you coins or braincells star the repo:
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
new file mode 100644
index 0000000..e998979
--- /dev/null
+++ b/THIRD_PARTY_NOTICES.md
@@ -0,0 +1,60 @@
+# third party data
+
+this project's game data comes from these projects. thanks to everyone who maintains them.
+
+- **craft recipes, enchants, essence shops, slayer xp:** [NotEnoughUpdates-REPO](https://github.com/NotEnoughUpdates/NotEnoughUpdates-REPO), MIT license, notice below
+- **shard icons:** [SkyShards](https://github.com/Campionnn/SkyShards), MIT license, notice below
+- **fusion recipes:** [SkyShards-Parser](https://github.com/Campionnn/SkyShards-Parser) by Campion, built from the Hypixel SkyBlock wiki
+- **rarity, collection unlocks, item names, prices:** the [Hypixel API](https://developer.hypixel.net)
+
+## NotEnoughUpdates-REPO
+
+```
+MIT License
+
+Copyright (c) 2020 Moulberry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
+
+## SkyShards
+
+```
+MIT License
+
+Copyright (c) 2026 Campion
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
diff --git a/index.html b/index.html
index e66280f..37ec7ba 100644
--- a/index.html
+++ b/index.html
@@ -542,6 +542,15 @@
justify-content: space-between;
color: var(--text-dim);
}
+ .data-warning {
+ border: 1px solid #d2992266;
+ background: rgba(210, 153, 34, 0.08);
+ color: var(--yellow);
+ padding: 6px 12px;
+ margin-bottom: 10px;
+ font-size: 11px;
+ line-height: 1.5;
+ }
.status-text b { color: var(--text); }
.status-text .g { color: var(--green); font-weight: 700; }
.status-text .r { color: var(--red); font-weight: 700; }
@@ -613,6 +622,9 @@
.rar-tag.rare, .rar-tag.special, .rar-tag.ice, .rar-tag.diamond { color: var(--blue); border-color: #58a6ff55; }
.rar-tag.uncommon, .rar-tag.turbo, .rar-tag.collection, .rar-tag.forest { color: var(--green); border-color: #3fb95055; }
.rar-tag.common, .rar-tag.regular { color: var(--text-dim); border-color: #7d859055; }
+ .rar-tag.mythic { color: #f178d8; border-color: #f178d855; }
+ .rar-tag.fossil { color: var(--yellow); border-color: #d2992255; }
+ .rar-tag.safari { color: var(--green); border-color: #3fb95055; }
.shard-name { font-weight: 700; }
.shard-name.legendary, .shard-name.ultimate, .shard-name.slayer, .shard-name.wither { color: var(--orange); }
@@ -620,6 +632,9 @@
.shard-name.rare, .shard-name.special, .shard-name.ice, .shard-name.diamond { color: var(--blue); }
.shard-name.uncommon, .shard-name.turbo, .shard-name.collection, .shard-name.forest { color: var(--green); }
.shard-name.common, .shard-name.regular { color: #cbd5e1; }
+ .shard-name.mythic { color: #f178d8; }
+ .shard-name.fossil { color: var(--yellow); }
+ .shard-name.safari { color: var(--green); }
.vol-stat { margin-left: auto; font-size: 10.5px; color: var(--text-dim); font-weight: 400; }
@@ -854,6 +869,7 @@
@@ -1033,6 +1049,7 @@
Rarity:
All
+
Mythic
Leg
Epic
Rare
@@ -1042,15 +1059,16 @@
Essence:
-
+
All
- Wither
- Undead
- Ice
- Diamond
+
+ Tax %:
+
+
+
@@ -1071,6 +1089,7 @@
Initializing data...
--:--:--
+
@@ -1085,6 +1104,10 @@
let fusionData = null;
let enchantsData = null;
let craftRecipesData = null;
+ let essenceRecipes = {};
+ let itemNames = {};
+ let slayerXp = {};
+ let dataManifest = null;
let productsData = {};
let shardPrices = {};
let bookPrices = {};
@@ -1108,7 +1131,8 @@
let activeMemberData = null;
let unlockedOnlyFilter = true;
- const BAZAAR_TAX = 0.01125;
+ let BAZAAR_TAX = 0.01125;
+ const DATA_STALE_HOURS = 48;
// Specific Skyblock Enchantment Exceptions
const EXPLICIT_LIMITS = {
@@ -1165,40 +1189,6 @@
"DEFAULT": 71680
};
- // Essence Shop Conversions
- const ESSENCE_RECIPES = {
- 'INFINITE_SPIRIT_LEAP': {
- name: 'Infinileap',
- category: 'wither',
- shop: 'Wither Essence Shop',
- materials: { 'ESSENCE_WITHER': 25000 }
- },
- 'INFINITE_SUPERBOOM_TNT': {
- name: 'Infinityboom TNT',
- category: 'undead',
- shop: 'Undead Essence Shop',
- materials: { 'ESSENCE_UNDEAD': 50000 }
- },
- 'ENCHANTMENT_EXPERIENCE_5': {
- name: 'Experience V Book',
- category: 'ice',
- shop: 'Ice Essence Shop',
- materials: { 'ESSENCE_ICE': 15000 }
- },
- 'PERFECTLY_CUT_DIAMOND': {
- name: 'Perfectly Cut Diamond',
- category: 'diamond',
- shop: 'Diamond Essence Shop',
- materials: { 'ESSENCE_DIAMOND': 1000, 'REFINED_DIAMOND': 2 }
- },
- 'WOODEN_BAIT': {
- name: 'Wooden Bait',
- category: 'forest',
- shop: 'Forest Essence Shop',
- materials: { 'ESSENCE_FOREST': 100 }
- }
- };
-
const cfg = {
inputMode: 'instabuy',
outputMode: 'instasell',
@@ -1307,9 +1297,7 @@
// Reset filter buttons
document.querySelectorAll('.ctrl-group .btn').forEach(btn => {
if (btn.textContent === 'All') btn.classList.add('active');
- else if (['Leg', 'Epic', 'Rare', 'Ultimate', 'Turbo', 'Regular', 'Uncommon', 'Wither', 'Undead', 'Ice', 'Diamond'].includes(btn.textContent)) {
- btn.classList.remove('active');
- }
+ else if (btn.closest('[id^="filter-group-"]')) btn.classList.remove('active');
});
const bannerIcon = document.getElementById('banner-icon');
@@ -1371,11 +1359,12 @@
const nc = craftRecipesData ? Object.keys(craftRecipesData).length : 1852;
const profStr = activeMemberData ? `Profile:
${getActiveProfileCuteName()} ` : `Database:
${nc} Recipes `;
document.getElementById('stat-meta1').innerHTML = profStr;
- document.getElementById('stat-meta2').innerHTML = activeMemberData ? `Unlocks:
Filter Active ` : `Type:
Items Folder JSONs `;
+ document.getElementById('stat-meta2').innerHTML = activeMemberData ? `Unlocks:
Filter Active ` : `Type:
Synced Recipes `;
}
function updateEssenceStats() {
- document.getElementById('stat-meta1').innerHTML = `Shops:
${Object.keys(ESSENCE_RECIPES).length} `;
+ const shops = new Set(Object.values(essenceRecipes).map(r => r.shop));
+ document.getElementById('stat-meta1').innerHTML = `Shops:
${shops.size} `;
document.getElementById('stat-meta2').innerHTML = `Type:
Essence Arbitrage `;
}
@@ -1385,21 +1374,37 @@
const userInput = document.getElementById('craft-username');
if (userInput && storedUser) userInput.value = storedUser;
+ const storedTax = localStorage.getItem('bazaar_tax_pct');
+ if (storedTax !== null) {
+ document.getElementById('tax-pct').value = storedTax;
+ BAZAAR_TAX = parseFloat(storedTax) / 100;
+ }
+
setStatus('Loading fusion-data.json, enchants.json, and craft-recipes.json...');
try {
- const [fResp, eResp, cResp] = await Promise.all([
+ const [fResp, eResp, cResp, gResp, mResp] = await Promise.all([
fetch('./fusion-data.json'),
fetch('./enchants.json'),
- fetch('./craft-recipes.json')
+ fetch('./craft-recipes.json'),
+ fetch('./game-data.json'),
+ fetch('./data-manifest.json')
]);
if (!fResp.ok) throw new Error('Could not load fusion-data.json');
fusionData = await fResp.json();
-
+
if (eResp.ok) enchantsData = await eResp.json();
if (cResp.ok) craftRecipesData = await cResp.json();
+ if (gResp.ok) {
+ const gameData = await gResp.json();
+ essenceRecipes = gameData.essenceRecipes;
+ itemNames = gameData.itemNames;
+ slayerXp = gameData.slayerXp;
+ }
+ if (mResp.ok) dataManifest = await mResp.json();
updateShardStats();
+ updateDataStatus();
applyPreset('insta');
setBatchQty(10);
@@ -1612,11 +1617,7 @@
}
function getSlayerLevelFromXp(type, xp) {
- const reqs = [0, 5, 15, 200, 1000, 5000, 20000, 100000, 400000, 1000000];
- for (let lvl = reqs.length - 1; lvl >= 1; lvl--) {
- if (xp >= reqs[lvl]) return lvl;
- }
- return 0;
+ return (slayerXp[type] || []).filter(needed => xp >= needed).length;
}
async function refreshPrices() {
@@ -1644,12 +1645,53 @@
renderCurrentModule();
setStatus(statusText());
updateTs();
+ updateDataStatus();
triggerStarPopupIfNeeded();
} catch (e) {
setStatus(`
ERROR: Bazaar fetch failed (${esc(e.message)}) `);
}
}
+ /* โโโ Data Freshness โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ */
+ function formatDataAge(ms) {
+ const hours = ms / 3600000;
+ if (hours < 1) return 'under 1h ago';
+ if (hours < 48) return `${Math.floor(hours)}h ago`;
+ return `${Math.floor(hours / 24)}d ago`;
+ }
+
+ // Recipes come from community projects, so say so when they might be behind the game.
+ function updateDataStatus() {
+ const warnings = [];
+ const label = document.getElementById('sb-data');
+
+ if (dataManifest?.syncedAt) {
+ const syncedAt = new Date(dataManifest.syncedAt);
+ const age = Date.now() - syncedAt.getTime();
+ label.textContent = `RECIPES SYNCED ${formatDataAge(age)}`;
+ label.title = `Synced ${syncedAt.toLocaleString()} for SkyBlock ${dataManifest.gameVersion}`;
+ if (age > DATA_STALE_HOURS * 3600000) {
+ warnings.push(`Recipe and fusion data last synced ${formatDataAge(age)}, so automatic updates may have stopped. Check recipes in-game before buying.`);
+ }
+ } else {
+ label.textContent = 'RECIPES SYNCED: UNKNOWN';
+ warnings.push("Couldn't check when recipe and fusion data last synced. Check recipes in-game before buying.");
+ }
+
+ if (fusionData) {
+ const known = new Set(Object.values(fusionData.shards).map(s => s.internal_id));
+ const unknown = Object.keys(productsData).filter(id => id.startsWith('SHARD_') && !known.has(id));
+ if (unknown.length) {
+ const names = unknown.map(id => cleanItemName(id.replace('SHARD_', ''))).join(', ');
+ warnings.push(`${unknown.length === 1 ? 'A shard' : `${unknown.length} shards`} on the Bazaar ${unknown.length === 1 ? 'has' : 'have'} no fusion data yet, so flips using ${unknown.length === 1 ? 'it are' : 'them are'} missing: ${names}.`);
+ }
+ }
+
+ const el = document.getElementById('data-warning');
+ el.innerHTML = warnings.map(w => `
[!] ${esc(w)}
`).join('');
+ el.style.display = warnings.length ? 'block' : 'none';
+ }
+
/* โโโ Price Builders โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ */
function parseOrderBook(prod) {
const qs = prod.quick_status || {};
@@ -2220,6 +2262,7 @@
}
function cleanItemName(productId) {
+ if (itemNames[productId]) return itemNames[productId];
const customNames = {
"BOOSTER_COOKIE": "Booster Cookie",
"RECOMBOBULATOR_3000": "Recombobulator 3000",
@@ -2964,7 +3007,7 @@
};
}
- /* โโโ Craft Flipper (Loaded directly from items/ folder recipes) โโ */
+ /* โโโ Craft Flipper (craft-recipes.json, see scripts/sync-data.mjs) โโ */
function computeCraftFlips() {
if (!Object.keys(productsData).length || !craftRecipesData) return;
const userTargetQty = Math.max(1, parseInt(cfg.batchQty, 10) || 1);
@@ -3099,13 +3142,21 @@
const isInstaBuy = (cfg.inputMode === 'instabuy');
const isInstaSell = (cfg.outputMode === 'instasell');
const out = [];
+ const categories = new Set();
- for (const [itemId, meta] of Object.entries(ESSENCE_RECIPES)) {
+ for (const [itemId, meta] of Object.entries(essenceRecipes)) {
const prod = productsData[itemId];
if (!prod) continue;
- const pOut = parseOrderBook(prod);
const mats = meta.materials;
+ // A material with no Bazaar price would count as free and fake the profit
+ if (!Object.keys(mats).every(m => productsData[m])) continue;
+ categories.add(meta.category);
+
+ const pOut = parseOrderBook(prod);
+ const outputPerPurchase = meta.count || 1;
+ const totalPurchases = Math.ceil(userTargetQty / outputPerPurchase);
+ const actualProducedUnits = totalPurchases * outputPerPurchase;
let totalMatCost = 0.0;
let maxExecutable = Infinity;
@@ -3113,10 +3164,8 @@
for (const [matId, qtyPerCraft] of Object.entries(mats)) {
const matProd = productsData[matId];
- if (!matProd) continue;
-
const pMat = parseOrderBook(matProd);
- const totalNeeded = qtyPerCraft * userTargetQty;
+ const totalNeeded = qtyPerCraft * totalPurchases;
let unitPrice = pMat.sell_offer_p;
if (isInstaBuy) {
@@ -3142,7 +3191,7 @@
let salePrice = pOut.sell_offer_p;
if (isInstaSell) {
- const sellRes = calculateWeightedInstaSell(pOut.sell_summary, userTargetQty);
+ const sellRes = calculateWeightedInstaSell(pOut.sell_summary, actualProducedUnits);
if (sellRes.filledQty > 0) salePrice = sellRes.avgPrice;
if (pOut.total_sell_avail < maxExecutable) maxExecutable = pOut.total_sell_avail;
} else {
@@ -3151,17 +3200,17 @@
if (maxExecutable === Infinity) maxExecutable = 999;
- const totalGrossRev = salePrice * userTargetQty;
+ const totalGrossRev = salePrice * actualProducedUnits;
const totalTax = totalGrossRev * BAZAAR_TAX;
const totalNetRev = totalGrossRev - totalTax;
const totalProfit = totalNetRev - totalMatCost;
if (totalProfit > 0) {
- const unitCost = totalMatCost / userTargetQty;
- const unitProfit = totalProfit / userTargetQty;
+ const unitCost = totalMatCost / actualProducedUnits;
+ const unitProfit = totalProfit / actualProducedUnits;
const roiPct = totalMatCost > 0 ? (totalProfit / totalMatCost * 100) : 0;
- const durations = calculateGenericDurations(matBreakdown, prod, userTargetQty);
+ const durations = calculateGenericDurations(matBreakdown, prod, actualProducedUnits);
// Max Total Time Filter
if (cfg.maxTotalTime !== null && durations.totalHours > cfg.maxTotalTime) {
@@ -3182,15 +3231,17 @@
salePrice,
unitProfit,
pct: roiPct,
- actualProducedUnits: userTargetQty,
+ actualProducedUnits,
maxExecutable,
- isOrderBookFullyAvailable: (maxExecutable >= userTargetQty),
+ isOrderBookFullyAvailable: (maxExecutable >= actualProducedUnits),
volume: pOut.volume,
durations
});
}
}
+ renderEssenceFilters(categories);
+
if (cfg.sortBy === 'percent') {
out.sort((a, b) => b.pct - a.pct);
} else if (cfg.sortBy === 'time') {
@@ -3203,6 +3254,19 @@
document.getElementById('sb-badge-essence').textContent = allEssenceResults.length;
}
+ // One button per essence type that currently has something sellable on the Bazaar
+ function renderEssenceFilters(categories) {
+ const group = document.getElementById('essence-filter-btns');
+ group.innerHTML = '';
+ for (const category of ['all', ...[...categories].sort()]) {
+ const btn = document.createElement('button');
+ btn.className = 'btn' + (activeFilterCategory === category ? ' active' : '');
+ btn.textContent = category === 'all' ? 'All' : category.split('_').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
+ btn.onclick = () => setFilterCategory(category, btn);
+ group.appendChild(btn);
+ }
+ }
+
/* โโโ Rendering Dispatcher โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ */
function renderCurrentModule() {
if (currentModule === 'shards') renderShardResults();
@@ -3526,9 +3590,11 @@
let reqBadgeStr = '';
if (r.requirement) {
if (r.requirement.type === 'collection') {
- reqBadgeStr = ` | ๐ Req: ${r.requirement.collection_name} ${r.requirement.tier}`;
+ reqBadgeStr = ` | ๐ Req: ${esc(r.requirement.collection_name)} ${r.requirement.tier}`;
} else if (r.requirement.type === 'slayer') {
- reqBadgeStr = ` | โ๏ธ Req: ${r.requirement.name}`;
+ reqBadgeStr = ` | โ๏ธ Req: ${esc(r.requirement.name)}`;
+ } else {
+ reqBadgeStr = ` | ๐ Req: ${esc(r.requirement.name)}`;
}
}
@@ -3559,7 +3625,7 @@
- [VERIFIED RECIPE INGREDIENTS]:
+ [RECIPE INGREDIENTS]:
${matListStr}
@@ -3604,7 +3670,7 @@
@@ -3663,7 +3729,7 @@
}
function esc(s) {
- return String(s || '').replace(/&/g, '&').replace(//g, '>');
+ return String(s || '').replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, ''');
}
function setStatus(html) {
@@ -3743,6 +3809,14 @@
recompute();
}
+ function setBazaarTax(pct) {
+ const value = parseFloat(pct);
+ if (!(value >= 0 && value <= 100)) return;
+ BAZAAR_TAX = value / 100;
+ localStorage.setItem('bazaar_tax_pct', String(value));
+ recompute();
+ }
+
function setSortBy(sort) {
cfg.sortBy = sort;
['percent', 'profit', 'fusions', 'time'].forEach(k => {
diff --git a/scripts/publish-sync.sh b/scripts/publish-sync.sh
new file mode 100644
index 0000000..aadb2dc
--- /dev/null
+++ b/scripts/publish-sync.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Publishes what sync-data.mjs wrote. Run by .github/workflows/sync-data.yml.
+#
+# publish-sync.sh publish
commit to main, which deploys the site
+# publish-sync.sh review leave the live site alone and ask for a review
+set -euo pipefail
+
+status=$1
+report=$2
+review_branch=data-sync-review
+data_paths=(fusion-data.json craft-recipes.json enchants.json game-data.json data-manifest.json shardIcons)
+
+git config user.name 'github-actions[bot]'
+git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+git add "${data_paths[@]}"
+
+if git diff --cached --quiet; then
+ echo 'Nothing to publish.'
+ exit 0
+fi
+
+open_review_pr() {
+ gh pr list --head "$review_branch" --state open --json number --jq '.[0].number // empty'
+}
+
+if [ "$status" = publish ]; then
+ git commit -F "$report"
+ git pull --rebase origin main
+ git push origin HEAD:main
+
+ # A newer good sync makes any review still waiting out of date.
+ stale_pr=$(open_review_pr)
+ if [ -n "$stale_pr" ]; then
+ gh pr close "$stale_pr" --delete-branch --comment 'Closed automatically: a newer data sync passed its checks and went live.'
+ fi
+ exit 0
+fi
+
+git switch -C "$review_branch"
+git commit -F "$report"
+git push --force origin "$review_branch"
+
+if [ -n "$(open_review_pr)" ]; then
+ echo 'Updated the review PR that was already open.'
+ exit 0
+fi
+
+title='Data sync needs review'
+if gh pr create --base main --head "$review_branch" --title "$title" --body-file "$report"; then
+ exit 0
+fi
+
+# GitHub stops Actions opening PRs unless that's enabled in the repo settings, so fall back to an issue.
+if [ -z "$(gh issue list --state open --search "\"$title\" in:title" --json number --jq '.[0].number // empty')" ]; then
+ compare="https://github.com/${GITHUB_REPOSITORY}/compare/main...${review_branch}?expand=1"
+ {
+ echo "The new data was held back from the live site. [Review and open a PR]($compare)."
+ echo
+ cat "$report"
+ } | gh issue create --title "$title" --body-file -
+fi
diff --git a/scripts/sync-data.mjs b/scripts/sync-data.mjs
new file mode 100644
index 0000000..f73e0ed
--- /dev/null
+++ b/scripts/sync-data.mjs
@@ -0,0 +1,518 @@
+#!/usr/bin/env node
+// Rebuilds the site's static game data from its upstream sources, checks it, and
+// decides whether it is safe to publish. See "data sources" in the README.
+//
+// node scripts/sync-data.mjs write the data files
+// node scripts/sync-data.mjs --dry-run build and validate only, write nothing
+// node scripts/sync-data.mjs --report also write a summary to
+//
+// Status (also written to $GITHUB_OUTPUT): publish, review (safe to look at, but
+// suspicious enough that a human should approve it) or skip. Exits 1 if the data is
+// broken, which leaves the last good files in place.
+
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const NEU_REPO = 'https://github.com/NotEnoughUpdates/NotEnoughUpdates-REPO';
+const PARSER_REPO = 'https://github.com/Campionnn/SkyShards-Parser';
+const ICONS_URL = 'https://raw.githubusercontent.com/Campionnn/SkyShards/master/public/shardIcons';
+const HYPIXEL_API = 'https://api.hypixel.net/v2';
+
+// How much may disappear in one sync before a human has to approve it.
+const MAX_FUSION_LOSS = 0.1;
+const MAX_CRAFT_LOSS = 0.05;
+const MAX_CRAFT_CHANGED = 0.05;
+const MAX_ESSENCE_LOSS = 0.25;
+const MAX_ENCHANT_LOSS = 0.05;
+const MAX_FUSE_AMOUNT_CHANGES = 10;
+// Publish at least this often even with no changes, so the site can tell that
+// syncing is still alive and GitHub doesn't pause the schedule for inactivity.
+const HEARTBEAT_HOURS = 23;
+
+const SLOTS = ['A1', 'A2', 'A3', 'B1', 'B2', 'B3', 'C1', 'C2', 'C3'];
+const SLAYER_TYPES = { Zombie: 'zombie', Spider: 'spider', Wolf: 'wolf', Enderman: 'enderman', Blaze: 'blaze', Vampire: 'vampire' };
+const ROMAN = { I: 1, II: 2, III: 3, IV: 4, V: 5, VI: 6, VII: 7, VIII: 8, IX: 9, X: 10, XI: 11, XII: 12 };
+// Slayer unlocks that no source publishes. Only used when NEU says nothing about the item.
+const SLAYER_FALLBACKS = {
+ GOLDEN_TOOTH: ['wolf', 2],
+ MASTIFF_BOOTS: ['wolf', 4],
+ MASTIFF_CHESTPLATE: ['wolf', 4],
+ MASTIFF_HELMET: ['wolf', 4],
+ MASTIFF_LEGGINGS: ['wolf', 4],
+ NULL_OVOID: ['enderman', 3],
+ REAPER_SCYTHE: ['zombie', 7],
+ REVENANT_VISCERA: ['zombie', 2],
+ SOULFLOW_ENGINE: ['enderman', 4],
+ TARANTULA_BOOTS: ['spider', 4],
+ TARANTULA_SILK: ['spider', 2],
+};
+
+const stripColor = s => String(s || '').replace(/ยง./g, '').trim();
+const capitalize = s => s.charAt(0).toUpperCase() + s.slice(1);
+const romanOf = n => Object.keys(ROMAN).find(k => ROMAN[k] === n) || String(n);
+const stable = v => JSON.stringify(v, (_, x) => (x && typeof x === 'object' && !Array.isArray(x)
+ ? Object.fromEntries(Object.entries(x).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))
+ : x));
+
+/** NEU writes "INK_SACK-3" and "EXPERIENCE;5" where the Bazaar says "INK_SACK:3" and "ENCHANTMENT_EXPERIENCE_5". */
+export function toBazaarId(neuId) {
+ const enchant = neuId.match(/^(.+);(\d+)$/);
+ if (enchant) return `ENCHANTMENT_${enchant[1]}_${enchant[2]}`;
+ return neuId.replace(/-(\d+)$/, ':$1');
+}
+
+/** Every book's display name is just "Enchanted Book", so spell the enchant out from its id instead. */
+function itemName(neuId, item) {
+ const displayName = stripColor(item?.displayname);
+ const book = neuId.match(/^(.+);(\d+)$/);
+ if (!book || displayName !== 'Enchanted Book') return displayName;
+ const words = book[1].split('_').map(w => capitalize(w.toLowerCase())).join(' ');
+ return `${words} ${romanOf(parseInt(book[2], 10))}`;
+}
+
+function parseIngredient(entry) {
+ const i = entry.lastIndexOf(':');
+ return i < 0 ? [toBazaarId(entry), 1] : [toBazaarId(entry.slice(0, i)), parseFloat(entry.slice(i + 1))];
+}
+
+function gridMaterials(recipe) {
+ const materials = {};
+ for (const slot of SLOTS) {
+ if (!recipe[slot]) continue;
+ const [id, qty] = parseIngredient(recipe[slot]);
+ materials[id] = (materials[id] || 0) + qty;
+ }
+ return materials;
+}
+
+/** Every crafting grid NEU lists for an item. Recipes can be in .recipe, .recipes[], or both. */
+function craftingVariants(neuId, item) {
+ const grids = [...(item.recipe ? [item.recipe] : []), ...(item.recipes || []).filter(r => r.type === 'crafting' || !r.type)];
+ const id = toBazaarId(neuId);
+ return grids
+ .filter(r => !r.overrideOutputId || r.overrideOutputId === neuId)
+ .map(r => ({ materials: gridMaterials(r), count: r.count || 1 }))
+ .filter(v => Object.keys(v.materials).length && !v.materials[id]);
+}
+
+/** Collection unlocks by lowercased item name, from Hypixel's own "- Recipe" unlock text. */
+export function buildUnlockMap(collections) {
+ const unlocks = {};
+ for (const category of Object.values(collections.collections)) {
+ for (const [collectionId, entry] of Object.entries(category.items)) {
+ for (const tier of entry.tiers) {
+ for (const text of tier.unlocks || []) {
+ if (!/Recipes?$/i.test(text)) continue;
+ unlocks[text.replace(/\s+Recipes?$/i, '').trim().toLowerCase()] ??= collectionRequirement(collectionId, entry, tier);
+ }
+ }
+ }
+ }
+ return unlocks;
+}
+
+function collectionRequirement(collectionId, entry, tier) {
+ return {
+ type: 'collection',
+ collection_id: collectionId,
+ collection_name: entry.name,
+ tier: tier.tier,
+ tier_code: `${collectionId}_${tier.tier}`,
+ amount_required: tier.amountRequired,
+ };
+}
+
+function slayerRequirement(type, level) {
+ return { type: 'slayer', slayer_type: type, level, name: `${capitalize(type)} Slayer ${level}` };
+}
+
+/** Hypixel's unlock list wins, because NEU's "Requires:" text is often a tier off. */
+export function buildRequirement(id, name, item, unlocks, collections) {
+ const official = unlocks[name.toLowerCase()];
+ if (official) return official;
+
+ const text = stripColor(item.crafttext).replace(/^Requires:?\s*/, '');
+ const slayer = text.match(/^(\w+) Slayer (\d+|[IVX]+)$/);
+ if (slayer && SLAYER_TYPES[slayer[1]]) return slayerRequirement(SLAYER_TYPES[slayer[1]], ROMAN[slayer[2]] || parseInt(slayer[2], 10));
+
+ const tiered = text.match(/^(.+?) ([IVX]+)$/);
+ if (tiered && ROMAN[tiered[2]]) {
+ for (const category of Object.values(collections.collections)) {
+ for (const [collectionId, entry] of Object.entries(category.items)) {
+ const tier = entry.tiers.find(t => t.tier === ROMAN[tiered[2]]);
+ if (entry.name === tiered[1] && tier) return collectionRequirement(collectionId, entry, tier);
+ }
+ }
+ }
+
+ if (text) return { type: 'other', name: text };
+ if (SLAYER_FALLBACKS[id]) return slayerRequirement(...SLAYER_FALLBACKS[id]);
+ return null;
+}
+
+/** Hypixel leaves the tier off common items, and doesn't list some vanilla ones at all. */
+function itemTier(id, item, hypixelItems) {
+ if (hypixelItems[id]) return hypixelItems[id].tier || 'COMMON';
+ const lastLore = stripColor((item.lore || []).at(-1));
+ const match = lastLore.match(/^(?:a )?(VERY SPECIAL|SPECIAL|SUPREME|DIVINE|MYTHIC|LEGENDARY|EPIC|RARE|UNCOMMON|COMMON)\b/);
+ return match ? match[1].replace(' ', '_') : 'COMMON';
+}
+
+export function buildCraftRecipes(neuItems, { collections, hypixelItems, bazaar }) {
+ const unlocks = buildUnlockMap(collections);
+ const crafts = {};
+ for (const neuId of Object.keys(neuItems).sort()) {
+ const item = neuItems[neuId];
+ // "NAME;n" is either a book or a pet. Pets never reach the Bazaar and their ids could clash with a book's.
+ if (neuId.includes(';') && stripColor(item.displayname) !== 'Enchanted Book') continue;
+ const variants = craftingVariants(neuId, item);
+ if (!variants.length) continue;
+ // Wood types and the like give several valid recipes. Prefer one you can buy all of.
+ const chosen = variants.find(v => Object.keys(v.materials).every(m => bazaar[m])) || variants[0];
+ const id = toBazaarId(neuId);
+ const name = itemName(neuId, item);
+ crafts[id] = {
+ item_id: id,
+ name,
+ count: chosen.count,
+ tier: itemTier(id, item, hypixelItems),
+ materials: chosen.materials,
+ requirement: buildRequirement(neuId, name, item, unlocks, collections),
+ };
+ }
+ return crafts;
+}
+
+/** What essence buys, from the NPC shop listings in NEU. */
+export function buildEssenceRecipes(neuItems) {
+ const recipes = {};
+ for (const npcId of Object.keys(neuItems).sort()) {
+ const npc = neuItems[npcId];
+ for (const row of npc.recipes || []) {
+ if (row.type !== 'npc_shop' || !(row.cost || []).some(c => c.startsWith('ESSENCE_'))) continue;
+ const [neuOutput, count] = String(row.result || '').split(':');
+ if (!neuOutput) continue;
+ const id = toBazaarId(neuOutput);
+ if (recipes[id]) continue;
+ const essence = row.cost.find(c => c.startsWith('ESSENCE_')).split(':')[0];
+ recipes[id] = {
+ name: itemName(neuOutput, neuItems[neuOutput]) || id,
+ category: essence.replace('ESSENCE_', '').toLowerCase(),
+ shop: stripColor(npc.displayname).replace(/\s*\((Rift )?NPC\)$/, ''),
+ count: parseInt(count, 10) || 1,
+ materials: Object.fromEntries(row.cost.map(parseIngredient)),
+ };
+ }
+ }
+ return recipes;
+}
+
+/** Real names for Bazaar products, so "INK_SACK:3" shows as Cocoa Beans. Books keep the site's own formatting. */
+export function buildItemNames(bazaar, hypixelItems, neuItems) {
+ const byBazaarId = Object.fromEntries(Object.entries(neuItems).map(([id, item]) => [toBazaarId(id), item]));
+ const names = {};
+ for (const id of Object.keys(bazaar).sort()) {
+ if (id.startsWith('ENCHANTMENT_')) continue;
+ const name = hypixelItems[id]?.name || stripColor(byBazaarId[id]?.displayname);
+ if (name) names[id] = stripColor(name);
+ }
+ return names;
+}
+
+/** Fusions ignoring input order, since A+B and B+A are the same recipe. */
+export function uniqueFusions(fusion) {
+ const unique = new Set();
+ for (const [output, byQty] of Object.entries(fusion.recipes)) {
+ for (const [qty, pairs] of Object.entries(byQty)) {
+ for (const [a, b] of pairs) unique.add(`${output}|${qty}|${[a, b].sort().join('|')}`);
+ }
+ }
+ return unique;
+}
+
+/**
+ * Third-party text ends up in the page, and that page's localStorage holds people's
+ * API keys. No real game name needs these characters. Double quotes are allowed in
+ * values because real names use them ("Fragranced" Brown Mushroom "Paste").
+ */
+export function findUnsafeText(value, where = 'data') {
+ const found = [];
+ const visit = (v, label) => {
+ if (typeof v === 'string') {
+ if (/[<>`]/.test(v)) found.push(`${label} has unsafe text: ${v.slice(0, 80)}`);
+ } else if (v && typeof v === 'object') {
+ for (const [k, child] of Object.entries(v)) {
+ if (/[<>"`]/.test(k)) found.push(`${label} has an unsafe key: ${k.slice(0, 80)}`);
+ visit(child, `${label}.${k}`);
+ }
+ }
+ };
+ visit(value, where);
+ return found;
+}
+
+/** Problems that make data unusable. Any of these stops the sync. */
+export function findErrors({ fusion, crafts, essence, enchants, slayerXp, itemNames }) {
+ const errors = [
+ ...findUnsafeText(fusion, 'fusion'),
+ ...findUnsafeText(crafts, 'crafts'),
+ ...findUnsafeText(essence, 'essence'),
+ ...findUnsafeText(enchants, 'enchants'),
+ ...findUnsafeText(itemNames, 'itemNames'),
+ ];
+ const shardEntries = Object.entries(fusion?.shards || {});
+ if (shardEntries.length < 100) errors.push(`only ${shardEntries.length} shards in fusion data`);
+ // Codes, rarities, tiers and categories go straight into HTML attributes, so they get a strict format.
+ for (const [code, s] of shardEntries) {
+ if (!/^[A-Z]\d+$/.test(code) || !s.name || !/^[a-z]+$/i.test(s.rarity || '') || !(s.fuse_amount > 0) || !/^SHARD_/.test(s.internal_id || '')) errors.push(`malformed shard ${code}: ${JSON.stringify(s)}`);
+ }
+ for (const [output, byQty] of Object.entries(fusion?.recipes || {})) {
+ if (!fusion.shards?.[output]) errors.push(`fusion output ${output} is not a known shard`);
+ for (const [qty, pairs] of Object.entries(byQty)) {
+ if (!(parseInt(qty, 10) > 0)) errors.push(`fusion ${output} has bad quantity ${qty}`);
+ for (const pair of pairs) {
+ if (pair.length !== 2 || !pair.every(code => fusion.shards?.[code])) errors.push(`fusion ${output} has unknown inputs ${JSON.stringify(pair)}`);
+ }
+ }
+ }
+ if (Object.keys(crafts || {}).length < 500) errors.push(`only ${Object.keys(crafts || {}).length} craft recipes`);
+ for (const [id, c] of Object.entries(crafts || {})) {
+ const quantities = Object.values(c.materials || {});
+ if (!c.name || !/^[A-Z_]+$/.test(c.tier || '') || !(c.count > 0) || !quantities.length || !quantities.every(q => q > 0)) errors.push(`malformed craft ${id}: ${JSON.stringify(c)}`);
+ }
+ if (!Object.keys(essence || {}).length) errors.push('no essence shop recipes');
+ for (const [id, e] of Object.entries(essence || {})) {
+ const quantities = Object.values(e.materials || {});
+ if (!/^[a-z_]+$/.test(e.category || '') || !(e.count > 0) || !quantities.length || !quantities.every(q => q > 0)) errors.push(`malformed essence recipe ${id}: ${JSON.stringify(e)}`);
+ }
+ for (const key of ['enchants', 'enchant_pools', 'enchants_xp_cost', 'max_xp_table_levels']) {
+ if (!enchants?.[key]) errors.push(`enchants.json has no ${key}`);
+ }
+ for (const type of Object.values(SLAYER_TYPES)) {
+ if (!slayerXp?.[type]?.length) errors.push(`no slayer XP table for ${type}`);
+ }
+ if (Object.keys(itemNames || {}).length < 100) errors.push('item name table looks empty');
+ return errors;
+}
+
+/** Changes that are possible but unusual enough that a person should look before they go live. */
+export function findReviewReasons(previous, next) {
+ const reasons = [];
+ const lostShare = (before, after) => (before ? (before - after) / before : 0);
+
+ if (previous.fusion) {
+ const before = uniqueFusions(previous.fusion);
+ const after = uniqueFusions(next.fusion);
+ const lost = [...before].filter(f => !after.has(f)).length;
+ if (before.size && lost / before.size > MAX_FUSION_LOSS) reasons.push(`${lost.toLocaleString('en')} of ${before.size.toLocaleString('en')} fusions would disappear`);
+ const removed = Object.keys(previous.fusion.shards).filter(code => !next.fusion.shards[code]);
+ if (removed.length) reasons.push(`shards removed: ${removed.map(code => previous.fusion.shards[code].name).join(', ')}`);
+ const fuseChanges = Object.keys(previous.fusion.shards).filter(code => next.fusion.shards[code] && next.fusion.shards[code].fuse_amount !== previous.fusion.shards[code].fuse_amount);
+ if (fuseChanges.length > MAX_FUSE_AMOUNT_CHANGES) reasons.push(`${fuseChanges.length} shards changed fuse amount`);
+ }
+ if (previous.crafts) {
+ const ids = Object.keys(previous.crafts);
+ const gone = ids.filter(id => !next.crafts[id]);
+ if (gone.length / ids.length > MAX_CRAFT_LOSS) reasons.push(`${gone.length} of ${ids.length} craft recipes would disappear`);
+ const changed = ids.filter(id => next.crafts[id] && (stable(next.crafts[id].materials) !== stable(previous.crafts[id].materials) || next.crafts[id].count !== previous.crafts[id].count));
+ if (changed.length / ids.length > MAX_CRAFT_CHANGED) reasons.push(`${changed.length} of ${ids.length} craft recipes changed ingredients`);
+ }
+ if (previous.essence && lostShare(Object.keys(previous.essence).length, Object.keys(next.essence).filter(id => previous.essence[id]).length) > MAX_ESSENCE_LOSS) {
+ reasons.push('many essence shop recipes would disappear');
+ }
+ if (previous.enchants?.enchants_xp_cost) {
+ const before = Object.keys(previous.enchants.enchants_xp_cost);
+ const kept = before.filter(k => next.enchants.enchants_xp_cost?.[k] !== undefined);
+ if (lostShare(before.length, kept.length) > MAX_ENCHANT_LOSS) reasons.push(`${before.length - kept.length} enchants would disappear`);
+ }
+ return reasons;
+}
+
+function summarize(previous, next) {
+ const lines = [];
+ const count = o => Object.keys(o || {}).length;
+ if (previous.fusion) {
+ const before = uniqueFusions(previous.fusion), after = uniqueFusions(next.fusion);
+ const added = [...after].filter(f => !before.has(f)).length, removed = [...before].filter(f => !after.has(f)).length;
+ if (added || removed) lines.push(`- fusions: ${after.size.toLocaleString('en')} total, ${added.toLocaleString('en')} added, ${removed.toLocaleString('en')} removed`);
+ const newShards = Object.keys(next.fusion.shards).filter(code => !previous.fusion.shards[code]);
+ if (newShards.length) lines.push(`- new shards: ${newShards.map(code => next.fusion.shards[code].name).join(', ')}`);
+ const fuse = Object.keys(next.fusion.shards).filter(code => previous.fusion.shards[code] && previous.fusion.shards[code].fuse_amount !== next.fusion.shards[code].fuse_amount);
+ if (fuse.length) lines.push(`- fuse amount changed: ${fuse.map(code => `${next.fusion.shards[code].name} ${previous.fusion.shards[code].fuse_amount} -> ${next.fusion.shards[code].fuse_amount}`).join(', ')}`);
+ }
+ if (stable(previous.crafts) !== stable(next.crafts)) {
+ const ids = Object.keys(previous.crafts || {});
+ const gone = ids.filter(id => !next.crafts[id]);
+ const added = Object.keys(next.crafts).filter(id => !previous.crafts?.[id]);
+ const changed = ids.filter(id => next.crafts[id] && stable(next.crafts[id]) !== stable(previous.crafts[id]));
+ const sample = list => list.slice(0, 10).join(', ') + (list.length > 10 ? ', ...' : '');
+ lines.push(`- crafts: ${count(next.crafts)} total, ${added.length} added, ${changed.length} changed, ${gone.length} removed`);
+ if (changed.length) lines.push(` - changed: ${sample(changed)}`);
+ if (added.length) lines.push(` - added: ${sample(added)}`);
+ if (gone.length) lines.push(` - removed: ${sample(gone)}`);
+ }
+ if (stable(previous.gameData) !== stable(next.gameData)) lines.push('- essence shop, item names or slayer XP changed');
+ if (stable(previous.enchants) !== stable(next.enchants)) lines.push('- enchant data changed');
+ return lines;
+}
+
+async function fetchRetry(url) {
+ let lastError;
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ try {
+ const resp = await fetch(url, { headers: { 'User-Agent': 'lyns-flipper-data-sync' } });
+ if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+ return resp;
+ } catch (e) {
+ lastError = e;
+ if (attempt < 3) await new Promise(r => setTimeout(r, attempt * 5000));
+ }
+ }
+ throw new Error(`${url}: ${lastError.message}`);
+}
+const fetchJson = async url => (await fetchRetry(url)).json();
+
+function latestCommit(repo) {
+ return execFileSync('git', ['ls-remote', repo, 'HEAD'], { encoding: 'utf8' }).split(/\s/)[0];
+}
+
+async function downloadNeu(commit) {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'neu-'));
+ const resp = await fetchRetry(`https://codeload.github.com/NotEnoughUpdates/NotEnoughUpdates-REPO/tar.gz/${commit}`);
+ fs.writeFileSync(path.join(dir, 'neu.tar.gz'), Buffer.from(await resp.arrayBuffer()));
+ execFileSync('tar', ['-xzf', 'neu.tar.gz'], { cwd: dir });
+ const root = path.join(dir, `NotEnoughUpdates-REPO-${commit}`);
+ const items = {};
+ for (const file of fs.readdirSync(path.join(root, 'items'))) {
+ if (file.endsWith('.json')) items[file.slice(0, -5)] = JSON.parse(fs.readFileSync(path.join(root, 'items', file), 'utf8'));
+ }
+ const readConstant = name => fs.readFileSync(path.join(root, 'constants', name), 'utf8');
+ return { dir, items, enchantsText: readConstant('enchants.json'), leveling: JSON.parse(readConstant('leveling.json')) };
+}
+
+const readExisting = file => {
+ try { return JSON.parse(fs.readFileSync(path.join(ROOT, file), 'utf8')); } catch { return undefined; }
+};
+
+async function downloadMissingIcons(fusion, dryRun) {
+ const missing = Object.keys(fusion.shards).filter(code => !fs.existsSync(path.join(ROOT, 'shardIcons', `${code}.png`)));
+ if (dryRun) return missing;
+ const downloaded = [];
+ for (const code of missing) {
+ try {
+ const resp = await fetchRetry(`${ICONS_URL}/${code}.png`);
+ fs.writeFileSync(path.join(ROOT, 'shardIcons', `${code}.png`), Buffer.from(await resp.arrayBuffer()));
+ downloaded.push(code);
+ } catch (e) {
+ console.warn(`no icon for ${code} (the site just hides it): ${e.message}`);
+ }
+ }
+ return downloaded;
+}
+
+async function main() {
+ const args = process.argv.slice(2);
+ const dryRun = args.includes('--dry-run');
+ const reportFile = args.includes('--report') ? args[args.indexOf('--report') + 1] : null;
+
+ const neuCommit = latestCommit(NEU_REPO), parserCommit = latestCommit(PARSER_REPO);
+ console.log(`NEU ${neuCommit}\nparser ${parserCommit}`);
+ const neu = await downloadNeu(neuCommit);
+ const [fusionResp, collections, itemsResource, bazaarResource] = await Promise.all([
+ fetchRetry(`https://raw.githubusercontent.com/Campionnn/SkyShards-Parser/${parserCommit}/dist/fusion-data.json`),
+ fetchJson(`${HYPIXEL_API}/resources/skyblock/collections`),
+ fetchJson(`${HYPIXEL_API}/resources/skyblock/items`),
+ fetchJson(`${HYPIXEL_API}/skyblock/bazaar`),
+ ]);
+ const fusionText = await fusionResp.text();
+ const hypixelItems = Object.fromEntries(itemsResource.items.map(i => [i.id, i]));
+ const bazaar = bazaarResource.products;
+
+ const next = {
+ fusion: JSON.parse(fusionText),
+ crafts: buildCraftRecipes(neu.items, { collections, hypixelItems, bazaar }),
+ enchants: JSON.parse(neu.enchantsText),
+ };
+ next.gameData = {
+ essenceRecipes: buildEssenceRecipes(neu.items),
+ itemNames: buildItemNames(bazaar, hypixelItems, neu.items),
+ slayerXp: neu.leveling.slayer_xp,
+ };
+ const previous = {
+ fusion: readExisting('fusion-data.json'),
+ crafts: readExisting('craft-recipes.json'),
+ enchants: readExisting('enchants.json'),
+ gameData: readExisting('game-data.json'),
+ };
+ previous.essence = previous.gameData?.essenceRecipes;
+ next.essence = next.gameData.essenceRecipes;
+ const manifest = readExisting('data-manifest.json');
+
+ const report = reportLines => {
+ const text = reportLines.join('\n') + '\n';
+ console.log(`\n${text}`);
+ if (reportFile) fs.writeFileSync(reportFile, text);
+ };
+ const finish = status => {
+ console.log(`status: ${status}`);
+ if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `status=${status}\n`);
+ fs.rmSync(neu.dir, { recursive: true, force: true });
+ };
+
+ const errors = findErrors({ ...next, slayerXp: next.gameData.slayerXp, itemNames: next.gameData.itemNames });
+ if (errors.length) {
+ report(['sync game data: failed validation', '', ...errors.slice(0, 50).map(e => `- ${e}`)]);
+ finish('failed');
+ process.exitCode = 1;
+ return;
+ }
+
+ const reasons = findReviewReasons(previous, next);
+ const changes = summarize(previous, next);
+ const missingIcons = await downloadMissingIcons(next.fusion, dryRun);
+ const dataChanged = ['fusion', 'crafts', 'enchants', 'gameData'].some(k => stable(previous[k]) !== stable(next[k])) || missingIcons.length > 0;
+ const syncAgeHours = manifest?.syncedAt ? (Date.now() - new Date(manifest.syncedAt).getTime()) / 3600000 : Infinity;
+
+ const knownShardIds = new Set(Object.values(next.fusion.shards).map(s => s.internal_id));
+ const unknownShards = Object.keys(bazaar).filter(id => id.startsWith('SHARD_') && !knownShardIds.has(id));
+
+ let status = 'skip';
+ if (dataChanged) status = reasons.length ? 'review' : 'publish';
+ else if (syncAgeHours >= HEARTBEAT_HOURS) status = 'publish';
+
+ const lines = [dataChanged ? 'sync game data' : 'sync game data: no changes', ''];
+ if (reasons.length) lines.push('Needs review because:', ...reasons.map(r => `- ${r}`), '');
+ lines.push(...(changes.length ? changes : ['No data changed.']));
+ if (unknownShards.length) lines.push('', `On the Bazaar but not in the fusion data yet: ${unknownShards.join(', ')}`);
+ lines.push('', `NEU ${neuCommit.slice(0, 7)}, SkyShards-Parser ${parserCommit.slice(0, 7)}, SkyBlock ${collections.version}`);
+ report(lines);
+
+ if (!dryRun && status !== 'skip') {
+ const writeJson = (file, value) => fs.writeFileSync(path.join(ROOT, file), `${JSON.stringify(value, null, 2)}\n`);
+ fs.writeFileSync(path.join(ROOT, 'fusion-data.json'), fusionText);
+ fs.writeFileSync(path.join(ROOT, 'enchants.json'), neu.enchantsText);
+ writeJson('craft-recipes.json', next.crafts);
+ writeJson('game-data.json', next.gameData);
+ writeJson('data-manifest.json', {
+ syncedAt: new Date().toISOString(),
+ gameVersion: collections.version,
+ sources: {
+ fusions: `${PARSER_REPO}/tree/${parserCommit}`,
+ recipes: `${NEU_REPO}/tree/${neuCommit}`,
+ },
+ });
+ }
+ finish(status);
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1]).href) {
+ main().catch(e => {
+ console.error(e);
+ if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, 'status=failed\n');
+ process.exitCode = 1;
+ });
+}
diff --git a/scripts/sync-data.test.mjs b/scripts/sync-data.test.mjs
new file mode 100644
index 0000000..157f119
--- /dev/null
+++ b/scripts/sync-data.test.mjs
@@ -0,0 +1,243 @@
+// Run with: node --test scripts/sync-data.test.mjs
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import {
+ buildCraftRecipes,
+ buildEssenceRecipes,
+ buildItemNames,
+ buildRequirement,
+ buildUnlockMap,
+ findErrors,
+ findReviewReasons,
+ toBazaarId,
+ uniqueFusions,
+} from './sync-data.mjs';
+
+const collections = {
+ collections: {
+ MINING: {
+ items: {
+ MITHRIL_ORE: {
+ name: 'Mithril',
+ tiers: [
+ { tier: 3, amountRequired: 500, unlocks: ['Enchanted Mithril Recipe'] },
+ { tier: 4, amountRequired: 1000, unlocks: ['+4 SkyBlock XP'] },
+ ],
+ },
+ },
+ },
+ },
+};
+const unlocks = buildUnlockMap(collections);
+const build = (items, bazaar = {}, hypixelItems = {}) => buildCraftRecipes(items, { collections, hypixelItems, bazaar });
+
+// Two shards plus padding, so findErrors' minimum size checks pass.
+function validData() {
+ const shards = {};
+ for (let i = 0; i < 120; i++) shards[`C${i}`] = { name: `Shard ${i}`, rarity: 'common', fuse_amount: 2, internal_id: `SHARD_${i}` };
+ const crafts = {};
+ for (let i = 0; i < 600; i++) crafts[`ITEM_${i}`] = { name: `Item ${i}`, count: 1, tier: 'COMMON', materials: { STICK: 2 }, requirement: null };
+ const itemNames = {};
+ for (let i = 0; i < 120; i++) itemNames[`ITEM_${i}`] = `Item ${i}`;
+ const xp = [1, 2];
+ return {
+ fusion: { shards, recipes: { C0: { 2: [['C1', 'C2'], ['C3', 'C4']] } } },
+ crafts,
+ essence: { WOODEN_BAIT: { category: 'forest', count: 1, materials: { ESSENCE_FOREST: 100 } } },
+ enchants: { enchants: {}, enchant_pools: [], enchants_xp_cost: { sharpness: [] }, max_xp_table_levels: {} },
+ slayerXp: { zombie: xp, spider: xp, wolf: xp, enderman: xp, blaze: xp, vampire: xp },
+ itemNames,
+ };
+}
+
+test('toBazaarId converts NEU ids to Bazaar ids', () => {
+ assert.equal(toBazaarId('INK_SACK-3'), 'INK_SACK:3');
+ assert.equal(toBazaarId('EXPERIENCE;5'), 'ENCHANTMENT_EXPERIENCE_5');
+ assert.equal(toBazaarId('ENCHANTED_DIAMOND'), 'ENCHANTED_DIAMOND');
+});
+
+test('crafts add up ingredients across slots and keep the output count', () => {
+ const { HOTSPOT_BAIT } = build({ HOTSPOT_BAIT: { displayname: 'ยงfHotspot Bait', recipe: { A1: 'CRAB:2', A2: 'CRAB:3', B1: 'INK_SACK-3:1', count: 32 } } });
+ assert.deepEqual(HOTSPOT_BAIT.materials, { CRAB: 5, 'INK_SACK:3': 1 });
+ assert.equal(HOTSPOT_BAIT.count, 32);
+ assert.equal(HOTSPOT_BAIT.name, 'Hotspot Bait');
+});
+
+test('crafts read recipes that only exist in .recipes[], and ignore other recipe types', () => {
+ const crafts = build({
+ A: { displayname: 'A', recipes: [{ type: 'crafting', A1: 'STICK:1' }] },
+ B: { displayname: 'B', recipes: [{ type: 'forge', inputs: ['STICK:1'] }, { type: 'npc_shop', cost: ['STICK:1'] }] },
+ });
+ assert.deepEqual(crafts.A.materials, { STICK: 1 });
+ assert.equal(crafts.B, undefined);
+});
+
+test('crafts prefer the recipe variant you can actually buy', () => {
+ const items = { CHARCOAL: { displayname: 'Charcoal', recipes: [{ type: 'crafting', A1: 'LOG-9:1' }, { type: 'crafting', A1: 'LOG:1' }] } };
+ assert.deepEqual(build(items, { LOG: {} }).CHARCOAL.materials, { LOG: 1 });
+ assert.deepEqual(build(items, {}).CHARCOAL.materials, { 'LOG:9': 1 });
+});
+
+test('crafts skip recipes for a different output and recipes that consume their own output', () => {
+ const crafts = build({
+ A: { displayname: 'A', recipes: [{ type: 'crafting', A1: 'STICK:1', overrideOutputId: 'OTHER' }] },
+ B: { displayname: 'B', recipe: { A1: 'B:1', A2: 'STICK:1' } },
+ });
+ assert.deepEqual(crafts, {});
+});
+
+test('crafts use Hypixel rarity, treating a missing tier as common', () => {
+ const crafts = build(
+ {
+ A: { displayname: 'A', recipe: { A1: 'STICK:1' } },
+ B: { displayname: 'B', recipe: { A1: 'STICK:1' } },
+ C: { displayname: 'C', lore: ['ยง9ยงlRARE'], recipe: { A1: 'STICK:1' } },
+ },
+ {},
+ { A: { tier: 'LEGENDARY' }, B: {} },
+ );
+ assert.deepEqual([crafts.A.tier, crafts.B.tier, crafts.C.tier], ['LEGENDARY', 'COMMON', 'RARE']);
+});
+
+test('book names come from the enchant id', () => {
+ const crafts = build({ 'ULTIMATE_WISE;5': { displayname: 'ยงfEnchanted Book', recipe: { A1: 'STICK:1' } } });
+ assert.equal(crafts.ENCHANTMENT_ULTIMATE_WISE_5.name, 'Ultimate Wise V');
+});
+
+test('pets are left out rather than treated as books', () => {
+ assert.deepEqual(build({ 'BAT;0': { displayname: 'ยง7[Lvl 1] Bat', recipe: { A1: 'STICK:1' } } }), {});
+});
+
+test('requirements prefer the Hypixel unlock list over NEU text', () => {
+ const req = buildRequirement('X', 'Enchanted Mithril', { crafttext: 'Requires: Mithril IV' }, unlocks, collections);
+ assert.equal(req.tier, 3);
+ assert.equal(req.tier_code, 'MITHRIL_ORE_3');
+ assert.equal(req.amount_required, 500);
+});
+
+test('requirements fall back to NEU text in all its spellings', () => {
+ const fromText = crafttext => buildRequirement('X', 'Thing', { crafttext }, unlocks, collections);
+ assert.equal(fromText('Requires: Mithril IV').tier_code, 'MITHRIL_ORE_4');
+ assert.equal(fromText('Requires Mithril IV').tier_code, 'MITHRIL_ORE_4');
+ assert.deepEqual(fromText('Requires: Vampire Slayer 2'), { type: 'slayer', slayer_type: 'vampire', level: 2, name: 'Vampire Slayer 2' });
+ assert.equal(fromText('Blaze Slayer III').level, 3);
+ assert.deepEqual(fromText('ยงaRequires: HotM 5'), { type: 'other', name: 'HotM 5' });
+ assert.equal(fromText(''), null);
+});
+
+test('hand-written slayer requirements only apply when NEU says nothing', () => {
+ assert.equal(buildRequirement('GOLDEN_TOOTH', 'Golden Tooth', { crafttext: '' }, unlocks, collections).name, 'Wolf Slayer 2');
+ assert.equal(buildRequirement('GOLDEN_TOOTH', 'Golden Tooth', { crafttext: 'Requires: Wolf Slayer 3' }, unlocks, collections).level, 3);
+});
+
+test('essence recipes come from NPC shops that take essence', () => {
+ const recipes = buildEssenceRecipes({
+ ROSEMARY_NPC: {
+ displayname: 'ยงfRosemary (NPC)',
+ recipes: [
+ { type: 'npc_shop', cost: ['ESSENCE_FOREST:100'], result: 'WOODEN_BAIT:4' },
+ { type: 'npc_shop', cost: ['ESSENCE_ICE:15000', 'REFINED_DIAMOND:2'], result: 'EXPERIENCE;5' },
+ { type: 'npc_shop', cost: ['SKYBLOCK_COIN:100'], result: 'STICK:1' },
+ ],
+ },
+ WOODEN_BAIT: { displayname: 'ยงfWooden Bait' },
+ 'EXPERIENCE;5': { displayname: 'Enchanted Book', lore: ['Combinable in Anvil'] },
+ });
+ assert.deepEqual(Object.keys(recipes).sort(), ['ENCHANTMENT_EXPERIENCE_5', 'WOODEN_BAIT']);
+ assert.deepEqual(recipes.WOODEN_BAIT, { name: 'Wooden Bait', category: 'forest', shop: 'Rosemary', count: 4, materials: { ESSENCE_FOREST: 100 } });
+ assert.equal(recipes.ENCHANTMENT_EXPERIENCE_5.name, 'Experience V');
+ assert.deepEqual(recipes.ENCHANTMENT_EXPERIENCE_5.materials, { ESSENCE_ICE: 15000, REFINED_DIAMOND: 2 });
+});
+
+test('item names prefer Hypixel and leave books to the site', () => {
+ const names = buildItemNames(
+ { 'INK_SACK:3': {}, STICK: {}, ENCHANTMENT_LUCK_1: {}, MYSTERY: {} },
+ { STICK: { name: 'Stick' } },
+ { 'INK_SACK-3': { displayname: 'ยงfCocoa Beans' }, STICK: { displayname: 'Wrong' } },
+ );
+ assert.deepEqual(names, { 'INK_SACK:3': 'Cocoa Beans', STICK: 'Stick' });
+});
+
+test('uniqueFusions treats A+B and B+A as one recipe', () => {
+ assert.equal(uniqueFusions({ recipes: { O: { 2: [['A', 'B'], ['B', 'A'], ['A', 'C']] } } }).size, 2);
+});
+
+test('valid data has no errors', () => {
+ assert.deepEqual(findErrors(validData()), []);
+});
+
+test('broken data is rejected', () => {
+ const broken = mutate => {
+ const data = validData();
+ mutate(data);
+ return findErrors(data).length > 0;
+ };
+ assert.ok(broken(d => { d.fusion.shards = {}; }));
+ assert.ok(broken(d => { d.fusion.shards.C5.fuse_amount = 0; }));
+ assert.ok(broken(d => { d.fusion.shards.C5.internal_id = 'NOT_A_SHARD'; }));
+ assert.ok(broken(d => { d.fusion.recipes.C0[2].push(['C1', 'NOPE']); }));
+ assert.ok(broken(d => { d.fusion.recipes.NOPE = { 2: [] }; }));
+ assert.ok(broken(d => { d.crafts = { ONLY: d.crafts.ITEM_0 }; }));
+ assert.ok(broken(d => { d.crafts.ITEM_0.materials = {}; }));
+ assert.ok(broken(d => { d.crafts.ITEM_0.materials.STICK = NaN; }));
+ assert.ok(broken(d => { d.essence = {}; }));
+ assert.ok(broken(d => { d.enchants = {}; }));
+ assert.ok(broken(d => { delete d.slayerXp.vampire; }));
+ assert.ok(broken(d => { d.itemNames = {}; }));
+});
+
+test('text that could inject HTML is rejected anywhere in the data', () => {
+ const rejected = mutate => {
+ const data = validData();
+ mutate(data);
+ return findErrors(data).some(e => e.includes('unsafe'));
+ };
+ assert.ok(rejected(d => { d.crafts.ITEM_0.name = '
'; }));
+ assert.ok(rejected(d => { d.crafts.ITEM_0.requirement = { type: 'other', name: '