Merge pull request #983 from WatWowMap/develop

Sync Dev to Main

.eslintignore

@@ -1,4 +1,5 @@
 dist
+dist-*
 node_modules
 public/missing-locales
 public/images/custom

.eslintrc

@@ -1,10 +1,5 @@
 {
-  "extends": [
-    "airbnb",
-    "airbnb/rules/react",
-    "eslint:recommended",
-    "prettier"
-  ],
+  "extends": ["airbnb", "airbnb/rules/react", "eslint:recommended", "prettier"],
   "parserOptions": {
     "ecmaVersion": "latest"
   },
@@ -74,42 +69,21 @@
   },
   "settings": {
     "node": {
-      "extensions": [
-        ".mjs",
-        ".js",
-        ".jsx",
-        ".ts",
-        ".tsx"
-      ]
+      "extensions": [".mjs", ".js", ".jsx", ".ts", ".tsx"]
     },
     "import/resolver": {
       "alias": {
         "map": [
-          [
-            "@components",
-            "./src/components/"
-          ],
-          [
-            "@services",
-            "./src/services/"
-          ],
-          [
-            "@hooks",
-            "./src/hooks/"
-          ],
-          [
-            "@assets",
-            "./src/assets/"
-          ]
+          ["@components", "./src/components/"],
+          ["@features", "./src/features/"],
+          ["@services", "./src/services/"],
+          ["@hooks", "./src/hooks/"],
+          ["@assets", "./src/assets/"],
+          ["@utils", "./src/utils/"],
+          ["@store", "./src/store/"]
         ],
-        "extensions": [
-          ".mjs",
-          ".js",
-          ".jsx",
-          ".ts",
-          ".tsx"
-        ]
+        "extensions": [".mjs", ".js", ".jsx", ".ts", ".tsx"]
       }
     }
   }
-}
+}

.github/workflows/locales.yml

@@ -1,5 +1,10 @@
 name: Locales
-on: workflow_call
+on:
+  workflow_call:
+    secrets:
+      OPENAI_API_KEY:
+        required: true
+        description: The API key for OpenAI
 
 permissions: write-all
 

.github/workflows/main.yml

@@ -12,6 +12,8 @@ jobs:
     uses: ./.github/workflows/config.yml
   locales:
     uses: ./.github/workflows/locales.yml
+    secrets:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   release:
     if: always()
     uses: ./.github/workflows/release.yml

.gitignore

@@ -1,5 +1,6 @@
 node_modules
 dist
+dist-*
 
 # Config files
 server/src/configs/*
@@ -10,6 +11,8 @@ server/src/configs/koji_backups/*
 !server/src/configs/areas.example.json
 !server/src/configs/local.example.json
 !server/src/configs/custom-environment-variables.json
+!server/src/configs/multi-domain-example/local.json
+!server/src/configs/multi-domain-example
 .env
 
 # Masterfile

.prettierignore

@@ -1,4 +1,5 @@
 dist
+dist-*
 node_modules
 public/missing-locales
 public/images/custom

docker-compose.example.yml

@@ -41,6 +41,7 @@ services:
       # ARRAY_VALUE_EXAMPLE: "[3, 4, 5]"
 
     volumes:
+      - ./server/.cache:/home/node/server/.cache
       # All of these are optional - comment out whichever ones you aren't using
       - ./server/src/configs/areas.json:/home/node/server/src/configs/areas.json
       - ./server/src/configs/local.json:/home/node/server/src/configs/local.json

jsconfig.json

@@ -12,9 +12,12 @@
     "paths": {
       "@assets/*": ["./src/assets/*"],
       "@components/*": ["./src/components/*"],
+      "@features/*": ["./src/features/*"],
       "@services/*": ["./src/services/*"],
-      "@hooks/*": ["./src/hooks/*"]
+      "@hooks/*": ["./src/hooks/*"],
+      "@utils/*": ["./src/utils/*"],
+      "@store/*": ["./src/store/*"]
     }
   },
-  "exclude": ["node_modules", "**/node_modules/*", "dist"]
+  "exclude": ["node_modules", "**/node_modules/*", "dist", "dist-*"]
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "reactmap",
-  "version": "1.30.0",
+  "version": "1.31.0-develop.7",
   "private": true,
   "description": "React based frontend map.",
   "license": "MIT",
@@ -175,6 +175,7 @@
     "source-map": "^0.7.4",
     "suncalc": "^1.9.0",
     "supercluster": "^8.0.1",
+    "uicons.js": "^1.1.1",
     "zustand": "4.4.6"
   },
   "devDependencies": {

packages/config/.configref

@@ -1 +1 @@
-24124
+24070

packages/config/lib/index.js

@@ -12,6 +12,15 @@ if (!process.env.NODE_CONFIG_DIR) {
   process.env.ALLOW_CONFIG_MUTATIONS = 'true'
 }
 
+if (process.env.NODE_CONFIG_ENV) {
+  if (
+    process.env.NODE_CONFIG_ENV.includes('.') ||
+    process.env.NODE_CONFIG_ENV.includes('/')
+  ) {
+    throw new Error('Invalid NODE_CONFIG_ENV, must not contain "." or "/"')
+  }
+}
+
 const config = require('config')
 
 config.getSafe = config.get

packages/locales/lib/generate.js

@@ -1,9 +1,11 @@
+/* eslint-disable prefer-template */
 // @ts-check
 /* eslint-disable no-restricted-syntax */
 /* eslint-disable guard-for-in */
 
 require('dotenv').config()
 const { OpenAI } = require('openai')
+const { encode } = require('gpt-tokenizer')
 
 const { log, HELPERS } = require('@rm/logger')
 
@@ -15,56 +17,49 @@ const openAI = process.env.OPENAI_API_KEY
     })
   : null
 
+const TOKEN_LIMIT = 1024
+
 /**
  * @typedef {Record<string, string>} I18nObject
  * @typedef {I18nObject | string} Node
  */
 
 /**
- * Recursively estimates the token size of a {@link Node}
- * @param {Node} content
- * @returns {number}
- */
-function estimateTokenCount(content) {
-  if (typeof content === 'string') {
-    return content.split(/[\s.'_A-Z0-9]/).length * 2
-  }
-  if (typeof content === 'object') {
-    let count = 0
-    for (const key in content) {
-      count += estimateTokenCount(content[key])
-      count += key.split(/[_A-Z0-9]/).length * 2
-    }
-    return count
-  }
-  return 1
-}
-
-/**
- * Splits the json into 2048 token chunks
+ * Splits the json into token chunks
  * @param {I18nObject} json
  * @returns {I18nObject[]}
  */
 function splitJson(json) {
   /** @type {I18nObject[]} */
   const chunks = []
   /** @type {I18nObject} */
-  let pool = {}
-  let poolSize = 0
-  for (const key in json) {
-    const nodeSize = estimateTokenCount(json[key]) + estimateTokenCount(key)
-    if (nodeSize + poolSize < 2048) {
-      poolSize += nodeSize
-      pool[key] = json[key]
-    } else {
-      chunks.push(pool)
-      pool = { [key]: json[key] }
-      poolSize = nodeSize
+  let currentChunk = {}
+  let currentTokenCount = 2
+
+  for (const [key, value] of Object.entries(json)) {
+    const string = `  "${key}": ${
+      typeof value === 'string'
+        ? `"${value}"`
+        : typeof value === 'number'
+        ? value
+        : `${value}`
+    },\n`
+    const newLineCount = (string.match(/\n/g) || []).length - 1
+    const tokenCount = encode(string).length
+    const totalTokenCount = tokenCount + newLineCount
+
+    if (currentTokenCount + totalTokenCount >= TOKEN_LIMIT) {
+      chunks.push(currentChunk)
+      currentChunk = {}
+      currentTokenCount = 2
     }
+    currentChunk[key] = value
+    currentTokenCount =
+      newLineCount > 0
+        ? encode(JSON.stringify(currentChunk, null, 2)).length
+        : currentTokenCount + totalTokenCount
   }
-  if (Object.keys(pool).length > 0) {
-    chunks.push(pool)
-  }
+  if (Object.keys(currentChunk).length > 0) chunks.push(currentChunk)
   return chunks
 }
 
@@ -96,18 +91,18 @@ function matchJSON(str) {
 }
 
 /**
- * Sends the result to OpenAI gpt-3.5-turbo model
+ * Sends the result to OpenAI gpt-4-turbo model
  * @param {string} locale
  * @param {Node} missingKeys
  * @returns
  */
 async function sendToGPT(locale, missingKeys) {
   return openAI.chat.completions.create({
-    model: 'gpt-3.5-turbo',
+    model: 'gpt-4-turbo-preview',
     messages: [
       {
         role: 'system',
-        content: `Translate an i18n locale json content to ${locale}. It's a key-value structure, don't translate the key. Consider the context of all of the values together to make better translation. All translations should be related to a Pokemon GO context.`,
+        content: `Translate an i18n English locale json content to ${locale}. It's a key-value structure, don't translate the key. Consider the context of all of the values together to make better translation. All translations should be related to a Pokemon GO context. Ensure that all key value pairs are matched correctly.`,
       },
       {
         role: 'user',
@@ -141,7 +136,10 @@ async function generate() {
         /** @type {I18nObject} */
         const missingKeys = Object.fromEntries(
           Object.entries(englishRef).filter(
-            ([key]) => !(key in merged) && !key.startsWith('locale_selection_'),
+            ([key]) =>
+              !(key in merged) &&
+              !key.startsWith('locale_selection_') &&
+              typeof englishRef[key] !== 'number',
           ),
         )
 
@@ -167,13 +165,15 @@ async function generate() {
               } catch (e) {
                 log.error(e, '\nUnable to parse returned translations\n', {
                   locale,
+                  reason: raw.choices[0].finish_reason,
                   content,
                   clean,
                 })
                 return {}
               }
             }),
           )
+
           return [locale, result.reduce((acc, x) => ({ ...acc, ...x }), merged)]
         } catch (error) {
           log.error(HELPERS.locales, error)
@@ -191,6 +191,7 @@ async function generate() {
 module.exports.generate = generate
 
 if (require.main === module) {
+  if (!process.env.OPENAI_API_KEY) throw new Error('OpenAI API key is missing')
   generate()
     .then((locales) => writeAll(locales, false, __dirname, './generated'))
     .then(() =>