feat: Have synced URLS with normal expressions

packages/core/src/utils/config.ts

@@ -446,7 +446,7 @@ export async function validateConfig(
     ...(config.preferredStreamExpressions?.map((e) => e.expression) ?? []),
     ...(config.includedStreamExpressions?.map((e) => e.expression) ?? []),
     ...(config.rankedStreamExpressions?.map((r) => r.expression) ?? []),
-  ];
+  ].filter((expr) => !(expr.startsWith('<SYNCED: ') && expr.endsWith('>')));
 
   for (const expression of expressionsToValidate) {
     try {
@@ -863,7 +863,7 @@ async function validateRegexes(config: UserData, skipErrors: boolean = false) {
     ...requiredRegexes,
     ...preferredRegexes.map((regex) => regex.pattern),
     ...rankedRegexes.map((regex) => regex.pattern),
-  ];
+  ].filter((pattern) => !(pattern.startsWith('<SYNCED: ') && pattern.endsWith('>')));
 
   if (!regexAllowed && regexes.length > 0) {
     const allowedPatterns = (await RegexAccess.allowedRegexPatterns()).patterns;
@@ -910,12 +910,28 @@ function validateSyncedRegexUrls(
 
   if (isUnrestricted) return;
 
+  const SYNCED_PREFIX = '<SYNCED: ';
+  const SYNCED_SUFFIX = '>';
+
+  // Extract URLs from <SYNCED: url> patterns in expression arrays
+  const extractSyncedUrls = (patterns: (string | { pattern?: string })[]): string[] =>
+    patterns
+      .map((p) => (typeof p === 'string' ? p : p.pattern || ''))
+      .filter((v) => v.startsWith(SYNCED_PREFIX) && v.endsWith(SYNCED_SUFFIX))
+      .map((v) => v.slice(SYNCED_PREFIX.length, -SYNCED_SUFFIX.length).trim());
+
   const allowedUrls = RegexAccess.getAllowedUrls();
   const urlsToCheck = [
     ...(config.syncedIncludedRegexUrls || []),
     ...(config.syncedExcludedRegexUrls || []),
     ...(config.syncedRequiredRegexUrls || []),
     ...(config.syncedPreferredRegexUrls || []),
+    ...(config.syncedRankedRegexUrls || []),
+    ...extractSyncedUrls(config.excludedRegexPatterns || []),
+    ...extractSyncedUrls(config.includedRegexPatterns || []),
+    ...extractSyncedUrls(config.requiredRegexPatterns || []),
+    ...extractSyncedUrls(config.preferredRegexPatterns || []),
+    ...extractSyncedUrls(config.rankedRegexPatterns || []),
   ];
 
   const invalidUrls = urlsToCheck.filter((url) => !allowedUrls.includes(url));
@@ -936,13 +952,30 @@ function validateSyncedSelUrls(config: UserData, skipErrors: boolean = false) {
 
   if (isUnrestricted) return;
 
+  const SYNCED_PREFIX = '<SYNCED: ';
+  const SYNCED_SUFFIX = '>';
+
+  // Extract URLs from <SYNCED: url> patterns in expression arrays
+  const extractSyncedUrls = (
+    expressions: (string | { expression?: string })[]
+  ): string[] =>
+    expressions
+      .map((e) => (typeof e === 'string' ? e : e.expression || ''))
+      .filter((v) => v.startsWith(SYNCED_PREFIX) && v.endsWith(SYNCED_SUFFIX))
+      .map((v) => v.slice(SYNCED_PREFIX.length, -SYNCED_SUFFIX.length).trim());
+
   const allowedUrls = SelAccess.getAllowedUrls();
   const urlsToCheck = [
     ...(config.syncedIncludedStreamExpressionUrls || []),
     ...(config.syncedExcludedStreamExpressionUrls || []),
     ...(config.syncedRequiredStreamExpressionUrls || []),
     ...(config.syncedPreferredStreamExpressionUrls || []),
     ...(config.syncedRankedStreamExpressionUrls || []),
+    ...extractSyncedUrls(config.includedStreamExpressions || []),
+    ...extractSyncedUrls(config.excludedStreamExpressions || []),
+    ...extractSyncedUrls(config.requiredStreamExpressions || []),
+    ...extractSyncedUrls(config.preferredStreamExpressions || []),
+    ...extractSyncedUrls(config.rankedStreamExpressions || []),
   ];
 
   const invalidUrls = urlsToCheck.filter((url) => !allowedUrls.includes(url));

packages/core/src/utils/regex-access.ts

@@ -264,37 +264,68 @@ export class RegexAccess {
     transform: (item: RegexPatternItem) => U,
     uniqueKey: (item: U) => string
   ): Promise<U[]> {
-    const patterns = await this.resolvePatterns(urls, userData);
-    if (patterns.length === 0) return existing;
 
-    const result = [...existing];
-    const existingSet = new Set(existing.map(uniqueKey));
     const overrides: SyncOverride[] = userData.regexOverrides || [];
 
-    for (const regex of patterns) {
-      const override = overrides.find(
-        (o) =>
-          o.pattern === regex.pattern ||
-          (regex.name && o.originalName === regex.name)
-      );
+    // Helper to process and transform patterns from a URL
+    const processPatterns = async (url: string): Promise<U[]> => {
+      const patterns = await this.resolvePatterns([url], userData);
+      const syncedItems: U[] = [];
+      for (const regex of patterns) {
+        const override = overrides.find(
+          (o) =>
+            o.pattern === regex.pattern ||
+            (regex.name && o.originalName === regex.name)
+        );
 
-      if (override?.disabled) continue;
-
-      const item = transform(
-        override
-          ? {
-              ...regex,
-              name: override.name ?? regex.name,
-              score:
-                override.score !== undefined ? override.score : regex.score,
-            }
-          : regex
-      );
+        if (override?.disabled) continue;
+
+        const item = transform(
+          override
+            ? {
+                ...regex,
+                name: override.name ?? regex.name,
+                score:
+                  override.score !== undefined ? override.score : regex.score,
+              }
+            : regex
+        );
+
+        syncedItems.push(item);
+      }
+      return syncedItems;
+    };
+
+    const SYNCED_PREFIX = '<SYNCED: ';
+    const SYNCED_SUFFIX = '>';
+    const result: U[] = [];
+    const usedUrls = new Set<string>();
 
+    for (const item of existing) {
       const key = uniqueKey(item);
-      if (!existingSet.has(key)) {
+
+      if (key.startsWith(SYNCED_PREFIX) && key.endsWith(SYNCED_SUFFIX)) {
+        const url = key.slice(SYNCED_PREFIX.length, -SYNCED_SUFFIX.length).trim();
+
+        if (url) {
+          usedUrls.add(url);
+          const syncedItems = await processPatterns(url);
+          result.push(...syncedItems);
+        } else {
+           result.push(item);
+        }
+      } else {
         result.push(item);
-        existingSet.add(key);
+      }
+    }
+
+    // Legacy fallback: append URLs from the `urls` array not already handled above
+    if (urls && urls.length > 0) {
+      for (const url of urls) {
+        if (!usedUrls.has(url)) {
+          const syncedItems = await processPatterns(url);
+          result.push(...syncedItems);
+        }
       }
     }
 

packages/core/src/utils/sel-access.ts

@@ -167,27 +167,63 @@ export class SelAccess {
     transform: (item: StreamExpressionItem) => U,
     uniqueKey: (item: U) => string
   ): Promise<U[]> {
-    const expressions = await this.resolveExpressions(urls, userData);
-    if (expressions.length === 0) return existing;
 
-    const result = [...existing];
-    const existingSet = new Set(existing.map(uniqueKey));
     const overrides: SyncOverride[] = userData.selOverrides || [];
 
-    for (const expr of expressions) {
-      const override = this._findSelOverride(expr, overrides);
+    // Helper to process and transform expressions from a URL
+    const processExpressions = async (url: string): Promise<U[]> => {
+      const expressions = await this.resolveExpressions([url], userData);
+      const syncedItems: U[] = [];
+      for (const expr of expressions) {
+        const override = this._findSelOverride(expr, overrides);
+        if (override?.disabled) continue;
 
-      if (override?.disabled) continue;
+        const overriddenExpr = override
+          ? this._applySelOverride(expr, override)
+          : expr;
 
-      const overriddenExpr = override
-        ? this._applySelOverride(expr, override)
-        : expr;
+        const item = transform(overriddenExpr);
+        syncedItems.push(item);
+      }
+      return syncedItems;
+    };
+
+    const SYNCED_PREFIX = '<SYNCED: ';
+    const SYNCED_SUFFIX = '>';
+    const result: U[] = [];
+    const usedUrls = new Set<string>();
 
-      const item = transform(overriddenExpr);
+    for (const item of existing) {
       const key = uniqueKey(item);
-      if (!existingSet.has(key)) {
+
+      if (key.startsWith(SYNCED_PREFIX) && key.endsWith(SYNCED_SUFFIX)) {
+        const isPlaceholderDisabled =
+          typeof item === 'object' && item !== null && (item as any).enabled === false;
+
+        const url = key.slice(SYNCED_PREFIX.length, -SYNCED_SUFFIX.length).trim();
+
+        if (isPlaceholderDisabled) {
+          usedUrls.add(url);
+          result.push(item); // Keep disabled placeholder without fetching
+        } else if (url) {
+          usedUrls.add(url);
+          const syncedItems = await processExpressions(url);
+          result.push(...syncedItems);
+        } else {
+           result.push(item);
+        }
+      } else {
         result.push(item);
-        existingSet.add(key);
+      }
+    }
+
+    // Legacy fallback: append URLs from the `urls` array not already handled above
+    if (urls && urls.length > 0) {
+      for (const url of urls) {
+        if (!usedUrls.has(url)) {
+          const syncedItems = await processExpressions(url);
+          result.push(...syncedItems);
+        }
       }
     }
 

packages/frontend/src/components/config-modal.tsx

@@ -3,7 +3,7 @@ import { Modal } from '@/components/ui/modal';
 import { TextInput } from '@/components/ui/text-input';
 import { Button } from '@/components/ui/button';
 import { loadUserConfig, APIError } from '@/lib/api';
-import { useUserData } from '@/context/userData';
+import { useUserData, applyMigrations } from '@/context/userData';
 import { toast } from 'sonner';
 import { PasswordInput } from './ui/password-input';
 
@@ -34,7 +34,7 @@ export function ConfigModal({
 
     try {
       const result = await loadUserConfig(uuid, password);
-      setUserData(() => result.userData);
+      setUserData(() => applyMigrations(result.userData));
       setUuid(uuid);
       setPassword(password);
       setEncryptedPassword(result.encryptedPassword);