docs: Enhance configuration management in README and update config handling in code to support saved configurations with higher priority

experdot · experdot · commit f6be7ec5c4d5 · 2025-08-18T18:45:16.000+08:00
diff --git a/README.md b/README.md
@@ -165,7 +165,14 @@ docker compose up -d --build
 
 ### 📋 配置说明
 
-使用官方 Docker 镜像时，需要在 docker-compose.dockerhub.yml 文件中配置环境变量传递给容器。系统支持优先级为：**Docker 环境变量 > .env 文件 > JSON 配置文件**
+使用官方 Docker 镜像时，需要在 docker-compose.dockerhub.yml 文件中配置环境变量传递给容器。系统支持灵活的配置优先级：
+
+**配置优先级**：`.saved.json 文件` > `Docker 环境变量` > `.env 文件` > `JSON 配置文件` > `默认值`
+
+-   **`.saved.json 文件`**：通过前端界面修改配置时动态生成，优先级最高，确保用户修改的配置能够生效
+-   **Docker 环境变量**：适合生产环境部署，通过 docker-compose 文件设置
+-   **`.env 文件`**：适合开发环境，存储敏感信息如 API 密钥
+-   **JSON 配置文件\*\***：基础配置模板，支持复杂的嵌套配置结构
 
 **完整的环境变量列表请查看：**
 
diff --git a/README_ENGLISH.md b/README_ENGLISH.md
@@ -137,6 +137,22 @@ If you want to understand the details of each module or perform secondary develo
 -   MySQL
 -   Qdrant
 
+### 📋 Configuration Instructions
+
+When using official Docker images, you need to configure environment variables in the docker-compose.dockerhub.yml file to pass them to containers. The system supports flexible configuration priority:
+
+**Configuration Priority**: `.saved.json files` > `Docker environment variables` > `.env files` > `JSON config files` > `default values`
+
+-   **`.saved.json files`**: Dynamically generated when modifying configuration through the frontend interface, highest priority, ensuring user-modified configurations take effect
+-   **Docker environment variables**: Suitable for production deployment, set through docker-compose files
+-   **`.env files`**: Suitable for development environments, storing sensitive information like API keys
+-   **JSON config files**: Base configuration templates, supporting complex nested configuration structures
+
+**For complete environment variable lists, see:**
+
+-   RAG Service: [`sources/gc-qa-rag-server/env.example`](./sources/gc-qa-rag-server/env.example)
+-   ETL Service: [`sources/gc-qa-rag-etl/env.example`](./sources/gc-qa-rag-etl/env.example)
+
 **Important**: Regardless of which deployment method you choose, you need to configure API keys first!
 
 For detailed steps, please refer to our [Quick Start Guide](./quickstart_ENGLISH.md).
diff --git a/sources/gc-qa-rag-etl/etlapp/common/config.py b/sources/gc-qa-rag-etl/etlapp/common/config.py
@@ -32,19 +32,29 @@ class VectorDbConfig:
     host: str
 
 
-def _get_config_value(key: str, config_raw: dict, default: Optional[str] = None) -> str:
-    """Get configuration value with priority: ENV > .env > JSON."""
-    # First check environment variables
-    env_value = os.getenv(key)
-    if env_value is not None:
-        return env_value
-    
-    # Then check nested JSON structure
+def _get_config_value(key: str, config_raw: dict, saved_config_raw: dict, default: Optional[str] = None) -> str:
+    """Get configuration value with priority: saved.json > ENV > .env > JSON."""
+    # First check saved configuration (highest priority)
     keys = key.lower().split('_')
     # Skip the 'gc_qa_rag' prefix for JSON lookup
     if len(keys) >= 4 and keys[0] == 'gc' and keys[1] == 'qa' and keys[2] == 'rag':
         keys = keys[3:]
     
+    # Check saved config first
+    current = saved_config_raw
+    try:
+        for k in keys:
+            current = current[k]
+        return str(current)
+    except (KeyError, TypeError):
+        pass
+    
+    # Then check environment variables
+    env_value = os.getenv(key)
+    if env_value is not None:
+        return env_value
+    
+    # Then check nested JSON structure
     current = config_raw
     try:
         for k in keys:
@@ -60,9 +70,9 @@ def _get_config_value(key: str, config_raw: dict, default: Optional[str] = None)
     raise ValueError(f"Configuration value not found for key: {key}")
 
 
-def _get_config_int(key: str, config_raw: dict, default: Optional[int] = None) -> int:
-    """Get integer configuration value with priority: ENV > .env > JSON."""
-    value = _get_config_value(key, config_raw, str(default) if default is not None else None)
+def _get_config_int(key: str, config_raw: dict, saved_config_raw: dict, default: Optional[int] = None) -> int:
+    """Get integer configuration value with priority: saved.json > ENV > .env > JSON."""
+    value = _get_config_value(key, config_raw, saved_config_raw, str(default) if default is not None else None)
     try:
         return int(value)
     except (ValueError, TypeError):
@@ -97,27 +107,38 @@ def from_environment(cls, environment: str) -> "Config":
             except json.JSONDecodeError as e:
                 print(f"Warning: Invalid JSON in configuration file: {e}")
 
+        # Try to load saved config (highest priority)
+        saved_config_raw = {}
+        saved_config_path = Path(f".config.{environment}.saved.json")
+        if saved_config_path.exists():
+            try:
+                with open(saved_config_path) as f:
+                    saved_config_raw = json.load(f)
+                print(f"Loaded saved configuration from: {saved_config_path}")
+            except json.JSONDecodeError as e:
+                print(f"Warning: Invalid JSON in saved configuration file: {e}")
+
         return cls(
             environment=environment,
             das=DasConfig(
-                base_url_page=_get_config_value("GC_QA_RAG_DAS_BASE_URL_PAGE", config_raw, ""),
-                base_url_thread=_get_config_value("GC_QA_RAG_DAS_BASE_URL_THREAD", config_raw, ""),
-                token=_get_config_value("GC_QA_RAG_DAS_TOKEN", config_raw, ""),
+                base_url_page=_get_config_value("GC_QA_RAG_DAS_BASE_URL_PAGE", config_raw, saved_config_raw, ""),
+                base_url_thread=_get_config_value("GC_QA_RAG_DAS_BASE_URL_THREAD", config_raw, saved_config_raw, ""),
+                token=_get_config_value("GC_QA_RAG_DAS_TOKEN", config_raw, saved_config_raw, ""),
             ),
             llm=LlmConfig(
-                api_key=_get_config_value("GC_QA_RAG_LLM_API_KEY", config_raw),
-                api_base=_get_config_value("GC_QA_RAG_LLM_API_BASE", config_raw, "https://dashscope.aliyuncs.com/compatible-mode/v1"),
-                model_name=_get_config_value("GC_QA_RAG_LLM_MODEL_NAME", config_raw, "qwen-plus"),
-                max_rpm=_get_config_int("GC_QA_RAG_LLM_MAX_RPM", config_raw, 100),
+                api_key=_get_config_value("GC_QA_RAG_LLM_API_KEY", config_raw, saved_config_raw),
+                api_base=_get_config_value("GC_QA_RAG_LLM_API_BASE", config_raw, saved_config_raw, "https://dashscope.aliyuncs.com/compatible-mode/v1"),
+                model_name=_get_config_value("GC_QA_RAG_LLM_MODEL_NAME", config_raw, saved_config_raw, "qwen-plus"),
+                max_rpm=_get_config_int("GC_QA_RAG_LLM_MAX_RPM", config_raw, saved_config_raw, 100),
             ),
             embedding=EmbeddingConfig(
-                api_key=_get_config_value("GC_QA_RAG_EMBEDDING_API_KEY", config_raw)
+                api_key=_get_config_value("GC_QA_RAG_EMBEDDING_API_KEY", config_raw, saved_config_raw)
             ),
             vector_db=VectorDbConfig(
-                host=_get_config_value("GC_QA_RAG_VECTOR_DB_HOST", config_raw, "http://host.docker.internal:6333")
+                host=_get_config_value("GC_QA_RAG_VECTOR_DB_HOST", config_raw, saved_config_raw, "http://host.docker.internal:6333")
             ),
-            root_path=_get_config_value("GC_QA_RAG_ROOT_PATH", config_raw, user_cache_dir("gc-qa-rag", ensure_exists=True)),
-            log_path=_get_config_value("GC_QA_RAG_LOG_PATH", config_raw, user_log_dir("gc-qa-rag", ensure_exists=True)),
+            root_path=_get_config_value("GC_QA_RAG_ROOT_PATH", config_raw, saved_config_raw, user_cache_dir("gc-qa-rag", ensure_exists=True)),
+            log_path=_get_config_value("GC_QA_RAG_LOG_PATH", config_raw, saved_config_raw, user_log_dir("gc-qa-rag", ensure_exists=True)),
         )
 
 
diff --git a/sources/gc-qa-rag-etl/etlapp_api/routers/config.py b/sources/gc-qa-rag-etl/etlapp_api/routers/config.py
@@ -23,21 +23,77 @@ def dataclass_to_dict(obj):
 
 @config_router.post("/update_config")
 async def update_config_api(request: Request):
-    data = await request.json()
-    from etlapp.common.config import app_config
+    try:
+        data = await request.json()
+        from etlapp.common.config import app_config
+
+        # 创建保存的配置文件路径
+        saved_config_path = Path(f".config.{app_config.environment}.saved.json")
+        
+        # 如果保存的配置文件已存在，先加载现有内容
+        saved_config_raw = {}
+        if saved_config_path.exists():
+            try:
+                with open(saved_config_path, "r", encoding="utf-8") as f:
+                    saved_config_raw = json.load(f)
+            except json.JSONDecodeError:
+                # 如果文件损坏，从空配置开始
+                saved_config_raw = {}
+
+        # 更新配置数据
+        for key in ["llm", "embedding", "vector_db", "root_path", "log_path", "das"]:
+            if key in data:
+                saved_config_raw[key] = data[key]
+
+        # 写入保存的配置文件
+        with open(saved_config_path, "w", encoding="utf-8") as f:
+            json.dump(saved_config_raw, f, ensure_ascii=False, indent=4)
+
+        # 重新加载配置
+        from etlapp.common.config import reload_config
+        reload_config()
 
-    config_path = Path(f".config.{app_config.environment}.json")
-    if not config_path.exists():
-        return JSONResponse(status_code=404, content={"error": "Config file not found"})
-    with open(config_path, "r", encoding="utf-8") as f:
-        config_raw = json.load(f)
-    for key in ["llm", "embedding", "vector_db", "root_path", "log_path"]:
-        if key in data:
-            config_raw[key] = data[key]
-    with open(config_path, "w", encoding="utf-8") as f:
-        json.dump(config_raw, f, ensure_ascii=False, indent=4)
-    # reload configuration
-    from etlapp.common.config import reload_config
-
-    reload_config()
-    return {"msg": "Config updated"}
+        return {
+            "msg": "Config updated successfully",
+            "saved_to": str(saved_config_path),
+            "note": "Configuration has been saved to .saved.json file with highest priority"
+        }
+
+    except Exception as e:
+        return JSONResponse(
+            status_code=500, 
+            content={"error": f"Failed to update configuration: {str(e)}"}
+        )
+
+
+@config_router.delete("/reset_config")
+async def reset_config_api():
+    """删除保存的配置文件，恢复到默认配置优先级"""
+    try:
+        from etlapp.common.config import app_config
+        
+        saved_config_path = Path(f".config.{app_config.environment}.saved.json")
+        
+        if saved_config_path.exists():
+            saved_config_path.unlink()  # 删除文件
+            
+            # 重新加载配置
+            from etlapp.common.config import reload_config
+            reload_config()
+            
+            return {
+                "msg": "Saved configuration reset successfully",
+                "deleted_file": str(saved_config_path),
+                "note": "Configuration priority has been restored to: ENV > .env > JSON"
+            }
+        else:
+            return {
+                "msg": "No saved configuration file found",
+                "note": "Already using default configuration priority"
+            }
+            
+    except Exception as e:
+        return JSONResponse(
+            status_code=500,
+            content={"error": f"Failed to reset configuration: {str(e)}"}
+        )
diff --git a/sources/gc-qa-rag-server/ragapp/common/config.py b/sources/gc-qa-rag-server/ragapp/common/config.py
@@ -29,19 +29,29 @@ class DbConfig:
     connection_string: str
 
 
-def _get_config_value(key: str, config_raw: dict, default: Optional[str] = None) -> str:
-    """Get configuration value with priority: ENV > .env > JSON."""
-    # First check environment variables
-    env_value = os.getenv(key)
-    if env_value is not None:
-        return env_value
-    
-    # Then check nested JSON structure
+def _get_config_value(key: str, config_raw: dict, saved_config_raw: dict, default: Optional[str] = None) -> str:
+    """Get configuration value with priority: saved.json > ENV > .env > JSON."""
+    # First check saved configuration (highest priority)
     keys = key.lower().split('_')
     # Skip the 'gc_qa_rag' prefix for JSON lookup
     if keys[0] == 'gc' and keys[1] == 'qa' and keys[2] == 'rag':
         keys = keys[3:]
     
+    # Check saved config first
+    current = saved_config_raw
+    try:
+        for k in keys:
+            current = current[k]
+        return str(current)
+    except (KeyError, TypeError):
+        pass
+    
+    # Then check environment variables
+    env_value = os.getenv(key)
+    if env_value is not None:
+        return env_value
+    
+    # Then check nested JSON structure
     current = config_raw
     try:
         for k in keys:
@@ -58,10 +68,25 @@ def _get_config_value(key: str, config_raw: dict, default: Optional[str] = None)
 
 
 def _get_llm_config(
-    config_raw: dict, config_type: str, default_config: Optional[LlmConfig] = None
+    config_raw: dict, saved_config_raw: dict, config_type: str, default_config: Optional[LlmConfig] = None
 ) -> LlmConfig:
     """Get LLM configuration with fallback to default config if specified config doesn't exist."""
-    # Try to get from environment variables first
+    # Try to get from saved config first (highest priority)
+    if config_type in saved_config_raw:
+        saved_config = saved_config_raw[config_type]
+        api_key = saved_config.get("api_key")
+        api_base = saved_config.get("api_base")
+        model_name = saved_config.get("model_name")
+        
+        # If all saved config values are set, use them
+        if api_key and api_base and model_name:
+            return LlmConfig(
+                api_key=api_key,
+                api_base=api_base,
+                model_name=model_name,
+            )
+    
+    # Try to get from environment variables
     env_prefix = f"GC_QA_RAG_{config_type.upper()}"
     api_key = os.getenv(f"{env_prefix}_API_KEY")
     api_base = os.getenv(f"{env_prefix}_API_BASE")
@@ -94,9 +119,9 @@ def _get_llm_config(
     
     # Last resort: use default LLM config from env/JSON
     return LlmConfig(
-        api_key=api_key or _get_config_value("GC_QA_RAG_LLM_DEFAULT_API_KEY", config_raw),
-        api_base=api_base or _get_config_value("GC_QA_RAG_LLM_DEFAULT_API_BASE", config_raw, "https://dashscope.aliyuncs.com/compatible-mode/v1"),
-        model_name=model_name or _get_config_value("GC_QA_RAG_LLM_DEFAULT_MODEL_NAME", config_raw, "qwen-plus"),
+        api_key=api_key or _get_config_value("GC_QA_RAG_LLM_DEFAULT_API_KEY", config_raw, saved_config_raw),
+        api_base=api_base or _get_config_value("GC_QA_RAG_LLM_DEFAULT_API_BASE", config_raw, saved_config_raw, "https://dashscope.aliyuncs.com/compatible-mode/v1"),
+        model_name=model_name or _get_config_value("GC_QA_RAG_LLM_DEFAULT_MODEL_NAME", config_raw, saved_config_raw, "qwen-plus"),
     )
 
 
@@ -130,27 +155,38 @@ def from_environment(cls, environment: str) -> "Config":
             except json.JSONDecodeError as e:
                 print(f"Warning: Invalid JSON in configuration file: {e}")
 
+        # Try to load saved config (highest priority)
+        saved_config_raw = {}
+        saved_config_path = Path(f".config.{environment}.saved.json")
+        if saved_config_path.exists():
+            try:
+                with open(saved_config_path) as f:
+                    saved_config_raw = json.load(f)
+                print(f"Loaded saved configuration from: {saved_config_path}")
+            except json.JSONDecodeError as e:
+                print(f"Warning: Invalid JSON in saved configuration file: {e}")
+
         # Initialize default config first
-        llm_default = _get_llm_config(config_raw, "llm_default")
+        llm_default = _get_llm_config(config_raw, saved_config_raw, "llm_default")
 
         return cls(
             environment=environment,
             llm_default=llm_default,
-            llm_summary=_get_llm_config(config_raw, "llm_summary", llm_default),
-            llm_think=_get_llm_config(config_raw, "llm_think", llm_default),
-            llm_query=_get_llm_config(config_raw, "llm_query", llm_default),
-            llm_research=_get_llm_config(config_raw, "llm_research", llm_default),
+            llm_summary=_get_llm_config(config_raw, saved_config_raw, "llm_summary", llm_default),
+            llm_think=_get_llm_config(config_raw, saved_config_raw, "llm_think", llm_default),
+            llm_query=_get_llm_config(config_raw, saved_config_raw, "llm_query", llm_default),
+            llm_research=_get_llm_config(config_raw, saved_config_raw, "llm_research", llm_default),
             embedding=EmbeddingConfig(
-                api_key=_get_config_value("GC_QA_RAG_EMBEDDING_API_KEY", config_raw)
+                api_key=_get_config_value("GC_QA_RAG_EMBEDDING_API_KEY", config_raw, saved_config_raw)
             ),
             vector_db=VectorDbConfig(
-                host=_get_config_value("GC_QA_RAG_VECTOR_DB_HOST", config_raw, "http://rag_qdrant_container:6333")
+                host=_get_config_value("GC_QA_RAG_VECTOR_DB_HOST", config_raw, saved_config_raw, "http://rag_qdrant_container:6333")
             ),
             db=DbConfig(
-                connection_string=_get_config_value("GC_QA_RAG_DB_CONNECTION_STRING", config_raw, "mysql+mysqlconnector://root:12345678@rag_mysql_container:3306/search_db")
+                connection_string=_get_config_value("GC_QA_RAG_DB_CONNECTION_STRING", config_raw, saved_config_raw, "mysql+mysqlconnector://root:12345678@rag_mysql_container:3306/search_db")
             ),
-            log_path=_get_config_value("GC_QA_RAG_LOG_PATH", config_raw, user_log_dir("gc-qa-rag-server", ensure_exists=True)),
-            etl_base_url=_get_config_value("GC_QA_RAG_ETL_BASE_URL", config_raw, "http://host.docker.internal:8001"),
+            log_path=_get_config_value("GC_QA_RAG_LOG_PATH", config_raw, saved_config_raw, user_log_dir("gc-qa-rag-server", ensure_exists=True)),
+            etl_base_url=_get_config_value("GC_QA_RAG_ETL_BASE_URL", config_raw, saved_config_raw, "http://host.docker.internal:8001"),
         )
 
 
