Feature Architecture & CI Improvements (#4)

* chore: 升级 tj-actions/changed-files 到 v46.0.1 * Add feature-based architecture for version-specific config Introduces a feature registry and auto-discovery system under modules/features/ to manage version-specific configuration logic. Adds base classes and decorators for feature registration, semantic version comparison, and applies features in global, infrastructure, plugins, and services modules. Implements features for trigger worker, plugin metrics, and external Prometheus (3.7.0+). Updates i18n translations and documentation to reflect the new architecture and configuration options. Includes tests for feature registration, version parsing, and applicability. * Add Docker registry secret generator and secret utils Added generate-image-repo-secret.sh for interactively creating Kubernetes imagePullSecret for private Docker registries. Updated .gitignore to better exclude sensitive files. Added utils/secrets.py for secret key generation. Updated README files with instructions for using the new secret generator script. * ci: add unit tests and module validation - Add test job to validate all Python module imports - Add feature tests, services tests, and S3 config tests - Add shell script validation with shellcheck - Update actions to v4/v5 versions - Add CI integration test plan documentation * ci: skip interactive tests in CI environment test_services.py and test_s3_config.py require interactive input, skip them until --ci mode is implemented * chore: add PR template * feat: add --ci non-interactive mode for CI testing - Add set_ci_mode() and is_ci_mode() to config.py - Add --ci flag to generate-values-prd.py - Update utils/prompts.py to return defaults in CI mode - Update i18n/language.py to use English in CI mode - Update utils/downloader.py to use latest version in CI mode - Update version_manager.py to avoid variable shadowing - Enable CI mode in test files for non-interactive testing
2026-07-01 20:14:02 -04:00 · 2026-02-04 14:19:36 +08:00
parent 7ba7a2e601
commit d06ab44e52
17 changed files with 410 additions and 27 deletions
@@ -0,0 +1,21 @@
+## Description
+
+<!-- Describe your changes in detail -->
+
+## Type of Change
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] Documentation update
+- [ ] CI/CD improvement
+
+## Checklist
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+
@@ -10,11 +10,11 @@ jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
        with:
-          python-version: '3.9'
+          python-version: '3.11'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
@@ -27,7 +27,7 @@ jobs:
  check-format:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: Check file endings
        run: |
          if [ "$(file generate-values-prd.py | grep -o 'CRLF')" = "CRLF" ]; then
@@ -35,3 +35,79 @@ jobs:
            exit 1
          fi

+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Validate module imports
+        run: |
+          echo "Validating Python module imports..."
+          python -c "
+          import sys
+          modules = [
+              'config',
+              'generator',
+              'version_manager',
+              'i18n',
+              'i18n.language',
+              'i18n.translations',
+              'utils',
+              'utils.colors',
+              'utils.downloader',
+              'utils.prompts',
+              'utils.secrets',
+              'modules',
+              'modules.global_config',
+              'modules.infrastructure',
+              'modules.mail',
+              'modules.networking',
+              'modules.plugins',
+              'modules.services',
+              'modules.features',
+              'modules.features.base',
+              'modules.features.external_prometheus',
+              'modules.features.plugin_metric',
+              'modules.features.trigger_worker',
+          ]
+
+          failed = []
+          for module in modules:
+              try:
+                  __import__(module)
+                  print(f'✓ {module}')
+              except Exception as e:
+                  print(f'✗ {module}: {e}')
+                  failed.append(module)
+
+          if failed:
+              print(f'\n❌ Failed to import {len(failed)} module(s)')
+              sys.exit(1)
+          else:
+              print(f'\n✅ All {len(modules)} modules imported successfully')
+          "
+      - name: Run feature tests
+        run: |
+          echo "Running feature tests..."
+          python test_features.py
+      # Note: test_services.py and test_s3_config.py require values.yaml
+      # They are not included in CI until integration tests are implemented
+      # See docs/CI-INTEGRATION-PLAN.md for the full plan
+
+  shell-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install shellcheck
+        run: sudo apt-get install -y shellcheck
+      - name: Run shellcheck
+        run: |
+          echo "Checking shell scripts..."
+          shellcheck generate-image-repo-secret.sh || echo "::warning::shellcheck found issues"
@@ -45,10 +45,16 @@ values-*.yaml

 # Sensitive files
 email-server.txt
-*password*
-*secret*
+*password*.txt
+*password*.yaml
+*.secret
+*.secret.*
+*-secret.yaml
+*-secret.json
 *.key
 *.pem
+.env.local
+.env.*.local

 # Temporary files
 *.tmp
@@ -112,8 +112,9 @@ The generated configuration file will be saved as `out/values-prd-{version}.yaml

 ```
 .
-├── generate-values-prd.py    # Main script file
-├── generator.py              # Core generator class
+├── generate-values-prd.py       # Main script file
+├── generate-image-repo-secret.sh # Docker registry secret generator
+├── generator.py                 # Core generator class
 ├── version_manager.py        # Version management
 ├── config.py                 # Configuration constants
 ├── pyproject.toml           # Python project configuration
@@ -184,6 +185,22 @@ The script automatically handles the following relationships:
 - **TLS Consistency**: TLS configuration automatically syncs with Ingress to avoid CORS issues
 - **Infrastructure Mutex**: Database, storage, and cache selections are mutually exclusive

+## 🔐 Docker Registry Secret
+
+If you're using a private Docker registry (e.g., for Dify EE images), you can use the provided script to create an imagePullSecret:
+
+```bash
+./generate-image-repo-secret.sh
+```
+
+The script will interactively prompt you for:
+- Docker Registry username
+- Docker Registry password
+- Kubernetes namespace (default: `default`)
+- Registry URL (default: `https://index.docker.io/v1/`)
+
+It creates a Kubernetes secret named `image-repo-secret` that can be referenced in your Helm values.
+
 ## 🔒 Security

 - Generated `values-prd-{version}.yaml` files contain sensitive information and are gitignored
@@ -120,8 +120,9 @@ python generate-values-prd.py --lang zh

 ```
 .
-├── generate-values-prd.py    # 主脚本文件
-├── values.yaml               # 基础配置文件模板（自动从 Helm Chart 仓库下载）
+├── generate-values-prd.py       # 主脚本文件
+├── generate-image-repo-secret.sh # Docker 镜像仓库密钥生成器
+├── values.yaml                  # 基础配置文件模板（自动从 Helm Chart 仓库下载）
 ├── values-prd.yaml          # 生成的生产环境配置（gitignore）
 ├── .cache/                  # 缓存目录（存储下载的 values.yaml）
 ├── pyproject.toml           # Python 项目配置
@@ -188,6 +189,22 @@ python generate-values-prd.py --lang zh
 - **TLS 联动**: TLS 配置与 Ingress 自动同步，避免 CORS 问题
 - **基础设施互斥**: 数据库、存储、缓存的选择互斥

+## 🔐 Docker 镜像仓库密钥
+
+如果你使用私有 Docker 镜像仓库（例如 Dify EE 镜像），可以使用提供的脚本创建 imagePullSecret：
+
+```bash
+./generate-image-repo-secret.sh
+```
+
+脚本会交互式地提示你输入：
+- Docker Registry 用户名
+- Docker Registry 密码
+- Kubernetes 命名空间（默认：`default`）
+- Registry 地址（默认：`https://index.docker.io/v1/`）
+
+脚本会创建一个名为 `image-repo-secret` 的 Kubernetes secret，可以在 Helm values 中引用。
+
 ## 🔒 安全注意事项

 - 生成的 `values-prd.yaml` 包含敏感信息，已添加到 `.gitignore`
@@ -25,3 +25,21 @@ DOWNLOAD_TIMEOUT = 10
 DEFAULT_CHART_VERSION = None  # None means latest
 DEFAULT_EE_VERSION = None  # None means interactive selection

+# CI Mode Configuration
+class _Config:
+    """Internal configuration state"""
+    ci_mode: bool = False
+
+
+_config = _Config()
+
+
+def set_ci_mode(enabled: bool = True):
+    """Enable or disable CI mode globally"""
+    _config.ci_mode = enabled
+
+
+def is_ci_mode() -> bool:
+    """Check if CI mode is enabled"""
+    return _config.ci_mode
+
@@ -0,0 +1,65 @@
+# CI 集成测试计划
+
+> 待实施的 CI 集成测试增强计划
+
+## 阶段 1: 添加非交互模式
+
+在 `generate-values-prd.py` 添加 `--ci` 参数：
+
+```python
+parser.add_argument(
+    "--ci", "--non-interactive",
+    action="store_true",
+    help="CI mode: use default values for all prompts"
+)
+```
+
+修改 `utils/prompts.py` 中的提示函数，支持自动使用默认值。
+
+## 阶段 2: 创建集成测试
+
+```
+tests/
+├── conftest.py              # pytest 配置
+├── test_integration.py      # 集成测试
+└── profiles/
+    └── standard.yaml        # 标准环境预设配置
+```
+
+**测试内容：**
+
+| 测试项 | 说明 |
+|--------|------|
+| YAML 有效性 | 生成的文件是有效的 YAML |
+| 必需字段检查 | 包含所有必需的配置项 |
+| 版本特性验证 | 3.7+ 包含 triggerDomain 等 |
+| 密钥生成验证 | 所有密钥已生成且长度正确 |
+
+## 阶段 3: 更新 CI 配置
+
+```yaml
+# .github/workflows/ci.yml
+integration-test:
+  strategy:
+    matrix:
+      chart-version: ['3.6.0', '3.6.5', '3.7.0', '3.7.1', '3.7.2']
+  steps:
+    - name: Install Helm
+      uses: azure/setup-helm@v3
+    
+    - name: Generate values for ${{ matrix.chart-version }}
+      run: |
+        python generate-values-prd.py \
+          --chart-version ${{ matrix.chart-version }} \
+          --ci --lang en
+    
+    - name: Validate generated YAML
+      run: python -m pytest tests/test_integration.py
+```
+
+## 状态
+
+- [ ] 阶段 1: 非交互模式
+- [ ] 阶段 2: 集成测试
+- [ ] 阶段 3: CI 配置更新
+
@@ -0,0 +1,64 @@
+#!/bin/bash    
+
+echo "=========================================="
+echo "Docker Registry Secret Generator"
+echo "=========================================="
+echo ""
+
+# 交互式获取用户名
+read -p "请输入 Docker Registry 用户名: " USERNAME
+if [ -z "$USERNAME" ]; then
+    echo "错误: 用户名不能为空"
+    exit 1
+fi
+
+# 交互式获取密码（隐藏输入）
+read -sp "请输入 Docker Registry 密码: " PASSWORD
+echo ""
+if [ -z "$PASSWORD" ]; then
+    echo "错误: 密码不能为空"
+    exit 1
+fi
+
+# 交互式获取 Kubernetes 命名空间
+read -p "请输入 Kubernetes 命名空间 [default]: " NAMESPACE
+NAMESPACE="${NAMESPACE:-default}"
+
+# 交互式获取 Registry 地址（带默认值）
+read -p "请输入 Docker Registry 地址 [https://index.docker.io/v1/]: " REGISTRY
+REGISTRY="${REGISTRY:-https://index.docker.io/v1/}"
+
+OUTPUT_FILE="./config.json"
+
+echo ""
+echo "正在生成 Docker config.json 文件..."
+
+AUTH=$(echo -n "$USERNAME:$PASSWORD" | base64 | tr -d '\n')
+
+cat > "$OUTPUT_FILE" <<EOF
+{
+"auths": {
+    "$REGISTRY": {
+    "auth": "$AUTH"
+    }
+}
+}
+EOF
+
+echo "Docker config.json 已生成: $OUTPUT_FILE"
+echo ""
+echo "正在创建 Kubernetes Secret..."
+kubectl -n $NAMESPACE create secret generic image-repo-secret --from-file=.dockerconfigjson=$OUTPUT_FILE --from-file=config.json=$OUTPUT_FILE --type=kubernetes.io/dockerconfigjson
+
+if [ $? -eq 0 ]; then
+    echo ""
+    echo "✓ Secret 'image-repo-secret' 已在命名空间 '$NAMESPACE' 中成功创建！"
+else
+    echo ""
+    echo "✗ 创建 Secret 失败，请检查错误信息"
+    rm "$OUTPUT_FILE"
+    exit 1
+fi
+
+rm "$OUTPUT_FILE"
+echo "临时文件已清理"
@@ -91,9 +91,19 @@ Examples:
        default=config.HELM_REPO_NAME,
        help=f"Helm repository name (default: {config.HELM_REPO_NAME})"
    )
+    parser.add_argument(
+        "--ci", "--non-interactive",
+        action="store_true",
+        dest="ci_mode",
+        help="CI mode: use default values for all prompts (non-interactive)"
+    )

    args = parser.parse_args()

+    # Enable CI mode if requested
+    if args.ci_mode:
+        config.set_ci_mode(True)
+
    # Language selection
    if args.lang:
        set_language(args.lang)
@@ -3,12 +3,19 @@
 import sys
 from utils import Colors, print_header, print_info, print_success, print_error
 from .translations import set_language, get_language, Translations
+import config

 _t = Translations.get


 def prompt_language_selection() -> str:
-    """Prompt user to select language"""
+    """Prompt user to select language. In CI mode, defaults to English."""
+    # CI mode: use English without prompting
+    if config.is_ci_mode():
+        set_language('en')
+        print_info("[CI] Language set to: English")
+        return 'en'
+
    print_header(_t('select_language', language='en'))

    languages = [
@@ -7,6 +7,10 @@ import sys
 import os
 import importlib.util

+# Enable CI mode for non-interactive testing
+import config
+config.set_ci_mode(True)
+
 # 加载 generate-values-prd.py 模块
 spec = importlib.util.spec_from_file_location("generate_values_prd", "generate-values-prd.py")
 generate_values_prd = importlib.util.module_from_spec(spec)
@@ -8,6 +8,10 @@ import sys
 import os
 import importlib.util

+# Enable CI mode for non-interactive testing
+import config
+config.set_ci_mode(True)
+
 # 加载 generate-values-prd.py 模块
 spec = importlib.util.spec_from_file_location("generate_values_prd", "generate-values-prd.py")
 generate_values_prd = importlib.util.module_from_spec(spec)
@@ -6,6 +6,11 @@ Test Module 6: Service Configuration

 import os
 import sys
+
+# Enable CI mode for non-interactive testing
+import config
+config.set_ci_mode(True)
+
 from i18n import set_language, get_translator
 from i18n.language import prompt_language_selection
 from generator import ValuesGenerator
@@ -17,7 +22,7 @@ def test_services_module():
    print("Test Module 6: Service Configuration")
    print("=" * 60)
    
-    # Language selection
+    # Language selection (will use English in CI mode)
    prompt_language_selection()
    _t = get_translator()
    
@@ -246,6 +246,16 @@ def prompt_helm_chart_version(
    repo_url = repo_url or config.HELM_REPO_URL
    repo_name = repo_name or config.HELM_REPO_NAME

+    # CI mode: return latest version without prompting
+    if config.is_ci_mode():
+        print_info("[CI] Fetching latest version...")
+        versions = get_published_versions(chart_name, repo_url, repo_name)
+        if versions:
+            latest = versions[0]
+            print_info(f"[CI] Using latest version: {latest}")
+            return latest
+        return None
+
    # Prompt user to choose version source
    print_info("")
    version_source = prompt_choice(
@@ -1,14 +1,28 @@
 """User interaction prompts"""

 from typing import Optional
-from .colors import Colors, print_error
+from .colors import Colors, print_error, print_info
 from i18n import get_translator
+import config

 _t = get_translator()


 def prompt(prompt_text: str, default: Optional[str] = None, required: bool = True) -> str:
-    """Prompt user for input"""
+    """Prompt user for input. In CI mode, returns default value automatically."""
+    # CI mode: return default without prompting
+    if config.is_ci_mode():
+        if default:
+            print_info(f"[CI] {prompt_text}: {default}")
+            return default
+        elif not required:
+            print_info(f"[CI] {prompt_text}: (empty)")
+            return ""
+        else:
+            # Required field with no default - use placeholder
+            print_info(f"[CI] {prompt_text}: (required, using placeholder)")
+            return "ci-placeholder"
+
    if default:
        prompt_str = f"{Colors.BOLD}{prompt_text}{Colors.ENDC} [{default}]: "
    else:
@@ -27,7 +41,13 @@ def prompt(prompt_text: str, default: Optional[str] = None, required: bool = Tru


 def prompt_yes_no(prompt_text: str, default: bool = True) -> bool:
-    """Prompt yes/no choice"""
+    """Prompt yes/no choice. In CI mode, returns default value automatically."""
+    # CI mode: return default without prompting
+    if config.is_ci_mode():
+        result_str = "Yes" if default else "No"
+        print_info(f"[CI] {prompt_text}: {result_str}")
+        return default
+
    default_str = "Y/n" if default else "y/N"
    prompt_str = f"{Colors.BOLD}{prompt_text}{Colors.ENDC} [{default_str}]: "

@@ -44,7 +64,13 @@ def prompt_yes_no(prompt_text: str, default: bool = True) -> bool:


 def prompt_choice(prompt_text: str, choices: list, default: Optional[str] = None) -> str:
-    """Prompt for choice"""
+    """Prompt for choice. In CI mode, returns default or first choice automatically."""
+    # CI mode: return default or first choice without prompting
+    if config.is_ci_mode():
+        result = default if default else choices[0]
+        print_info(f"[CI] {prompt_text}: {result}")
+        return result
+
    print(f"\n{Colors.BOLD}{prompt_text}{Colors.ENDC}")
    default_marker = _t('default')
    for i, choice in enumerate(choices, 1):
@@ -0,0 +1,24 @@
+"""Secret generation utilities"""
+
+import subprocess
+from .colors import print_warning
+from i18n import get_translator
+
+_t = get_translator()
+
+
+def generate_secret(length: int = 42) -> str:
+    """Generate secret key"""
+    try:
+        result = subprocess.run(
+            ['openssl', 'rand', '-base64', str(length)],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        return result.stdout.strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        print_warning(_t('openssl_fallback'))
+        import secrets
+        return secrets.token_urlsafe(length)
+
@@ -3,6 +3,7 @@
 import sys
 from typing import Dict, Any, Optional

+import config
 from utils import Colors, print_header, print_info, print_success, print_error, print_warning
 from i18n import get_translator

@@ -55,9 +56,9 @@ class VersionManager:
    @classmethod
    def get_version_modules(cls, version: str) -> list:
        """Get list of modules supported by version"""
-        config = cls.get_version_info(version)
-        if config:
-            return config.get("modules", [])
+        version_cfg = cls.get_version_info(version)
+        if version_cfg:
+            return version_cfg.get("modules", [])
        return []

    @classmethod
@@ -68,19 +69,27 @@ class VersionManager:

    @classmethod
    def prompt_version_selection(cls) -> str:
-        """Interactive version selection"""
+        """Interactive version selection. In CI mode, returns first (latest) version."""
+        versions = cls.get_available_versions()
+
+        # CI mode: return first version without prompting
+        if config.is_ci_mode():
+            first_version = versions[0] if versions else "3.x"
+            version_config = cls.get_version_info(first_version)
+            print_info(f"[CI] Using version: {version_config.get('name', first_version)}")
+            return first_version
+
        print_header(_t('select_dify_version'))
        print_info(_t('select_version_prompt'))
        print()

-        versions = cls.get_available_versions()
        version_options = []

        for i, version in enumerate(versions, 1):
-            config = cls.get_version_info(version)
-            name = config.get("name", f"Version {version}")
-            desc = config.get("description", "")
-            modules = config.get("modules", [])
+            version_cfg = cls.get_version_info(version)
+            name = version_cfg.get("name", f"Version {version}")
+            desc = version_cfg.get("description", "")
+            modules = version_cfg.get("modules", [])

            print(f"  {i}. {name}")
            print(f"     {_t('version')}: {version}")
@@ -102,8 +111,8 @@ class VersionManager:
                idx = int(choice) - 1
                if 0 <= idx < len(versions):
                    selected_version = version_options[idx]
-                    config = cls.get_version_info(selected_version)
-                    print_success(f"{_t('selected')}: {config.get('name', selected_version)}")
+                    version_cfg = cls.get_version_info(selected_version)
+                    print_success(f"{_t('selected')}: {version_cfg.get('name', selected_version)}")
                    return selected_version
                else:
                    range_text = _t('enter_number_range')