From a506c3e9b9c6b86502c7688c38b6372104062905 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 3 Jan 2026 11:04:32 +0000 Subject: [PATCH 1/8] Initial plan From 7e1150f2d314b601e46aa63082b994a4e748bf55 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 3 Jan 2026 11:14:10 +0000 Subject: [PATCH 2/8] Add comprehensive documentation, workflows, and developer tooling improvements - Add Dependabot configuration for automated dependency updates - Add dependency review workflow for security scanning - Add pre-commit hooks configuration with linters - Add EditorConfig for consistent coding style - Create ARCHITECTURE.md with system design documentation - Create TROUBLESHOOTING.md for common issues - Create DEVELOPMENT_GUIDE.md with best practices - Create TESTING.md with comprehensive testing guide - Enhance PR template with detailed checklist - Add documentation and question issue templates - Add spell-check workflow with typos configuration - Add link-check workflow for documentation - Enhance README with quick links navigation table - Add workflow concurrency controls to save CI resources Co-authored-by: Ayushmore1214 <194600182+Ayushmore1214@users.noreply.github.com> --- .editorconfig | 59 ++ .github/ISSUE_TEMPLATE/documentation.md | 34 ++ .github/ISSUE_TEMPLATE/question.md | 29 + .github/PULL_REQUEST_TEMPLATE.md | 51 ++ .github/dependabot.yml | 92 +++ .github/workflows/build.yaml | 5 + .github/workflows/dependency-review.yaml | 28 + .github/workflows/link-check.yaml | 28 + .github/workflows/lint.yaml | 5 + .github/workflows/spell-check.yaml | 23 + .github/workflows/test.yaml | 5 + .markdown-link-check.json | 27 + .pre-commit-config.yaml | 57 ++ .typos.toml | 35 ++ .yamllint.yaml | 22 + ARCHITECTURE.md | 327 +++++++++++ DEVELOPMENT_GUIDE.md | 623 +++++++++++++++++++++ README.md | 14 + TESTING.md | 685 +++++++++++++++++++++++ TROUBLESHOOTING.md | 649 +++++++++++++++++++++ 20 files changed, 2798 insertions(+) create mode 100644 .editorconfig create mode 100644 .github/ISSUE_TEMPLATE/documentation.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/dependency-review.yaml create mode 100644 .github/workflows/link-check.yaml create mode 100644 .github/workflows/spell-check.yaml create mode 100644 .markdown-link-check.json create mode 100644 .pre-commit-config.yaml create mode 100644 .typos.toml create mode 100644 .yamllint.yaml create mode 100644 ARCHITECTURE.md create mode 100644 DEVELOPMENT_GUIDE.md create mode 100644 TESTING.md create mode 100644 TROUBLESHOOTING.md diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..055b74738f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,59 @@ +# EditorConfig helps maintain consistent coding styles across different editors and IDEs +# See https://editorconfig.org for more information + +root = true + +# Default settings for all files +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +# Go files +[*.go] +indent_style = tab +indent_size = 4 + +# Makefile +[Makefile] +indent_style = tab + +# Markdown files +[*.md] +trim_trailing_whitespace = false +max_line_length = off + +# YAML files +[*.{yml,yaml}] +indent_size = 2 + +# JSON files +[*.json] +indent_size = 2 + +# JavaScript/TypeScript +[*.{js,jsx,ts,tsx}] +indent_size = 2 + +# Shell scripts +[*.sh] +indent_size = 2 + +# Python files +[*.py] +indent_size = 4 + +# HTML files +[*.html] +indent_size = 2 + +# CSS/SCSS files +[*.{css,scss}] +indent_size = 2 + +# Dockerfile +[Dockerfile*] +indent_size = 2 diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000000..c09c0c6d86 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,34 @@ +--- +name: Documentation Issue +about: Report issues with documentation or suggest improvements +title: '[DOCS] ' +labels: 'documentation' +assignees: '' +--- + +## Documentation Issue + +### Type of Issue +- [ ] Documentation is incorrect or outdated +- [ ] Documentation is missing +- [ ] Documentation is unclear or confusing +- [ ] Suggestion for improvement + +### Location +**URL or file path where the issue exists**: + +### Description +**What's wrong or missing?**: + +**Expected documentation**: + +**Actual documentation**: + +### Suggested Fix +(If you have suggestions on how to improve the documentation) + +### Additional Context +(Add any other context, screenshots, or examples) + +### Willing to Contribute +- [ ] I'm willing to submit a PR to fix this diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000000..1bbc5e19bd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,29 @@ +--- +name: Question +about: Ask a question about PipeCD +title: '[QUESTION] ' +labels: 'question' +assignees: '' +--- + +## Question + +### Question +**What would you like to know?**: + +### Context +**What are you trying to accomplish?**: + +**What have you tried so far?**: + +### Environment (if relevant) +- PipeCD version: +- Platform (Kubernetes, Terraform, etc.): +- Deployment environment: + +### Additional Information +(Add any other context, logs, or screenshots that might help) + +--- + +**Note**: For general discussions or open-ended questions, consider using [GitHub Discussions](https://github.com/pipe-cd/pipecd/discussions) instead. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6916d37bd5..bea230a767 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,5 @@ +## Description + **What this PR does**: **Why we need it**: @@ -6,8 +8,57 @@ Fixes # +## Type of Change + +Please select the relevant options: + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update +- [ ] Performance improvement +- [ ] Code refactoring +- [ ] Dependencies update +- [ ] CI/CD changes + +## Testing + +- [ ] Unit tests added/updated +- [ ] Integration tests added/updated +- [ ] Manual testing performed +- [ ] All existing tests pass + +**Test coverage**: (if applicable) + +**Testing steps**: +1. +2. +3. + +## Checklist + +- [ ] My code follows the project's coding standards +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings or errors +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes +- [ ] I have run `make check` and resolved all issues +- [ ] I have signed my commits (`git commit -s`) + +## User-Facing Changes + **Does this PR introduce a user-facing change?**: - **How are users affected by this change**: - **Is this breaking change**: - **How to migrate (if breaking change)**: + +## Screenshots/Recordings + +(If applicable, add screenshots or recordings to help explain your changes) + +## Additional Notes + +(Any additional information that reviewers should know) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..0792b6fd6f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,92 @@ +# Dependabot configuration for automated dependency updates +# This helps keep dependencies secure and up-to-date +version: 2 +updates: + # Enable version updates for Go modules + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "go" + commit-message: + prefix: "chore(deps)" + groups: + go-dependencies: + patterns: + - "*" + update-types: + - "minor" + - "patch" + + # Enable version updates for npm (web directory) + - package-ecosystem: "npm" + directory: "/web" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "javascript" + commit-message: + prefix: "chore(deps)" + groups: + npm-dependencies: + patterns: + - "*" + update-types: + - "minor" + - "patch" + + # Enable version updates for npm (docs directory) + - package-ecosystem: "npm" + directory: "/docs" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "javascript" + - "documentation" + commit-message: + prefix: "chore(deps)" + + # Enable version updates for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "github-actions" + commit-message: + prefix: "chore(deps)" + groups: + github-actions: + patterns: + - "*" + + # Enable version updates for Docker + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + open-pull-requests-limit: 3 + labels: + - "dependencies" + - "docker" + commit-message: + prefix: "chore(deps)" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dee768c3ba..6e2af601ab 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -12,6 +12,11 @@ on: - 'release-v*' - 'feat/*' +# Cancel in-progress runs for the same workflow and ref +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: GO_VERSION: 1.25.0 NODE_VERSION: 18.12.0 diff --git a/.github/workflows/dependency-review.yaml b/.github/workflows/dependency-review.yaml new file mode 100644 index 0000000000..36a8ef703d --- /dev/null +++ b/.github/workflows/dependency-review.yaml @@ -0,0 +1,28 @@ +name: dependency-review + +on: + pull_request: + branches: + - master + - 'release-v*' + +permissions: + contents: read + pull-requests: write + +jobs: + dependency-review: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Dependency Review + uses: actions/dependency-review-action@v4 + with: + # Fail the build if there are any vulnerabilities + fail-on-severity: moderate + # Comment on the PR with the dependency review results + comment-summary-in-pr: always + # Allow licenses + allow-licenses: Apache-2.0, MIT, BSD-2-Clause, BSD-3-Clause, ISC diff --git a/.github/workflows/link-check.yaml b/.github/workflows/link-check.yaml new file mode 100644 index 0000000000..92c33c56b9 --- /dev/null +++ b/.github/workflows/link-check.yaml @@ -0,0 +1,28 @@ +name: link-check + +on: + pull_request: + branches: + - master + - 'release-v*' + paths: + - '**.md' + - 'docs/**' + schedule: + # Run weekly on Monday at 00:00 UTC + - cron: '0 0 * * 1' + workflow_dispatch: + +jobs: + markdown-link-check: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Check links in markdown files + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + config-file: '.markdown-link-check.json' + use-quiet-mode: 'yes' + use-verbose-mode: 'no' diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index db70d646f4..6f676301e3 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -10,6 +10,11 @@ on: - "release-v*" - "feat/*" +# Cancel in-progress runs for the same workflow and ref +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: GO_VERSION: 1.25.0 NODE_VERSION: 18.12.0 diff --git a/.github/workflows/spell-check.yaml b/.github/workflows/spell-check.yaml new file mode 100644 index 0000000000..9014b84710 --- /dev/null +++ b/.github/workflows/spell-check.yaml @@ -0,0 +1,23 @@ +name: spell-check + +on: + pull_request: + branches: + - master + - 'release-v*' + paths: + - '**.md' + - 'docs/**' + - '.github/workflows/spell-check.yaml' + +jobs: + spell-check: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Check spelling + uses: crate-ci/typos@v1.24.6 + with: + config: .typos.toml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b91cf1612d..a8df8d4bf7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,6 +12,11 @@ on: - 'release-v*' - 'feat/*' +# Cancel in-progress runs for the same workflow and ref +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: GO_VERSION: 1.25.0 NODE_VERSION: 18.12.0 diff --git a/.markdown-link-check.json b/.markdown-link-check.json new file mode 100644 index 0000000000..6ab9883750 --- /dev/null +++ b/.markdown-link-check.json @@ -0,0 +1,27 @@ +{ + "ignorePatterns": [ + { + "pattern": "^http://localhost" + }, + { + "pattern": "^http://127.0.0.1" + }, + { + "pattern": "^https://pipecd.dev/docs/" + } + ], + "replacementPatterns": [], + "httpHeaders": [ + { + "urls": ["https://github.com"], + "headers": { + "Accept": "application/vnd.github.v3+json" + } + } + ], + "timeout": "20s", + "retryOn429": true, + "retryCount": 3, + "fallbackRetryDelay": "30s", + "aliveStatusCodes": [200, 206, 299, 403] +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..3d8cc0b3e2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,57 @@ +# Pre-commit hooks for maintaining code quality +# Install: pip install pre-commit +# Setup: pre-commit install +# Run manually: pre-commit run --all-files + +repos: + # General file checks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + - id: check-yaml + args: [--allow-multiple-documents] + - id: check-added-large-files + args: ['--maxkb=1024'] + - id: check-merge-conflict + - id: check-case-conflict + - id: mixed-line-ending + - id: detect-private-key + - id: check-json + exclude: ^web/ + + # Go formatting and linting + - repo: https://github.com/golangci/golangci-lint + rev: v2.4.0 + hooks: + - id: golangci-lint + args: [--config=.golangci.yml] + + # Markdown linting + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.41.0 + hooks: + - id: markdownlint + args: [--fix] + exclude: ^docs/ + + # YAML linting + - repo: https://github.com/adrienverge/yamllint + rev: v1.35.1 + hooks: + - id: yamllint + args: [-c=.yamllint.yaml] + + # Shell script linting + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + + # Dockerfile linting + - repo: https://github.com/hadolint/hadolint + rev: v2.12.0 + hooks: + - id: hadolint-docker diff --git a/.typos.toml b/.typos.toml new file mode 100644 index 0000000000..d3764633d4 --- /dev/null +++ b/.typos.toml @@ -0,0 +1,35 @@ +# Configuration for typos spell checker +# See https://github.com/crate-ci/typos + +[default] +extend-ignore-re = [ + # Ignore git SHAs + "[0-9a-f]{7,40}", + # Ignore base64 encoded strings + "[A-Za-z0-9+/]{20,}={0,2}", +] + +[files] +extend-exclude = [ + "*.sum", + "*.lock", + "package-lock.json", + "yarn.lock", + ".artifacts/", + "vendor/", + "node_modules/", + "web/build/", + "docs/public/", +] + +[default.extend-words] +# Add project-specific words that are not typos +pipecd = "pipecd" +piped = "piped" +pipectl = "pipectl" +kubectl = "kubectl" +kubernetes = "kubernetes" +kubeconfig = "kubeconfig" +gRPC = "gRPC" +GitOps = "GitOps" +CNCF = "CNCF" diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000000..fb8a4e94fa --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,22 @@ +# YAML linting configuration for pre-commit hooks +--- +extends: default + +rules: + line-length: + max: 120 + level: warning + indentation: + spaces: 2 + comments: + min-spaces-from-content: 1 + comments-indentation: {} + document-start: disable + truthy: + allowed-values: ['true', 'false', 'on', 'off'] + +ignore: | + .github/ + web/ + docs/ + manifests/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000000..ade23a8a64 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,327 @@ +# PipeCD Architecture + +This document provides a comprehensive overview of PipeCD's architecture, components, and design principles. + +## Table of Contents + +- [Overview](#overview) +- [Core Components](#core-components) +- [Architecture Diagram](#architecture-diagram) +- [Component Interaction](#component-interaction) +- [Data Flow](#data-flow) +- [Security Model](#security-model) +- [Scalability & Performance](#scalability--performance) +- [Technology Stack](#technology-stack) + +## Overview + +PipeCD is a GitOps-based continuous delivery platform designed for managing deployments across multiple application types and cloud platforms. The architecture follows a control plane/agent model with strong security principles. + +### Design Principles + +1. **GitOps-First**: All deployment configurations and application manifests are stored in Git +2. **Multi-Cloud Native**: Support for Kubernetes, Terraform, Cloud Run, Lambda, ECS, and more +3. **Security by Design**: No credentials required outside the application cluster +4. **Scalability**: Designed to handle thousands of applications across multiple environments +5. **Observability**: Built-in metrics, logging, and deployment insights + +## Core Components + +### 1. Control Plane (PipeCD Server) + +The control plane is the centralized management component that: + +- **Responsibilities**: + - Stores deployment metadata and history + - Provides gRPC API for piped agents + - Manages authentication and authorization + - Serves the web UI + - Aggregates metrics and insights + +- **Technology**: Go, gRPC, Protocol Buffers +- **Storage**: Supports multiple datastores (MySQL, PostgreSQL, Firestore, etc.) +- **Deployment**: Can run as a Kubernetes deployment or standalone binary + +### 2. Piped (Agent) + +Piped is the agent component that runs in your infrastructure: + +- **Responsibilities**: + - Watches Git repositories for changes + - Executes deployments according to pipeline definitions + - Reports deployment status to control plane + - Performs health checks and analysis + - Manages platform-specific operations (kubectl, terraform, etc.) + +- **Technology**: Go, plugin architecture +- **Deployment**: Runs as a pod in Kubernetes or as a standalone process +- **Security**: Holds all deployment credentials locally + +### 3. Launcher + +A helper component for remote upgrades: + +- **Responsibilities**: + - Manages piped agent lifecycle + - Enables remote upgrade capabilities + - Ensures piped availability + +### 4. Pipectl + +Command-line tool for PipeCD: + +- **Responsibilities**: + - Register/manage pipeds + - Encrypt secrets + - Trigger deployments + - Query deployment status + +### 5. Web UI + +Modern web interface for PipeCD: + +- **Responsibilities**: + - Visualize deployments and pipelines + - Manage applications and environments + - View insights and metrics + - Configure settings + +- **Technology**: TypeScript, React, Material-UI + +## Architecture Diagram + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Users/Developers │ +└───────────────┬─────────────────────────────┬───────────────────┘ + │ │ + │ Web UI │ pipectl CLI + │ │ + ▼ ▼ +┌───────────────────────────────────────────────────────────────────┐ +│ │ +│ Control Plane (PipeCD) │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────┐ │ +│ │ Web Server │ │ gRPC Server │ │ Auth Service │ │ +│ └──────────────┘ └──────────────┘ └────────────────────┘ │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────┐ │ +│ │ API Service │ │ Datastore │ │ Insight Aggregator│ │ +│ └──────────────┘ └──────────────┘ └────────────────────┘ │ +│ │ +└─────────────────────────────┬─────────────────────────────────────┘ + │ + │ gRPC Communication + │ + ┌─────────────────────┼─────────────────────┐ + │ │ │ + ▼ ▼ ▼ +┌───────────────┐ ┌───────────────┐ ┌───────────────┐ +│ │ │ │ │ │ +│ Piped Agent │ │ Piped Agent │ │ Piped Agent │ +│ (Cluster A) │ │ (Cluster B) │ │ (Cloud Env) │ +│ │ │ │ │ │ +│ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ +│ │Git Sync │ │ │ │Git Sync │ │ │ │Git Sync │ │ +│ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ +│ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ +│ │Deployer │ │ │ │Deployer │ │ │ │Deployer │ │ +│ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ +│ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ +│ │Analyzer │ │ │ │Analyzer │ │ │ │Analyzer │ │ +│ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ +│ │ │ │ │ │ +└───────┬───────┘ └───────┬───────┘ └───────┬───────┘ + │ │ │ + ▼ ▼ ▼ +┌───────────────┐ ┌───────────────┐ ┌───────────────┐ +│ Kubernetes │ │ Kubernetes │ │ Cloud Run / │ +│ Cluster │ │ Cluster │ │ Lambda / ECS │ +└───────────────┘ └───────────────┘ └───────────────┘ + │ │ │ + ▼ ▼ ▼ +┌───────────────────────────────────────────────────────────┐ +│ Git Repositories │ +│ (Application Manifests & Pipeline Configs) │ +└───────────────────────────────────────────────────────────┘ +``` + +## Component Interaction + +### Deployment Flow + +1. **Change Detection**: + - Piped polls Git repository at regular intervals (configurable) + - Detects changes in application manifests or pipeline definitions + +2. **Deployment Creation**: + - Piped creates deployment request + - Sends deployment plan to Control Plane + - Control Plane stores deployment metadata + +3. **Deployment Execution**: + - Piped executes deployment pipeline stages + - Reports progress to Control Plane + - Performs automated analysis (if configured) + +4. **Status Updates**: + - Real-time updates sent to Control Plane + - Web UI reflects current deployment state + - Notifications sent (if configured) + +### Communication Patterns + +- **Control Plane ↔ Piped**: Bidirectional gRPC streams +- **Control Plane ↔ Web UI**: HTTP/WebSocket +- **Control Plane ↔ Datastore**: Database protocol +- **Piped ↔ Git**: Git protocol (SSH/HTTPS) +- **Piped ↔ Platform**: Platform-specific APIs (kubectl, terraform, etc.) + +## Data Flow + +### Deployment Data + +``` +Git Repo → Piped (sync) → Piped (plan) → Control Plane (store) + ↓ + Platform (apply) + ↓ + Control Plane (update status) + ↓ + Web UI (display) +``` + +### Metrics & Insights + +``` +Piped (collect) → Control Plane (aggregate) → Web UI (visualize) +``` + +## Security Model + +### Key Security Features + +1. **Credential Isolation**: + - Deployment credentials never leave the piped environment + - Control Plane has no access to cluster credentials + +2. **Authentication**: + - Static admin accounts (for quickstart) + - SSO integration (GitHub, Google, etc.) + - RBAC for fine-grained access control + +3. **Encrypted Communication**: + - TLS for all gRPC connections + - mTLS support for enhanced security + +4. **Secret Management**: + - Secrets encrypted at rest + - Support for external secret managers (coming soon) + +5. **Audit Logging**: + - All API calls logged + - Deployment history maintained + +## Scalability & Performance + +### Horizontal Scaling + +- **Control Plane**: Can run multiple replicas behind a load balancer +- **Piped Agents**: One per environment/cluster (scales with infrastructure) + +### Performance Optimizations + +- **Caching**: Redis cache for frequently accessed data +- **Database Indexing**: Optimized queries for deployment history +- **Streaming**: gRPC streaming for real-time updates +- **Connection Pooling**: Efficient resource utilization + +### Resource Requirements + +**Control Plane**: +- CPU: 2-4 cores (production) +- Memory: 4-8 GB +- Storage: Depends on deployment history retention + +**Piped Agent**: +- CPU: 1-2 cores +- Memory: 1-2 GB +- Storage: Minimal (logs and temporary files) + +## Technology Stack + +### Backend + +- **Language**: Go 1.21+ +- **Framework**: gRPC, Protocol Buffers +- **Database**: MySQL, PostgreSQL, Firestore, DynamoDB +- **Cache**: Redis (optional) + +### Frontend + +- **Language**: TypeScript +- **Framework**: React 18 +- **UI Library**: Material-UI +- **Build Tool**: Webpack, Yarn + +### DevOps + +- **Container**: Docker +- **Orchestration**: Kubernetes +- **Packaging**: Helm +- **CI/CD**: GitHub Actions + +### Observability + +- **Metrics**: Prometheus-compatible endpoints +- **Logging**: Structured logging (JSON) +- **Tracing**: OpenTelemetry support + +## Plugin Architecture + +PipeCD supports a plugin system for platform providers: + +``` +┌─────────────────────────────────────┐ +│ Piped Core │ +├─────────────────────────────────────┤ +│ Plugin Interface (gRPC) │ +├─────────────────────────────────────┤ +│ ┌────────┐ ┌────────┐ ┌────────┐│ +│ │K8s │ │Terraform│ │Lambda ││ +│ │Plugin │ │Plugin │ │Plugin ││ +│ └────────┘ └────────┘ └────────┘│ +└─────────────────────────────────────┘ +``` + +### Supported Platforms + +- Kubernetes +- Terraform +- Cloud Run +- AWS Lambda +- AWS ECS +- Azure (coming soon) + +## References + +- [PipeCD Documentation](https://pipecd.dev/docs) +- [API Reference](https://pipecd.dev/docs/api-reference) +- [Design Proposals (RFCs)](./docs/rfcs) + +## Contributing + +To contribute to PipeCD architecture: + +1. Review existing [RFCs](./docs/rfcs) +2. Propose new features via GitHub Discussions +3. Submit design proposals for major changes +4. Follow the [Contributing Guide](./CONTRIBUTING.md) + +--- + +For questions about the architecture, please: +- Join our [Slack channel](https://cloud-native.slack.com/archives/C01B27F9T0X) +- Attend our [community meetings](https://bit.ly/pipecd-mtg-notes) +- Open a [GitHub Discussion](https://github.com/pipe-cd/pipecd/discussions) diff --git a/DEVELOPMENT_GUIDE.md b/DEVELOPMENT_GUIDE.md new file mode 100644 index 0000000000..4d9cb62516 --- /dev/null +++ b/DEVELOPMENT_GUIDE.md @@ -0,0 +1,623 @@ +# PipeCD Development Guide + +A comprehensive guide for developers contributing to PipeCD. + +## Table of Contents + +- [Getting Started](#getting-started) +- [Development Environment Setup](#development-environment-setup) +- [Project Structure](#project-structure) +- [Development Workflow](#development-workflow) +- [Coding Standards](#coding-standards) +- [Testing Guidelines](#testing-guidelines) +- [Debugging Tips](#debugging-tips) +- [Common Tasks](#common-tasks) +- [Performance Optimization](#performance-optimization) +- [Best Practices](#best-practices) + +## Getting Started + +### Prerequisites + +Ensure you have the following tools installed: + +```bash +# Required +- Go 1.21+ (check go.mod for exact version) +- Node.js 18.12+ (for web development) +- Yarn (for web dependencies) +- Docker (for container builds) +- kubectl (for Kubernetes operations) +- Make (for build automation) + +# Recommended +- Git 2.30+ +- A good IDE (VS Code, GoLand, or similar) +- pre-commit (for git hooks) +- grpcurl (for testing gRPC endpoints) +``` + +### Quick Setup + +```bash +# Clone the repository +git clone https://github.com/pipe-cd/pipecd.git +cd pipecd + +# Install pre-commit hooks (recommended) +pip install pre-commit +pre-commit install + +# Update dependencies +make update/go-deps +make update/web-deps + +# Verify everything works +make check +``` + +## Development Environment Setup + +### VS Code Setup + +Recommended extensions: + +```json +{ + "recommendations": [ + "golang.go", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "ms-kubernetes-tools.vscode-kubernetes-tools", + "redhat.vscode-yaml", + "streetsidesoftware.code-spell-checker" + ] +} +``` + +Save this as `.vscode/extensions.json`. + +Workspace settings (`.vscode/settings.json`): + +```json +{ + "go.lintTool": "golangci-lint", + "go.lintFlags": ["--config=.golangci.yml"], + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": true + }, + "[go]": { + "editor.defaultFormatter": "golang.go", + "editor.insertSpaces": false + }, + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[yaml]": { + "editor.defaultFormatter": "redhat.vscode-yaml" + } +} +``` + +### GoLand/IntelliJ Setup + +1. **Import Project**: Open the project directory +2. **Go SDK**: Set Go SDK to version specified in go.mod +3. **Code Style**: + - Go: Use tabs, tab size 4 + - TypeScript: Use spaces, indent size 2 +4. **Enable golangci-lint**: + - Settings → Tools → File Watchers → Add golangci-lint + - Arguments: `run --config .golangci.yml` + +## Project Structure + +``` +pipecd/ +├── cmd/ # Main applications +│ ├── pipecd/ # Control plane server +│ ├── piped/ # Agent (piped) +│ ├── pipectl/ # CLI tool +│ └── launcher/ # Launcher for remote upgrades +├── pkg/ # Shared libraries +│ ├── app/ # Application logic +│ ├── config/ # Configuration handling +│ ├── datastore/ # Database interfaces +│ ├── model/ # Data models +│ └── version/ # Version information +├── web/ # Frontend React application +│ ├── src/ +│ │ ├── components/ # React components +│ │ ├── modules/ # Feature modules +│ │ └── api/ # API clients +│ └── package.json +├── docs/ # Documentation site +├── manifests/ # Kubernetes manifests & Helm charts +├── examples/ # Example configurations +├── test/ # Integration tests +├── hack/ # Scripts for development +└── tool/ # Development tools +``` + +### Key Directories + +- **cmd/**: Each subdirectory contains a `main.go` for a binary +- **pkg/**: Reusable packages, organized by functionality +- **web/**: TypeScript/React frontend +- **manifests/**: Helm charts for deployment +- **docs/**: Hugo-based documentation site + +## Development Workflow + +### 1. Create a Feature Branch + +```bash +git checkout -b feature/your-feature-name +# or +git checkout -b fix/issue-description +``` + +### 2. Make Changes + +Follow the [coding standards](#coding-standards) and [testing guidelines](#testing-guidelines). + +### 3. Run Checks Locally + +```bash +# Format code +make fmt/go +make fmt/web + +# Run linters +make lint/go +make lint/web + +# Run tests +make test/go +make test/web + +# Or run all checks at once +make check +``` + +### 4. Commit Changes + +```bash +# Sign your commits (required) +git commit -s -m "Add feature X" + +# Follow commit message conventions +# Format: (): +# Examples: +# feat(piped): add support for Azure deployments +# fix(web): resolve deployment list filtering issue +# docs: update contributing guide +``` + +### 5. Push and Create PR + +```bash +git push origin feature/your-feature-name +# Then create a PR on GitHub +``` + +## Coding Standards + +### Go + +**Follow these conventions:** + +1. **Use gofmt and goimports**: + ```bash + gofmt -s -w . + goimports -w . + ``` + +2. **Error Handling**: + ```go + // Good: Handle errors explicitly + if err != nil { + return fmt.Errorf("failed to do X: %w", err) + } + + // Bad: Ignoring errors + _ = doSomething() + ``` + +3. **Context Propagation**: + ```go + // Always accept context as first parameter + func DoSomething(ctx context.Context, param string) error { + // ... + } + ``` + +4. **Naming Conventions**: + ```go + // Interfaces: end with -er + type Deployer interface { ... } + + // Constructors: NewXxx + func NewDeployer() *Deployer { ... } + + // Acronyms: keep consistent case + var apiURL string // Good + var apiUrl string // Bad + ``` + +5. **Comments**: + ```go + // Package comment: describe package purpose + package deployer + + // Public function: describe what it does + // DeployApplication deploys the specified application to the target environment. + func DeployApplication(ctx context.Context, app *Application) error { + // ... + } + ``` + +6. **Testing**: + ```go + // Test files: _test.go suffix + // Test functions: TestXxx + func TestDeployApplication(t *testing.T) { + // Use table-driven tests + tests := []struct { + name string + app *Application + want error + }{ + // test cases + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // test logic + }) + } + } + ``` + +### TypeScript/React + +1. **Use TypeScript Strictly**: + ```typescript + // Good: explicit types + interface DeploymentProps { + id: string; + status: DeploymentStatus; + } + + // Bad: using any + const data: any = ... + ``` + +2. **Component Structure**: + ```typescript + // Functional components with hooks + export const DeploymentList: FC = ({ projectId }) => { + const [deployments, setDeployments] = useState([]); + + useEffect(() => { + // fetch deployments + }, [projectId]); + + return
...
; + }; + ``` + +3. **File Organization**: + ``` + components/ + DeploymentList/ + index.tsx # Main component + index.test.tsx # Tests + styles.ts # Styles (if needed) + ``` + +4. **Styling**: + ```typescript + // Use Material-UI's styling solution + import { makeStyles } from '@material-ui/core/styles'; + + const useStyles = makeStyles((theme) => ({ + root: { + padding: theme.spacing(2), + }, + })); + ``` + +### YAML + +1. **Indentation**: Use 2 spaces +2. **Quotes**: Use double quotes for strings with special characters +3. **Comments**: Explain non-obvious configurations + +```yaml +# Good +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: my-app + labels: + team: platform + pipeline: + stages: + - name: K8S_SYNC + with: + # Wait for rollout to complete before marking as success + waitForRollout: true +``` + +## Testing Guidelines + +### Unit Tests + +**Go:** + +```bash +# Run all tests +make test/go + +# Run specific package tests +go test ./pkg/app/piped/... + +# Run with coverage +make test/go COVERAGE=true + +# Run specific test +go test -run TestDeployApplication ./pkg/app/piped/deployer +``` + +**TypeScript:** + +```bash +# Run all tests +make test/web + +# Run specific test file +yarn --cwd web test DeploymentList.test.tsx + +# Run in watch mode +yarn --cwd web test --watch +``` + +### Integration Tests + +```bash +# Run integration tests +make test/integration + +# Run specific integration test +go test -tags=integration ./test/integration/... +``` + +### Writing Good Tests + +1. **Table-Driven Tests** (Go): + ```go + func TestCalculate(t *testing.T) { + tests := []struct { + name string + input int + expected int + }{ + {"zero", 0, 0}, + {"positive", 5, 10}, + {"negative", -5, -10}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := Calculate(tt.input) + if got != tt.expected { + t.Errorf("got %d, want %d", got, tt.expected) + } + }) + } + } + ``` + +2. **Use Mocks Appropriately**: + ```go + // Generate mocks with mockgen + //go:generate mockgen -source=interface.go -destination=mock.go -package=mocks + ``` + +3. **Test Coverage**: + - Aim for >80% coverage on new code + - Focus on critical paths + - Don't test trivial getters/setters + +## Debugging Tips + +### Debugging Go Code + +1. **Use Delve**: + ```bash + # Install delve + go install github.com/go-delve/delve/cmd/dlv@latest + + # Debug a test + dlv test ./pkg/app/piped -- -test.run TestDeployApplication + + # Debug a binary + dlv exec .artifacts/piped -- --config-file=/path/to/config.yaml + ``` + +2. **Print Debugging**: + ```go + // Use structured logging + import "go.uber.org/zap" + + logger.Debug("deployment started", zap.String("id", deploymentID)) + ``` + +3. **Remote Debugging**: + ```bash + # Forward debug port + kubectl port-forward -n pipecd pod/piped-xxx 2345:2345 + + # Connect with IDE debugger + ``` + +### Debugging TypeScript/React + +1. **Browser DevTools**: + - Use React DevTools extension + - Check Console for errors + - Use Network tab for API calls + - Use Profiler for performance + +2. **VS Code Debugging**: + ```json + { + "type": "chrome", + "request": "launch", + "name": "Debug Web", + "url": "http://localhost:9090", + "webRoot": "${workspaceFolder}/web" + } + ``` + +## Common Tasks + +### Adding a New gRPC API + +1. **Define in proto file**: + ```protobuf + service PipedService { + rpc GetDeployment(GetDeploymentRequest) returns (GetDeploymentResponse) {} + } + ``` + +2. **Generate code**: + ```bash + make gen/api + ``` + +3. **Implement server**: + ```go + func (s *PipedService) GetDeployment(ctx context.Context, req *pipedservice.GetDeploymentRequest) (*pipedservice.GetDeploymentResponse, error) { + // implementation + } + ``` + +4. **Add client call** (web): + ```typescript + const deployment = await pipedClient.getDeployment({ id: deploymentId }); + ``` + +### Adding a New Platform Provider + +1. Create plugin directory: + ``` + pkg/app/pipedv1/plugin/yourplatform/ + ``` + +2. Implement provider interface +3. Register provider +4. Add tests +5. Update documentation + +### Updating Dependencies + +```bash +# Go dependencies +make update/go-deps + +# Web dependencies +make update/web-deps + +# Check for outdated dependencies +go list -u -m all +yarn --cwd web outdated +``` + +## Performance Optimization + +### Profiling + +**CPU Profiling**: + +```bash +# Enable profiling in config +# Access at http://localhost:6060/debug/pprof/ + +# Capture CPU profile +curl http://localhost:6060/debug/pprof/profile?seconds=30 > cpu.prof +go tool pprof cpu.prof +``` + +**Memory Profiling**: + +```bash +curl http://localhost:6060/debug/pprof/heap > heap.prof +go tool pprof heap.prof +``` + +### Database Optimization + +1. **Add indexes for frequent queries** +2. **Use connection pooling** +3. **Batch operations when possible** +4. **Monitor slow queries** + +### Frontend Optimization + +1. **Code splitting**: Use React.lazy() +2. **Memoization**: Use React.memo, useMemo, useCallback +3. **Virtualization**: For long lists +4. **Bundle analysis**: `yarn --cwd web analyze` + +## Best Practices + +### Security + +1. **Never commit secrets** +2. **Use context for cancellation** +3. **Validate all inputs** +4. **Sanitize user-provided data** +5. **Use prepared statements for SQL** + +### Code Organization + +1. **Keep functions small** (< 50 lines) +2. **Single responsibility** per function/component +3. **DRY** (Don't Repeat Yourself) +4. **SOLID principles** + +### Git Workflow + +1. **Commit early, commit often** +2. **Write meaningful commit messages** +3. **Keep commits atomic** +4. **Rebase before merging** (if needed) +5. **Sign all commits** (`git commit -s`) + +### Documentation + +1. **Update docs with code changes** +2. **Add comments for complex logic** +3. **Keep README files updated** +4. **Document breaking changes** + +## Resources + +- [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) +- [Effective Go](https://golang.org/doc/effective_go) +- [React Best Practices](https://react.dev/learn) +- [TypeScript Handbook](https://www.typescriptlang.org/docs/) +- [Material-UI Documentation](https://material-ui.com/) + +## Getting Help + +- **Slack**: [#pipecd on CNCF Slack](https://cloud-native.slack.com/archives/C01B27F9T0X) +- **Meetings**: [Community meetings](https://bit.ly/pipecd-mtg-notes) +- **Discussions**: [GitHub Discussions](https://github.com/pipe-cd/pipecd/discussions) + +--- + +Happy coding! 🚀 diff --git a/README.md b/README.md index 3064f6b753..2d3e9be0b4 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,20 @@ For more details, explore the [PipeCD Documentation](https://pipecd.dev/docs). --- +### Quick Links + +| Resource | Description | +|----------|-------------| +| 📚 [Documentation](https://pipecd.dev/docs/) | Complete documentation and guides | +| 🚀 [Quickstart](https://pipecd.dev/docs/quickstart/) | Get started in minutes | +| 🎮 [Playground](https://play.pipecd.dev?project=play) | Try PipeCD without installation | +| 🏗️ [Architecture](./ARCHITECTURE.md) | System design and components | +| 🔧 [Development Guide](./DEVELOPMENT_GUIDE.md) | Contributing and development setup | +| 🧪 [Testing Guide](./TESTING.md) | Writing and running tests | +| 🐛 [Troubleshooting](./TROUBLESHOOTING.md) | Common issues and solutions | +| 💬 [Slack Channel](https://cloud-native.slack.com/archives/C01B27F9T0X) | Join the community | +| 📅 [Community Meetings](https://bit.ly/pipecd-mtg-notes) | Bi-weekly meetings | + ### Getting Started - The [quickstart guide](https://pipecd.dev/docs/quickstart/) shows how to set up PipeCD components and deploy a hello-world application with PipeCD for testing purposes. diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000000..70a8c65d2e --- /dev/null +++ b/TESTING.md @@ -0,0 +1,685 @@ +# Testing Guide for PipeCD + +Comprehensive guide for writing and running tests in PipeCD. + +## Table of Contents + +- [Testing Philosophy](#testing-philosophy) +- [Test Types](#test-types) +- [Running Tests](#running-tests) +- [Writing Unit Tests](#writing-unit-tests) +- [Writing Integration Tests](#writing-integration-tests) +- [Testing Best Practices](#testing-best-practices) +- [Mocking](#mocking) +- [Test Coverage](#test-coverage) +- [CI/CD Testing](#cicd-testing) + +## Testing Philosophy + +PipeCD follows these testing principles: + +1. **Test Pyramid**: More unit tests, fewer integration tests, even fewer E2E tests +2. **Fast Feedback**: Tests should run quickly for rapid development +3. **Reliability**: Tests should be deterministic and not flaky +4. **Maintainability**: Tests should be easy to understand and update +5. **Coverage**: Critical paths should have high test coverage + +## Test Types + +### Unit Tests + +**Purpose**: Test individual functions/methods in isolation + +**Characteristics**: +- Fast execution (< 1 second per test) +- No external dependencies +- Use mocks for dependencies +- High code coverage + +**Location**: `*_test.go` files alongside source code + +### Integration Tests + +**Purpose**: Test interaction between components + +**Characteristics**: +- Test real integrations (database, APIs) +- Slower than unit tests +- May require setup/teardown +- Test realistic scenarios + +**Location**: `test/integration/` directory + +### End-to-End Tests + +**Purpose**: Test complete user workflows + +**Characteristics**: +- Test full system +- Slowest tests +- Most realistic +- Fewer in number + +**Location**: `test/e2e/` directory + +## Running Tests + +### Go Tests + +```bash +# Run all unit tests +make test/go + +# Run tests with coverage +make test/go COVERAGE=true + +# Run specific package +go test ./pkg/app/piped/deployer/... + +# Run specific test +go test -run TestDeploymentController ./pkg/app/piped/deployer/ + +# Run with verbose output +go test -v ./pkg/... + +# Run with race detector +go test -race ./pkg/... + +# Run integration tests +make test/integration + +# Run with timeout +go test -timeout 30s ./pkg/... +``` + +### Web Tests + +```bash +# Run all web tests +make test/web + +# Run specific test file +yarn --cwd web test DeploymentList.test.tsx + +# Run in watch mode +yarn --cwd web test --watch + +# Run with coverage +yarn --cwd web test --coverage + +# Update snapshots +yarn --cwd web test -u +``` + +### Quick Test Commands + +```bash +# Test everything +make test + +# Run only fast tests +make test/go SKIP_INTEGRATION=true + +# Run specific module tests +make test/go MODULES=./pkg/app/piped +``` + +## Writing Unit Tests + +### Go Unit Tests + +**Structure**: Follow table-driven test pattern + +```go +package deployer + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDeployApplication(t *testing.T) { + t.Parallel() // Run tests in parallel when possible + + tests := []struct { + name string + app *Application + expectError bool + errContains string + }{ + { + name: "successful deployment", + app: &Application{ + ID: "app-1", + Name: "test-app", + }, + expectError: false, + }, + { + name: "missing app ID", + app: &Application{ + Name: "test-app", + }, + expectError: true, + errContains: "app ID is required", + }, + } + + for _, tt := range tests { + tt := tt // Capture range variable + t.Run(tt.name, func(t *testing.T) { + t.Parallel() // Run sub-tests in parallel + + // Arrange + deployer := NewDeployer() + + // Act + err := deployer.Deploy(context.Background(), tt.app) + + // Assert + if tt.expectError { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + } else { + require.NoError(t, err) + } + }) + } +} +``` + +**Using Assertions**: + +```go +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExample(t *testing.T) { + // Use 'require' for critical checks (stops test on failure) + require.NotNil(t, obj) + require.NoError(t, err) + + // Use 'assert' for non-critical checks (continues test on failure) + assert.Equal(t, expected, actual) + assert.True(t, condition) + assert.Contains(t, slice, element) +} +``` + +### TypeScript/React Tests + +**Component Testing**: + +```typescript +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { DeploymentList } from './DeploymentList'; + +describe('DeploymentList', () => { + it('renders deployment list', async () => { + // Arrange + const deployments = [ + { id: '1', name: 'app-1', status: 'SUCCESS' }, + { id: '2', name: 'app-2', status: 'RUNNING' }, + ]; + + // Act + render(); + + // Assert + expect(screen.getByText('app-1')).toBeInTheDocument(); + expect(screen.getByText('app-2')).toBeInTheDocument(); + }); + + it('filters deployments by status', async () => { + // Arrange + const deployments = [...]; + render(); + + // Act + fireEvent.click(screen.getByRole('button', { name: /filter/i })); + fireEvent.click(screen.getByText('SUCCESS')); + + // Assert + await waitFor(() => { + expect(screen.queryByText('app-2')).not.toBeInTheDocument(); + }); + }); +}); +``` + +**Hook Testing**: + +```typescript +import { renderHook, act } from '@testing-library/react-hooks'; +import { useDeployments } from './useDeployments'; + +describe('useDeployments', () => { + it('fetches deployments', async () => { + const { result, waitForNextUpdate } = renderHook(() => + useDeployments('project-1') + ); + + expect(result.current.loading).toBe(true); + + await waitForNextUpdate(); + + expect(result.current.loading).toBe(false); + expect(result.current.deployments).toHaveLength(2); + }); +}); +``` + +## Writing Integration Tests + +### Database Integration Tests + +```go +// +build integration + +package datastore_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDeploymentStore(t *testing.T) { + // Setup test database + db := setupTestDB(t) + defer db.Close() + + store := NewDeploymentStore(db) + + t.Run("create and get deployment", func(t *testing.T) { + ctx := context.Background() + + // Create deployment + deployment := &Deployment{ + ID: "deploy-1", + Name: "test-deployment", + } + err := store.Create(ctx, deployment) + require.NoError(t, err) + + // Retrieve deployment + retrieved, err := store.Get(ctx, deployment.ID) + require.NoError(t, err) + require.Equal(t, deployment.Name, retrieved.Name) + }) +} + +func setupTestDB(t *testing.T) *sql.DB { + // Setup test database (e.g., SQLite in-memory) + db, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + + // Run migrations + err = runMigrations(db) + require.NoError(t, err) + + return db +} +``` + +### API Integration Tests + +```go +// +build integration + +func TestPipedAPI(t *testing.T) { + // Start test server + server := startTestServer(t) + defer server.Close() + + client := NewPipedClient(server.URL) + + t.Run("register piped", func(t *testing.T) { + req := &RegisterPipedRequest{ + Name: "test-piped", + } + + resp, err := client.RegisterPiped(context.Background(), req) + require.NoError(t, err) + require.NotEmpty(t, resp.PipedID) + }) +} +``` + +## Testing Best Practices + +### 1. Test Naming + +```go +// Good: Descriptive names +func TestDeploymentController_HandleSuccessfulDeployment(t *testing.T) {} +func TestKubernetesProvider_ApplyManifestWithInvalidYAML(t *testing.T) {} + +// Bad: Vague names +func TestDeploy(t *testing.T) {} +func TestHandle(t *testing.T) {} +``` + +### 2. Arrange-Act-Assert Pattern + +```go +func TestExample(t *testing.T) { + // Arrange - Set up test data and dependencies + deployer := NewDeployer() + app := &Application{ID: "app-1"} + + // Act - Execute the code under test + err := deployer.Deploy(context.Background(), app) + + // Assert - Verify the results + require.NoError(t, err) +} +``` + +### 3. Test Independence + +```go +// Good: Each test is independent +func TestCreate(t *testing.T) { + db := setupTestDB(t) + defer cleanupDB(db) + // test logic +} + +func TestUpdate(t *testing.T) { + db := setupTestDB(t) + defer cleanupDB(db) + // test logic +} + +// Bad: Tests depend on each other +var globalDB *sql.DB + +func TestCreate(t *testing.T) { + globalDB = setupTestDB(t) // Affects other tests + // test logic +} +``` + +### 4. Use Test Fixtures + +```go +// fixtures.go +func NewTestDeployment(opts ...func(*Deployment)) *Deployment { + d := &Deployment{ + ID: "deploy-1", + Name: "test-deployment", + Status: DeploymentStatus_RUNNING, + } + for _, opt := range opts { + opt(d) + } + return d +} + +// Usage in tests +func TestDeployment(t *testing.T) { + deployment := NewTestDeployment( + func(d *Deployment) { + d.Status = DeploymentStatus_SUCCESS + }, + ) + // test logic +} +``` + +### 5. Test Error Conditions + +```go +func TestDeployWithError(t *testing.T) { + tests := []struct { + name string + setup func() error + expectError string + }{ + { + name: "database error", + setup: func() error { return errors.New("db error") }, + expectError: "failed to save deployment", + }, + { + name: "network timeout", + setup: func() error { return context.DeadlineExceeded }, + expectError: "deployment timeout", + }, + } + // test logic +} +``` + +## Mocking + +### Using mockgen + +```go +//go:generate mockgen -source=deployer.go -destination=mock_deployer.go -package=mocks + +// Interface to mock +type Deployer interface { + Deploy(ctx context.Context, app *Application) error +} + +// Using the mock in tests +import "github.com/golang/mock/gomock" + +func TestWithMock(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDeployer := mocks.NewMockDeployer(ctrl) + mockDeployer.EXPECT(). + Deploy(gomock.Any(), gomock.Any()). + Return(nil). + Times(1) + + // Use mock in test +} +``` + +### Manual Mocks + +```go +// Simple mock implementation +type MockDatastore struct { + GetFunc func(ctx context.Context, id string) (*Deployment, error) +} + +func (m *MockDatastore) Get(ctx context.Context, id string) (*Deployment, error) { + if m.GetFunc != nil { + return m.GetFunc(ctx, id) + } + return nil, errors.New("not implemented") +} + +// Usage +func TestWithManualMock(t *testing.T) { + mock := &MockDatastore{ + GetFunc: func(ctx context.Context, id string) (*Deployment, error) { + return &Deployment{ID: id}, nil + }, + } + // test logic +} +``` + +### TypeScript Mocking with Jest + +```typescript +// Mock API client +jest.mock('../api/deployment', () => ({ + getDeployment: jest.fn(), + listDeployments: jest.fn(), +})); + +import { getDeployment } from '../api/deployment'; + +describe('DeploymentComponent', () => { + it('fetches deployment data', async () => { + // Setup mock + (getDeployment as jest.Mock).mockResolvedValue({ + id: '1', + name: 'test-app', + }); + + // Test component + render(); + + // Verify mock was called + expect(getDeployment).toHaveBeenCalledWith('1'); + }); +}); +``` + +## Test Coverage + +### Measuring Coverage + +```bash +# Go coverage +make test/go COVERAGE=true + +# View coverage report +go tool cover -html=coverage.out + +# Web coverage +yarn --cwd web test --coverage + +# View web coverage +open web/coverage/lcov-report/index.html +``` + +### Coverage Goals + +- **Critical code**: 90%+ coverage +- **Business logic**: 80%+ coverage +- **UI components**: 70%+ coverage +- **Overall project**: 75%+ coverage + +### What to Cover + +**High Priority**: +- Business logic +- Error handling +- Security-critical code +- Data transformations + +**Lower Priority**: +- Simple getters/setters +- Type definitions +- Generated code + +## CI/CD Testing + +### GitHub Actions Workflow + +Tests run automatically on: +- Pull requests +- Pushes to master +- Release branches + +### Test Matrix + +```yaml +strategy: + matrix: + go-version: [1.21, 1.22] + os: [ubuntu-latest, macos-latest] +``` + +### Debugging CI Failures + +```bash +# Run tests with same flags as CI +go test -race -coverprofile=coverage.out ./... + +# Check for flaky tests +go test -count=100 ./pkg/... + +# Run with verbose output +go test -v ./... +``` + +## Common Testing Patterns + +### Table-Driven Tests + +```go +tests := []struct { + name string + input string + want string + wantErr bool +}{ + {"valid input", "test", "TEST", false}, + {"empty input", "", "", true}, +} + +for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Transform(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("got %v, want %v", got, tt.want) + } + }) +} +``` + +### Subtests + +```go +func TestDeployment(t *testing.T) { + t.Run("kubernetes", func(t *testing.T) { + // k8s specific tests + }) + + t.Run("terraform", func(t *testing.T) { + // terraform specific tests + }) +} +``` + +### Test Helpers + +```go +// Helper function +func assertDeploymentStatus(t *testing.T, d *Deployment, expected DeploymentStatus) { + t.Helper() // Mark as helper to get correct line numbers + if d.Status != expected { + t.Errorf("got status %v, want %v", d.Status, expected) + } +} + +// Usage +func TestDeployment(t *testing.T) { + deployment := createTestDeployment() + assertDeploymentStatus(t, deployment, DeploymentStatus_SUCCESS) +} +``` + +## Resources + +- [Go Testing Package](https://pkg.go.dev/testing) +- [Testify Documentation](https://github.com/stretchr/testify) +- [React Testing Library](https://testing-library.com/docs/react-testing-library/intro/) +- [Jest Documentation](https://jestjs.io/docs/getting-started) + +## Questions? + +- Ask in [#pipecd Slack](https://cloud-native.slack.com/archives/C01B27F9T0X) +- Open a [GitHub Discussion](https://github.com/pipe-cd/pipecd/discussions) diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md new file mode 100644 index 0000000000..b0e4e9bf88 --- /dev/null +++ b/TROUBLESHOOTING.md @@ -0,0 +1,649 @@ +# PipeCD Troubleshooting Guide + +This guide helps you diagnose and resolve common issues with PipeCD. + +## Table of Contents + +- [General Troubleshooting](#general-troubleshooting) +- [Control Plane Issues](#control-plane-issues) +- [Piped Agent Issues](#piped-agent-issues) +- [Deployment Issues](#deployment-issues) +- [Web UI Issues](#web-ui-issues) +- [Performance Issues](#performance-issues) +- [Network & Connectivity](#network--connectivity) +- [Database Issues](#database-issues) +- [Getting Help](#getting-help) + +## General Troubleshooting + +### Check Component Versions + +Ensure all components are running compatible versions: + +```bash +# Check PipeCD control plane version +kubectl exec -n pipecd deployment/pipecd -- /pipecd version + +# Check Piped version +kubectl exec -n pipecd deployment/piped -- /piped version + +# Check pipectl version +pipectl version +``` + +### Enable Debug Logging + +Enable debug logging for more detailed information: + +**Control Plane:** +```yaml +# In pipecd configuration +spec: + logLevel: debug +``` + +**Piped:** +```yaml +# In piped configuration +spec: + logLevel: debug +``` + +### View Logs + +```bash +# Control Plane logs +kubectl logs -n pipecd deployment/pipecd -f + +# Piped logs +kubectl logs -n pipecd deployment/piped -f + +# View logs for specific container +kubectl logs -n pipecd deployment/pipecd -c pipecd -f + +# Previous container logs (if crashed) +kubectl logs -n pipecd deployment/piped --previous +``` + +## Control Plane Issues + +### Issue: Control Plane Fails to Start + +**Symptoms**: Pod in CrashLoopBackOff state + +**Common Causes & Solutions**: + +1. **Database Connection Failure** + ```bash + # Check logs for database errors + kubectl logs -n pipecd deployment/pipecd | grep -i "database\|connection" + + # Verify database credentials + kubectl get secret -n pipecd pipecd-secrets -o yaml + + # Test database connectivity + kubectl run -n pipecd mysql-client --rm -it --image=mysql:8.0 -- \ + mysql -h -u -p + ``` + +2. **Missing Configuration** + ```bash + # Verify ConfigMap exists + kubectl get configmap -n pipecd pipecd-config + + # Check configuration content + kubectl get configmap -n pipecd pipecd-config -o yaml + ``` + +3. **Insufficient Resources** + ```bash + # Check resource limits + kubectl describe pod -n pipecd -l app=pipecd + + # Adjust resources in deployment + kubectl edit deployment -n pipecd pipecd + ``` + +### Issue: Unable to Login to Web UI + +**Symptoms**: Authentication fails or redirects loop + +**Solutions**: + +1. **Check Static Admin Configuration** + ```bash + # Verify admin account exists in config + kubectl get configmap -n pipecd pipecd-config -o yaml | grep -A 5 "staticAdmin" + ``` + +2. **SSO Configuration Issues** + ```bash + # Check SSO settings + kubectl get configmap -n pipecd pipecd-config -o yaml | grep -A 10 "sso" + + # Verify callback URLs match + # GitHub: Settings -> Developer settings -> OAuth Apps + # Google: Cloud Console -> APIs & Services -> Credentials + ``` + +3. **Cookie/Session Issues** + - Clear browser cache and cookies + - Try incognito/private browsing mode + - Check browser console for errors (F12) + +### Issue: API Requests Failing + +**Symptoms**: gRPC errors, timeout errors + +**Solutions**: + +```bash +# Check service endpoints +kubectl get svc -n pipecd + +# Test gRPC connectivity +grpcurl -plaintext localhost:9080 list + +# Verify TLS certificates (if using) +openssl s_client -connect :443 -servername + +# Check for network policies +kubectl get networkpolicies -n pipecd +``` + +## Piped Agent Issues + +### Issue: Piped Unable to Connect to Control Plane + +**Symptoms**: "connection refused", "context deadline exceeded" + +**Solutions**: + +1. **Verify API Address** + ```yaml + # In piped config, check apiAddress + spec: + apiAddress: :443 + ``` + +2. **Check Network Connectivity** + ```bash + # From piped pod + kubectl exec -n pipecd deployment/piped -- nc -zv 443 + + # Check DNS resolution + kubectl exec -n pipecd deployment/piped -- nslookup + ``` + +3. **Verify Piped Credentials** + ```bash + # Check piped ID and key + kubectl get secret -n pipecd piped-secret -o yaml + + # Regenerate piped key if needed (via Web UI) + ``` + +4. **Firewall/Network Policies** + - Ensure firewall allows outbound HTTPS (443) + - Check cloud provider security groups + - Verify Kubernetes NetworkPolicies + +### Issue: Piped Not Syncing Git Repository + +**Symptoms**: No deployments triggered despite Git changes + +**Solutions**: + +1. **Verify Git Configuration** + ```yaml + # Check piped config + spec: + repositories: + - repoId: example + remote: git@github.com:org/repo.git + branch: main + ``` + +2. **Check SSH Keys** + ```bash + # For SSH repositories + kubectl exec -n pipecd deployment/piped -- ssh -T git@github.com + + # Verify SSH key secret + kubectl get secret -n pipecd piped-git-ssh-key + ``` + +3. **Check Git Sync Interval** + ```yaml + spec: + syncInterval: 1m # Increase if too frequent + ``` + +4. **Review Git Sync Logs** + ```bash + kubectl logs -n pipecd deployment/piped | grep -i "git\|sync" + ``` + +### Issue: Piped Using Old Version + +**Symptoms**: Features not working, incompatibility errors + +**Solutions**: + +```bash +# Check current version +kubectl exec -n pipecd deployment/piped -- /piped version + +# Update image in deployment +kubectl set image deployment/piped -n pipecd piped=ghcr.io/pipe-cd/piped:v0.x.x + +# Or use launcher for auto-updates +# Configure remote upgrade in piped config +``` + +## Deployment Issues + +### Issue: Deployment Stuck in Pending + +**Symptoms**: Deployment doesn't progress + +**Solutions**: + +1. **Check Piped Status** + ```bash + # Ensure piped is running + kubectl get pods -n pipecd -l app=piped + + # Check piped logs + kubectl logs -n pipecd deployment/piped -f + ``` + +2. **Verify Pipeline Configuration** + ```yaml + # Check .pipe.yaml in repository + apiVersion: pipecd.dev/v1beta1 + kind: KubernetesApp + spec: + pipeline: + stages: + - name: K8S_SYNC + ``` + +3. **Check Resource Quota** + ```bash + # Verify namespace quotas + kubectl describe resourcequota -n + ``` + +### Issue: Deployment Fails with Analysis Error + +**Symptoms**: Analysis stage fails, deployment rolls back + +**Solutions**: + +1. **Check Analysis Configuration** + ```yaml + # Verify metrics provider is configured + spec: + analysisProviders: + - name: prometheus + type: PROMETHEUS + config: + address: http://prometheus:9090 + ``` + +2. **Verify Metrics Query** + ```bash + # Test Prometheus query manually + curl "http://prometheus:9090/api/v1/query?query=" + ``` + +3. **Review Analysis Logs** + ```bash + kubectl logs -n pipecd deployment/piped | grep -i "analysis" + ``` + +### Issue: Kubernetes Manifest Apply Fails + +**Symptoms**: "error applying manifest", "invalid resource" + +**Solutions**: + +1. **Validate Kubernetes Manifests** + ```bash + # Dry-run apply + kubectl apply --dry-run=server -f manifest.yaml + + # Validate YAML + kubectl apply --validate=true -f manifest.yaml + ``` + +2. **Check RBAC Permissions** + ```bash + # Verify piped service account permissions + kubectl auth can-i create deployments --as=system:serviceaccount:pipecd:piped -n + ``` + +3. **Review Piped Logs for Details** + ```bash + kubectl logs -n pipecd deployment/piped | grep -B5 -A5 "error" + ``` + +### Issue: Terraform Deployment Fails + +**Symptoms**: Terraform plan/apply errors + +**Solutions**: + +1. **Check Terraform Version** + ```bash + # Verify terraform binary version + kubectl exec -n pipecd deployment/piped -- terraform version + ``` + +2. **Verify Cloud Credentials** + ```bash + # Check environment variables or mounted secrets + kubectl describe pod -n pipecd -l app=piped + ``` + +3. **Enable Terraform Debug Logs** + ```yaml + # In pipeline config + spec: + pipeline: + stages: + - name: TERRAFORM_APPLY + with: + args: + - -debug + ``` + +## Web UI Issues + +### Issue: Web UI Not Loading + +**Symptoms**: Blank page, 404 errors + +**Solutions**: + +1. **Check Service and Ingress** + ```bash + # Verify service + kubectl get svc -n pipecd pipecd + + # Check ingress + kubectl get ingress -n pipecd + kubectl describe ingress -n pipecd pipecd + ``` + +2. **Browser Console Errors** + - Open browser DevTools (F12) + - Check Console tab for JavaScript errors + - Check Network tab for failed requests + +3. **Clear Browser Cache** + ``` + Chrome: Ctrl+Shift+Delete + Firefox: Ctrl+Shift+Delete + Safari: Cmd+Option+E + ``` + +### Issue: Deployment List Not Showing + +**Symptoms**: Empty deployment list despite having deployments + +**Solutions**: + +1. **Check Project Filter** + - Verify correct project selected in UI + - Check project ID in URL parameters + +2. **Verify Database Connection** + ```bash + # Check control plane logs + kubectl logs -n pipecd deployment/pipecd | grep -i "database" + ``` + +3. **Check API Communication** + ``` + # Browser DevTools -> Network tab + # Look for failed API calls to /grpc.* + ``` + +## Performance Issues + +### Issue: Slow Deployment Processing + +**Symptoms**: Deployments take longer than expected + +**Solutions**: + +1. **Check Resource Usage** + ```bash + # Control Plane + kubectl top pod -n pipecd -l app=pipecd + + # Piped + kubectl top pod -n pipecd -l app=piped + ``` + +2. **Optimize Database Queries** + ```bash + # Check for slow queries + # MySQL: Enable slow query log + # PostgreSQL: pg_stat_statements + ``` + +3. **Reduce Git Sync Frequency** + ```yaml + spec: + syncInterval: 5m # Increase from 1m if needed + ``` + +### Issue: High Memory Usage + +**Symptoms**: OOMKilled errors, frequent restarts + +**Solutions**: + +```bash +# Increase memory limits +kubectl patch deployment -n pipecd pipecd -p '{"spec":{"template":{"spec":{"containers":[{"name":"pipecd","resources":{"limits":{"memory":"4Gi"}}}]}}}}' + +# Enable garbage collection tuning +# Set GOGC environment variable +kubectl set env deployment/pipecd -n pipecd GOGC=80 +``` + +## Network & Connectivity + +### Issue: Cannot Reach External Services + +**Symptoms**: Timeout connecting to webhooks, metrics providers + +**Solutions**: + +1. **Check Network Policies** + ```bash + kubectl get networkpolicies -n pipecd + kubectl describe networkpolicy -n pipecd + ``` + +2. **Verify DNS Resolution** + ```bash + kubectl exec -n pipecd deployment/piped -- nslookup google.com + ``` + +3. **Test Connectivity** + ```bash + kubectl exec -n pipecd deployment/piped -- curl -v https://api.github.com + ``` + +### Issue: Webhook Notifications Not Working + +**Symptoms**: No Slack/Discord messages sent + +**Solutions**: + +1. **Verify Webhook URL** + ```yaml + # Check notification config + spec: + notifications: + - name: slack + type: SLACK + config: + webhookUrl: https://hooks.slack.com/services/... + ``` + +2. **Test Webhook Manually** + ```bash + curl -X POST -H 'Content-type: application/json' \ + --data '{"text":"Test message"}' \ + + ``` + +3. **Check Firewall Rules** + - Ensure outbound HTTPS allowed + - Verify no proxy blocking requests + +## Database Issues + +### Issue: Database Migration Fails + +**Symptoms**: Control plane won't start, migration errors + +**Solutions**: + +```bash +# Check migration status +kubectl logs -n pipecd deployment/pipecd | grep -i "migration" + +# Manually run migrations (if needed) +kubectl exec -n pipecd deployment/pipecd -- /pipecd migrate + +# Rollback migration (last resort) +# Backup database first! +``` + +### Issue: Database Connection Pool Exhausted + +**Symptoms**: "too many connections", slow queries + +**Solutions**: + +```yaml +# Adjust connection pool settings +spec: + datastore: + maxOpenConns: 50 + maxIdleConns: 10 + connMaxLifetime: 300s +``` + +## Getting Help + +If you've tried the above solutions and still have issues: + +### 1. Gather Diagnostic Information + +```bash +# Collect logs +kubectl logs -n pipecd deployment/pipecd > pipecd.log +kubectl logs -n pipecd deployment/piped > piped.log + +# Get pod status +kubectl describe pod -n pipecd > pod-status.txt + +# Export configurations (remove sensitive data!) +kubectl get configmap -n pipecd -o yaml > configs.yaml +``` + +### 2. Search Existing Issues + +- Check [GitHub Issues](https://github.com/pipe-cd/pipecd/issues) +- Search [Discussions](https://github.com/pipe-cd/pipecd/discussions) + +### 3. Ask the Community + +- Join [CNCF Slack #pipecd](https://cloud-native.slack.com/archives/C01B27F9T0X) +- Attend [community meetings](https://bit.ly/pipecd-mtg-notes) +- Post in [GitHub Discussions](https://github.com/pipe-cd/pipecd/discussions) + +### 4. Report a Bug + +If you believe you've found a bug: + +1. Create a [bug report](https://github.com/pipe-cd/pipecd/issues/new?template=bug-report.md) +2. Include: + - PipeCD version + - Component affected (control plane, piped, web UI) + - Steps to reproduce + - Expected vs actual behavior + - Relevant logs (sanitized) + - Environment details (Kubernetes version, cloud provider, etc.) + +## Debugging Tools + +### Useful Commands + +```bash +# Port forward to control plane +kubectl port-forward -n pipecd svc/pipecd 9080:9080 + +# Port forward to piped (admin port) +kubectl port-forward -n pipecd svc/piped 9085:9085 + +# Access pprof for profiling +curl http://localhost:9085/debug/pprof/heap > heap.prof +go tool pprof heap.prof + +# Check gRPC health +grpcurl -plaintext localhost:9080 grpc.health.v1.Health/Check + +# Inspect database +kubectl exec -it -n pipecd deployment/pipecd -- sh +# Then use mysql or psql client +``` + +### Enable Profiling + +```yaml +# In component config +spec: + profiling: + enabled: true + port: 6060 +``` + +## Common Error Messages + +### "context deadline exceeded" +- Network connectivity issue +- Timeout too short +- Service not responding + +### "permission denied" +- RBAC permissions insufficient +- Service account misconfigured +- File permissions issue + +### "resource not found" +- Wrong namespace +- Resource deleted +- Wrong cluster context + +### "invalid configuration" +- YAML syntax error +- Missing required fields +- Incompatible values + +### "authentication failed" +- Invalid credentials +- Expired token +- Wrong API key + +--- + +**Note**: This guide is continuously updated. For the latest troubleshooting tips, check the [official documentation](https://pipecd.dev/docs). From 9f51cff0a1ef09f08ab2c10b5e88eb6ed7399089 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 3 Jan 2026 11:17:32 +0000 Subject: [PATCH 3/8] Add additional workflows, API documentation, and enhanced security policy - Add release notes generation workflow with release-drafter - Add performance benchmarking workflow - Add GitHub Actions update workflow - Add changelog automation workflow with git-cliff - Create comprehensive API_DOCUMENTATION.md - Enhance SECURITY.md with detailed reporting process and best practices - Add cliff.toml for automated changelog generation Co-authored-by: Ayushmore1214 <194600182+Ayushmore1214@users.noreply.github.com> --- .github/release-drafter.yml | 81 ++++ .github/workflows/benchmark.yaml | 77 ++++ .github/workflows/changelog.yaml | 43 +++ .github/workflows/release-notes.yaml | 65 ++++ .github/workflows/update-actions.yaml | 52 +++ API_DOCUMENTATION.md | 515 ++++++++++++++++++++++++++ SECURITY.md | 139 ++++++- cliff.toml | 56 +++ 8 files changed, 1026 insertions(+), 2 deletions(-) create mode 100644 .github/release-drafter.yml create mode 100644 .github/workflows/benchmark.yaml create mode 100644 .github/workflows/changelog.yaml create mode 100644 .github/workflows/release-notes.yaml create mode 100644 .github/workflows/update-actions.yaml create mode 100644 API_DOCUMENTATION.md create mode 100644 cliff.toml diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 0000000000..240752f771 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,81 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' + +categories: + - title: '🚀 Features' + labels: + - 'kind/feature' + - 'enhancement' + - 'kind/enhancement' + - title: '🐛 Bug Fixes' + labels: + - 'kind/bug' + - 'bug' + - 'fix' + - title: '📚 Documentation' + labels: + - 'documentation' + - 'docs' + - title: '🔒 Security' + labels: + - 'security' + - title: '⚡ Performance' + labels: + - 'performance' + - title: '🧹 Maintenance' + labels: + - 'chore' + - 'dependencies' + - 'refactor' + +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' +change-title-escapes: '\<*_&' + +version-resolver: + major: + labels: + - 'major' + - 'breaking' + minor: + labels: + - 'minor' + - 'kind/feature' + patch: + labels: + - 'patch' + - 'kind/bug' + default: patch + +exclude-labels: + - 'skip-changelog' + - 'not-auto-close' + +autolabeler: + - label: 'documentation' + files: + - '*.md' + - 'docs/**/*' + - label: 'kind/bug' + branch: + - '/fix\/.+/' + title: + - '/fix/i' + - label: 'kind/feature' + branch: + - '/feat(ure)?\/.+/' + title: + - '/feat(ure)?/i' + - label: 'dependencies' + files: + - 'go.mod' + - 'go.sum' + - 'package.json' + - 'package-lock.json' + - 'yarn.lock' + +template: | + ## What's Changed + + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 0000000000..c8fc1df164 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,77 @@ +name: benchmark + +on: + push: + branches: + - master + pull_request: + branches: + - master + paths: + - '**.go' + - 'go.mod' + - 'go.sum' + - '.github/workflows/benchmark.yaml' + +# Only run the latest benchmark for each PR +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + GO_VERSION: 1.25.0 + +jobs: + benchmark: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Run benchmarks + run: | + # Run benchmarks and save results + go test -bench=. -benchmem -run=^$ ./... | tee benchmark_results.txt + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: benchmark_results.txt + retention-days: 30 + + - name: Compare benchmarks (PR only) + if: github.event_name == 'pull_request' + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: benchmark_results.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + alert-threshold: '150%' + fail-on-alert: false + auto-push: false + + - name: Comment benchmark results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const results = fs.readFileSync('benchmark_results.txt', 'utf8'); + + // Extract summary (first 1000 chars) + const summary = results.substring(0, 1000); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `## 📊 Benchmark Results\n\n\`\`\`\n${summary}\n...\n\`\`\`\n\nFull results are available in the workflow artifacts.` + }); diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml new file mode 100644 index 0000000000..490bbf7c4f --- /dev/null +++ b/.github/workflows/changelog.yaml @@ -0,0 +1,43 @@ +name: changelog + +on: + push: + branches: + - master + workflow_dispatch: + +permissions: + contents: write + pull-requests: read + +jobs: + update-changelog: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Generate changelog + id: changelog + uses: orhun/git-cliff-action@v4 + with: + config: cliff.toml + args: --verbose + env: + OUTPUT: CHANGELOG.md + + - name: Commit changelog + run: | + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + + # Only commit if there are changes + if [[ $(git diff --stat) != '' ]]; then + git add CHANGELOG.md + git commit -m "docs: update CHANGELOG.md" + git push + else + echo "No changes to CHANGELOG.md" + fi diff --git a/.github/workflows/release-notes.yaml b/.github/workflows/release-notes.yaml new file mode 100644 index 0000000000..95571038d8 --- /dev/null +++ b/.github/workflows/release-notes.yaml @@ -0,0 +1,65 @@ +name: release-notes + +on: + release: + types: [published] + +permissions: + contents: write + +jobs: + generate-release-notes: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Generate Release Notes + id: generate_notes + uses: release-drafter/release-drafter@v6 + with: + config-name: release-drafter.yml + publish: true + tag: ${{ github.event.release.tag_name }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Update Release + uses: actions/github-script@v7 + with: + script: | + const { data: release } = await github.rest.repos.getRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: context.payload.release.id + }); + + const body = `${{ steps.generate_notes.outputs.body }} + + ## Installation + + ### Helm Chart + \`\`\`bash + helm repo add pipecd https://charts.pipecd.dev + helm repo update + helm install pipecd pipecd/pipecd --version ${{ github.event.release.tag_name }} + \`\`\` + + ### Binary Downloads + - [Linux AMD64](https://github.com/pipe-cd/pipecd/releases/download/${{ github.event.release.tag_name }}/pipecd_linux_amd64.tar.gz) + - [Linux ARM64](https://github.com/pipe-cd/pipecd/releases/download/${{ github.event.release.tag_name }}/pipecd_linux_arm64.tar.gz) + - [macOS AMD64](https://github.com/pipe-cd/pipecd/releases/download/${{ github.event.release.tag_name }}/pipecd_darwin_amd64.tar.gz) + - [macOS ARM64](https://github.com/pipe-cd/pipecd/releases/download/${{ github.event.release.tag_name }}/pipecd_darwin_arm64.tar.gz) + + ## What's Changed + ${release.body || ''} + `; + + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: context.payload.release.id, + body: body + }); diff --git a/.github/workflows/update-actions.yaml b/.github/workflows/update-actions.yaml new file mode 100644 index 0000000000..d7e9812364 --- /dev/null +++ b/.github/workflows/update-actions.yaml @@ -0,0 +1,52 @@ +name: update-actions + +on: + schedule: + # Run monthly on the first day at 00:00 UTC + - cron: '0 0 1 * *' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update-actions: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Update GitHub Actions + id: update + run: | + # This is a placeholder for automated action updates + # In production, you would use a tool like Renovate or Dependabot + echo "Actions are managed by Dependabot" + echo "updated=false" >> $GITHUB_OUTPUT + + - name: Create Pull Request + if: steps.update.outputs.updated == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: 'chore(deps): update GitHub Actions' + title: 'chore(deps): Update GitHub Actions to latest versions' + body: | + ## Description + + This PR updates GitHub Actions to their latest versions. + + ## Changes + - Updated actions to latest stable versions + - Maintained compatibility with existing workflows + + ## Testing + - [ ] All workflows pass with updated actions + - [ ] No breaking changes detected + branch: update-actions + delete-branch: true + labels: | + dependencies + github-actions + automated diff --git a/API_DOCUMENTATION.md b/API_DOCUMENTATION.md new file mode 100644 index 0000000000..985d4918c2 --- /dev/null +++ b/API_DOCUMENTATION.md @@ -0,0 +1,515 @@ +# API Documentation Guide + +This guide explains how to use and contribute to PipeCD's API documentation. + +## Table of Contents + +- [API Overview](#api-overview) +- [gRPC API](#grpc-api) +- [REST API](#rest-api) +- [API Authentication](#api-authentication) +- [Using the API](#using-the-api) +- [API Client Libraries](#api-client-libraries) +- [Contributing to API Docs](#contributing-to-api-docs) + +## API Overview + +PipeCD exposes two types of APIs: + +1. **gRPC API**: Primary API for piped agents and programmatic access +2. **REST API**: HTTP endpoints for web UI and simple integrations + +### API Versioning + +PipeCD follows semantic versioning for its APIs: +- **Major version**: Breaking changes +- **Minor version**: New features, backward compatible +- **Patch version**: Bug fixes + +Current API version: **v1beta1** + +## gRPC API + +### Service Definitions + +PipeCD's gRPC services are defined in Protocol Buffer files: + +``` +pkg/app/api/ +├── api.proto # Common types +├── pipedservice.proto # Piped agent API +├── webservice.proto # Web UI API +└── service.proto # Control plane API +``` + +### Generating API Clients + +```bash +# Generate Go clients +make gen/api + +# Generate TypeScript clients for web +make gen/web-api +``` + +### Available Services + +#### 1. PipedService + +Used by piped agents to communicate with control plane. + +**Key methods**: +- `RegisterPiped`: Register a new piped agent +- `ReportDeployment`: Report deployment status +- `GetDeployment`: Retrieve deployment details +- `ListDeployments`: List deployments + +**Example**: + +```go +import ( + "context" + "google.golang.org/grpc" + pipedservice "github.com/pipe-cd/pipecd/pkg/app/api/pipedservice" +) + +func main() { + conn, err := grpc.Dial("control-plane:443", grpc.WithInsecure()) + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + client := pipedservice.NewPipedServiceClient(conn) + + resp, err := client.GetDeployment(context.Background(), &pipedservice.GetDeploymentRequest{ + DeploymentId: "deployment-123", + }) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Deployment: %v\n", resp.Deployment) +} +``` + +#### 2. WebService + +Used by the web UI and external integrations. + +**Key methods**: +- `GetProject`: Get project details +- `ListApplications`: List applications +- `GetApplication`: Get application details +- `SyncApplication`: Trigger application sync + +**Example**: + +```go +import ( + webservice "github.com/pipe-cd/pipecd/pkg/app/api/webservice" +) + +client := webservice.NewWebServiceClient(conn) + +resp, err := client.ListApplications(ctx, &webservice.ListApplicationsRequest{ + ProjectId: "my-project", +}) +``` + +#### 3. APIService + +General API service for administrative tasks. + +**Key methods**: +- `GetInsight`: Get deployment insights +- `GetDeploymentStatistics`: Get deployment statistics +- `ListDeploymentConfigTemplates`: List config templates + +### API Request/Response Examples + +#### Register Piped + +**Request**: +```protobuf +message RegisterPipedRequest { + string name = 1; + string desc = 2; + repeated string envIds = 3; +} +``` + +**Response**: +```protobuf +message RegisterPipedResponse { + string id = 1; + string key = 2; +} +``` + +#### Get Deployment + +**Request**: +```protobuf +message GetDeploymentRequest { + string deployment_id = 1; +} +``` + +**Response**: +```protobuf +message GetDeploymentResponse { + Deployment deployment = 1; +} +``` + +## REST API + +### Base URL + +``` +https://your-pipecd-instance.com/api/v1 +``` + +### Authentication + +All REST API requests require authentication via API key or session token: + +```bash +curl -H "Authorization: Bearer YOUR_API_KEY" \ + https://pipecd.example.com/api/v1/projects +``` + +### Endpoints + +#### Projects + +```bash +# List projects +GET /api/v1/projects + +# Get project +GET /api/v1/projects/{project_id} +``` + +#### Applications + +```bash +# List applications +GET /api/v1/projects/{project_id}/applications + +# Get application +GET /api/v1/applications/{app_id} + +# Sync application +POST /api/v1/applications/{app_id}/sync +``` + +#### Deployments + +```bash +# List deployments +GET /api/v1/deployments?project_id={project_id} + +# Get deployment +GET /api/v1/deployments/{deployment_id} + +# Cancel deployment +POST /api/v1/deployments/{deployment_id}/cancel +``` + +#### Pipeds + +```bash +# List pipeds +GET /api/v1/pipeds?project_id={project_id} + +# Get piped +GET /api/v1/pipeds/{piped_id} + +# Recreate piped key +POST /api/v1/pipeds/{piped_id}/recreate-key +``` + +## API Authentication + +### Using API Keys + +1. **Create API Key** (via Web UI): + - Navigate to Settings → API Keys + - Click "Add API Key" + - Set permissions and expiration + - Copy the generated key + +2. **Use API Key**: + ```bash + curl -H "Authorization: Bearer YOUR_API_KEY" \ + https://pipecd.example.com/api/v1/applications + ``` + +### Using Service Account + +For programmatic access: + +```yaml +# Service account configuration +apiVersion: pipecd.dev/v1beta1 +kind: ServiceAccount +metadata: + name: ci-pipeline +spec: + role: DEPLOYER + projectId: my-project +``` + +Generate token: +```bash +pipectl service-account create \ + --name ci-pipeline \ + --role DEPLOYER \ + --project my-project +``` + +## Using the API + +### With curl + +```bash +# Get deployment status +curl -X GET \ + -H "Authorization: Bearer YOUR_API_KEY" \ + https://pipecd.example.com/api/v1/deployments/deploy-123 + +# Trigger sync +curl -X POST \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"commandId": "cmd-123"}' \ + https://pipecd.example.com/api/v1/applications/app-123/sync +``` + +### With grpcurl + +```bash +# List gRPC services +grpcurl -plaintext localhost:9080 list + +# List methods for a service +grpcurl -plaintext localhost:9080 list pipecd.service.webservice.WebService + +# Call a method +grpcurl -plaintext \ + -d '{"deployment_id": "deploy-123"}' \ + localhost:9080 \ + pipecd.service.webservice.WebService/GetDeployment +``` + +### With Go + +```go +package main + +import ( + "context" + "log" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + webservice "github.com/pipe-cd/pipecd/pkg/app/api/webservice" +) + +func main() { + creds := credentials.NewTLS(&tls.Config{}) + conn, err := grpc.Dial( + "pipecd.example.com:443", + grpc.WithTransportCredentials(creds), + grpc.WithPerRPCCredentials(newAPIKeyAuth("YOUR_API_KEY")), + ) + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + client := webservice.NewWebServiceClient(conn) + + // List applications + resp, err := client.ListApplications(context.Background(), &webservice.ListApplicationsRequest{ + ProjectId: "my-project", + }) + if err != nil { + log.Fatal(err) + } + + for _, app := range resp.Applications { + log.Printf("App: %s (%s)\n", app.Name, app.Id) + } +} + +// API Key authentication +type apiKeyAuth struct { + apiKey string +} + +func newAPIKeyAuth(key string) credentials.PerRPCCredentials { + return &apiKeyAuth{apiKey: key} +} + +func (a *apiKeyAuth) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + return map[string]string{ + "authorization": "Bearer " + a.apiKey, + }, nil +} + +func (a *apiKeyAuth) RequireTransportSecurity() bool { + return true +} +``` + +### With Python + +```python +import grpc +from pipecd.api import webservice_pb2, webservice_pb2_grpc + +# Create channel +channel = grpc.secure_channel( + 'pipecd.example.com:443', + grpc.ssl_channel_credentials() +) + +# Create client +client = webservice_pb2_grpc.WebServiceStub(channel) + +# Add metadata (API key) +metadata = [('authorization', 'Bearer YOUR_API_KEY')] + +# Make request +request = webservice_pb2.ListApplicationsRequest( + project_id='my-project' +) + +response = client.ListApplications(request, metadata=metadata) + +for app in response.applications: + print(f"App: {app.name} ({app.id})") +``` + +## API Client Libraries + +### Official Clients + +- **Go**: Built-in (generated from proto files) +- **TypeScript**: Used by web UI + +### Community Clients + +> Note: Community clients are maintained by third parties and may not be up-to-date. + +- Python: [pipecd-python-client](https://github.com/example/pipecd-python-client) +- Ruby: [pipecd-ruby](https://github.com/example/pipecd-ruby) + +## Contributing to API Docs + +### Adding New API Methods + +1. **Define in proto file**: + ```protobuf + service WebService { + rpc GetNewResource(GetNewResourceRequest) returns (GetNewResourceResponse) {} + } + + message GetNewResourceRequest { + string resource_id = 1; + } + + message GetNewResourceResponse { + Resource resource = 1; + } + ``` + +2. **Regenerate code**: + ```bash + make gen/api + ``` + +3. **Implement server**: + ```go + func (s *webService) GetNewResource(ctx context.Context, req *webservice.GetNewResourceRequest) (*webservice.GetNewResourceResponse, error) { + // Implementation + } + ``` + +4. **Document in this file**: + - Add method description + - Provide request/response examples + - Include usage examples + +### API Documentation Standards + +- **Clear descriptions**: Explain what the API does +- **Request/response examples**: Show actual usage +- **Error codes**: Document possible errors +- **Deprecation notices**: Mark deprecated APIs +- **Version information**: Note when API was added/changed + +### Testing API Changes + +```bash +# Run API tests +make test/api + +# Test with grpcurl +grpcurl -plaintext localhost:9080 list + +# Test with curl (for REST endpoints) +curl -v http://localhost:8080/api/v1/health +``` + +## API Rate Limiting + +PipeCD implements rate limiting to prevent abuse: + +- **Default limits**: 100 requests/minute per API key +- **Burst limit**: 200 requests +- **Headers**: + - `X-RateLimit-Limit`: Max requests per window + - `X-RateLimit-Remaining`: Remaining requests + - `X-RateLimit-Reset`: Time when limit resets + +## Error Handling + +### gRPC Status Codes + +| Code | Description | Example | +|------|-------------|---------| +| OK | Success | Request completed successfully | +| INVALID_ARGUMENT | Invalid request | Missing required field | +| NOT_FOUND | Resource not found | Deployment ID doesn't exist | +| PERMISSION_DENIED | Insufficient permissions | API key lacks permission | +| UNAUTHENTICATED | Authentication failed | Invalid API key | +| INTERNAL | Server error | Database connection failed | + +### Error Response Example + +```json +{ + "error": { + "code": "NOT_FOUND", + "message": "Deployment not found: deploy-123", + "details": [] + } +} +``` + +## Resources + +- [Protocol Buffers](https://protobuf.dev/) +- [gRPC Documentation](https://grpc.io/docs/) +- [PipeCD API Reference](https://pipecd.dev/docs/api-reference/) + +## Support + +- [Slack Channel](https://cloud-native.slack.com/archives/C01B27F9T0X) +- [GitHub Discussions](https://github.com/pipe-cd/pipecd/discussions) +- [API Issues](https://github.com/pipe-cd/pipecd/issues?q=is%3Aissue+label%3Aapi) diff --git a/SECURITY.md b/SECURITY.md index 9689ff1f64..84d624d139 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,12 +1,147 @@ # Security Policy +## Supported Versions + +We release security updates for the following versions: + +| Version | Supported | +| ------- | ------------------ | +| 0.x.x | :white_check_mark: | +| < 0.x | :x: | + +> **Note**: Replace with actual version numbers. Generally, we support the latest minor version and provide security patches. + ## Reporting a Vulnerability -Drop an email to maintainers to report security issues. +We take the security of PipeCD seriously. If you believe you have found a security vulnerability, please report it to us as described below. + +### Where to Report + +**Please do NOT report security vulnerabilities through public GitHub issues.** -| Name | GitHub ID (not Twitter ID) | Email | +Instead, please report them via one of the following methods: + +1. **Email**: Contact the maintainers directly (see table below) +2. **Private vulnerability disclosure**: Use [GitHub Security Advisories](https://github.com/pipe-cd/pipecd/security/advisories/new) + +### Security Contacts + +| Name | GitHub ID | Email | |-------------------------|--------------------------------------------------|----------------------------------------------| | Tran Cong Khanh | [@khanhtc1202](https://github.com/khanhtc1202) | khanhtc1202@gmail.com | | Yoshiki Fujikane | [@ffjlabo](https://github.com/ffjlabo) | ffjlabo@gmail.com | | Shinnosuke Sawada-Dazai | [@Warashi](https://github.com/Warashi) | shin@warashi.dev | | Tetsuya Kikuchi | [@t-kikuc](https://github.com/t-kikuc) | tkikuchi07f@gmail.com | + +### What to Include + +Please include the following information in your report: + +- **Type of vulnerability** (e.g., XSS, SQL injection, authentication bypass) +- **Full path of affected source file(s)** +- **Location of the affected source code** (tag/branch/commit or direct URL) +- **Step-by-step instructions to reproduce the issue** +- **Proof-of-concept or exploit code** (if possible) +- **Impact of the issue**, including how an attacker might exploit it + +### Response Timeline + +- **Initial Response**: Within 48 hours of receiving your report +- **Status Update**: Within 7 days with an assessment of the report +- **Fix Timeline**: Depends on severity and complexity + - Critical: Within 7 days + - High: Within 30 days + - Medium: Within 90 days + - Low: Best effort + +### Disclosure Policy + +We follow a coordinated disclosure approach: + +1. **Private disclosure**: Report is received and acknowledged +2. **Investigation**: We investigate and develop a fix +3. **Fix development**: Fix is developed and tested +4. **Release**: Security patch is released +5. **Public disclosure**: Details are made public after users have had time to update (typically 7-14 days after release) + +### Security Updates + +Security updates will be announced via: + +- GitHub Security Advisories +- Release notes +- CNCF Slack #pipecd channel +- Twitter/X [@pipecd_dev](https://twitter.com/pipecd_dev) + +## Security Best Practices + +### For Users + +1. **Keep PipeCD Updated**: Always use the latest stable version +2. **Use TLS**: Enable TLS for all communications +3. **Rotate Secrets**: Regularly rotate API keys and credentials +4. **Principle of Least Privilege**: Grant minimal necessary permissions +5. **Network Security**: Use network policies to restrict access +6. **Audit Logs**: Regularly review audit logs for suspicious activity + +### For Contributors + +1. **Input Validation**: Always validate and sanitize user inputs +2. **Authentication**: Use strong authentication mechanisms +3. **Authorization**: Implement proper RBAC checks +4. **Secrets Management**: Never commit secrets to the repository +5. **Dependencies**: Keep dependencies up-to-date and scan for vulnerabilities +6. **Code Review**: All code must be reviewed before merging +7. **Security Testing**: Run security scans (CodeQL, dependency checks) + +## Security Features + +PipeCD includes the following security features: + +- **mTLS Support**: Mutual TLS for secure communication +- **RBAC**: Role-based access control +- **Audit Logging**: Comprehensive audit trail +- **Secret Encryption**: Secrets encrypted at rest +- **API Authentication**: API key and OAuth-based authentication +- **Network Isolation**: No credentials leave the deployment environment + +## Known Security Limitations + +- Secrets in Git repositories should be encrypted using external tools +- API keys have configurable but not enforced expiration +- Rate limiting is per-instance, not global + +## Security Tools + +We use the following tools to maintain security: + +- **CodeQL**: Automated code scanning +- **Dependabot**: Dependency vulnerability scanning +- **gosec**: Go security checker +- **trivy**: Container image scanning +- **npm audit**: JavaScript dependency scanning + +## Vulnerability Management + +### CVE Assignment + +For confirmed vulnerabilities, we will: +1. Request a CVE number +2. Create a GitHub Security Advisory +3. Publish details after the fix is released + +### Hall of Fame + +We recognize security researchers who responsibly disclose vulnerabilities: +- [Security Researchers Hall of Fame](https://pipecd.dev/security-hall-of-fame) (coming soon) + +## Questions? + +If you have questions about this policy, please: +- Join [#pipecd on CNCF Slack](https://cloud-native.slack.com/archives/C01B27F9T0X) +- Email the maintainers +- Open a [GitHub Discussion](https://github.com/pipe-cd/pipecd/discussions) + +--- + +**Note**: This security policy may be updated from time to time. Please check back regularly for updates. diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 0000000000..a7ae2b3aa6 --- /dev/null +++ b/cliff.toml @@ -0,0 +1,56 @@ +# Configuration for git-cliff +# See https://git-cliff.org/docs/configuration + +[changelog] +header = """ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +""" +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [Unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | upper_first }} + {% for commit in commits %} + - {% if commit.breaking %}[**BREAKING**] {% endif %}{{ commit.message | upper_first }} ([{{ commit.id | truncate(length=7, end="") }}]({{ commit.id }}))\ + {% endfor %} +{% endfor %}\n +""" +footer = """ + +""" +trim = true + +[git] +conventional_commits = true +filter_unconventional = true +split_commits = false +commit_preprocessors = [] +commit_parsers = [ + { message = "^feat", group = "Features" }, + { message = "^fix", group = "Bug Fixes" }, + { message = "^doc", group = "Documentation" }, + { message = "^perf", group = "Performance" }, + { message = "^refactor", group = "Refactoring" }, + { message = "^style", group = "Styling" }, + { message = "^test", group = "Testing" }, + { message = "^chore\\(release\\): prepare for", skip = true }, + { message = "^chore\\(deps\\)", group = "Dependencies" }, + { message = "^chore", group = "Miscellaneous Tasks" }, + { body = ".*security", group = "Security" }, +] +protect_breaking_commits = false +filter_commits = false +tag_pattern = "v[0-9]*" +skip_tags = "v0.1.0-beta.1" +ignore_tags = "" +topo_order = false +sort_commits = "oldest" From b7a764da4093dcafbaee31e7aad0a08be28e7d15 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 3 Jan 2026 11:20:51 +0000 Subject: [PATCH 4/8] Add code style guide, roadmap, discussion templates, and improve gitignore - Create comprehensive CODE_STYLE.md with language-specific guidelines - Add ROADMAP.md outlining future plans and features - Create GitHub discussion templates for feature proposals and show-and-tell - Enhance .gitignore with more comprehensive patterns - Update README.md to include all new documentation links Co-authored-by: Ayushmore1214 <194600182+Ayushmore1214@users.noreply.github.com> --- .../DISCUSSION_TEMPLATE/feature-proposal.yml | 50 ++ .github/DISCUSSION_TEMPLATE/show-and-tell.yml | 34 + .gitignore | 77 ++- CODE_STYLE.md | 615 ++++++++++++++++++ README.md | 3 + ROADMAP.md | 231 +++++++ 6 files changed, 1006 insertions(+), 4 deletions(-) create mode 100644 .github/DISCUSSION_TEMPLATE/feature-proposal.yml create mode 100644 .github/DISCUSSION_TEMPLATE/show-and-tell.yml create mode 100644 CODE_STYLE.md create mode 100644 ROADMAP.md diff --git a/.github/DISCUSSION_TEMPLATE/feature-proposal.yml b/.github/DISCUSSION_TEMPLATE/feature-proposal.yml new file mode 100644 index 0000000000..ef14154ecc --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/feature-proposal.yml @@ -0,0 +1,50 @@ +--- +title: "Feature Proposal: " +labels: ["enhancement", "needs-triage"] +--- + +## Problem Statement + +**What problem are you trying to solve?** + +(Describe the current situation and the pain point) + +## Proposed Solution + +**How do you propose to solve this problem?** + +(Describe your proposed solution in detail) + +## Alternatives Considered + +**What other solutions did you consider?** + +- Alternative 1: ... +- Alternative 2: ... + +## Use Cases + +**Who will benefit from this feature?** + +1. Use case 1: ... +2. Use case 2: ... + +## Implementation Considerations + +**What should be considered when implementing this?** + +- Technical complexity +- Breaking changes +- Performance impact +- Security implications +- Documentation needs + +## Additional Context + +(Add any other context, mockups, or examples) + +## Willing to Contribute + +- [ ] I am willing to submit a PR to implement this feature +- [ ] I can help with design/documentation +- [ ] I can help with testing diff --git a/.github/DISCUSSION_TEMPLATE/show-and-tell.yml b/.github/DISCUSSION_TEMPLATE/show-and-tell.yml new file mode 100644 index 0000000000..3ee4dd5f9f --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/show-and-tell.yml @@ -0,0 +1,34 @@ +--- +title: "Show and Tell: " +labels: ["show-and-tell"] +--- + +## What did you build? + +(Brief description of what you created) + +## Screenshots/Demo + +(Add screenshots, GIFs, or links to demos) + +## How does it work? + +(Explain how you built it and how it works) + +## Technologies Used + +- Technology 1 +- Technology 2 +- ... + +## What you learned + +(Share your experience and lessons learned) + +## Resources + +(Links to code, documentation, blog posts, etc.) + +## Questions for the community + +(Any questions you have or feedback you're looking for) diff --git a/.gitignore b/.gitignore index b7b46890b6..0eacf1596b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,16 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out +coverage.out +coverage.txt +*.coverprofile +# OS specific files .DS_Store +Thumbs.db +*.swp +*.swo +*~ # Bazel /bazel-bin @@ -30,29 +38,49 @@ go.work go.work.sum -# nodejs +# Node.js node_modules +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.npm +.eslintcache -# web +# Web web/dist web/.env web/.cache web/coverage +web/build pkg/app/web/.env # IDE config files .ijwb .idea - +.vscode/settings.json +.vscode/launch.json +*.sublime-project +*.sublime-workspace +.project +.classpath +.settings/ + +# Development .dev +*.local # Terraform workspace .terraform .terraform-credentials +*.tfstate +*.tfstate.* +*.tfvars +# Kubernetes .rendered-manifests +*.kubeconfig -# manfiests +# Helm manifests manifests/pipecd/charts manifests/piped/charts manifests/site/charts @@ -66,3 +94,44 @@ gomock_reflect_*/ # hack hack/oidc/realm.local.json + +# Logs +*.log +logs/ + +# Temporary files +tmp/ +temp/ +*.tmp + +# Documentation builds +docs/public/ +docs/resources/ +docs/.hugo_build.lock + +# Database +*.db +*.sqlite +*.sqlite3 + +# Environment variables +.env +.env.local +.env.*.local + +# Pre-commit +.pre-commit-config-local.yaml + +# Profiling +*.prof +*.pprof +cpu.prof +mem.prof +heap.prof + +# Archives +*.tar.gz +*.zip +*.tar +*.rar + diff --git a/CODE_STYLE.md b/CODE_STYLE.md new file mode 100644 index 0000000000..c84d409a85 --- /dev/null +++ b/CODE_STYLE.md @@ -0,0 +1,615 @@ +# Code Style Guide for PipeCD + +This document outlines the coding standards and style conventions for contributing to PipeCD. + +## Table of Contents + +- [General Principles](#general-principles) +- [Go Style Guide](#go-style-guide) +- [TypeScript/React Style Guide](#typescriptreact-style-guide) +- [YAML Style Guide](#yaml-style-guide) +- [Shell Script Style Guide](#shell-script-style-guide) +- [Commit Message Style](#commit-message-style) +- [Documentation Style](#documentation-style) + +## General Principles + +1. **Readability First**: Code is read more often than written +2. **Consistency**: Follow existing patterns in the codebase +3. **Simplicity**: Prefer simple, straightforward solutions +4. **Self-Documenting**: Write code that explains itself +5. **DRY**: Don't Repeat Yourself - extract common logic +6. **SOLID**: Follow SOLID principles for object-oriented code + +## Go Style Guide + +### Formatting + +**Use gofmt and goimports**: +```bash +# Format all Go files +gofmt -s -w . + +# Organize imports +goimports -w . +``` + +### Naming Conventions + +**Packages**: +```go +// Good: lowercase, singular, concise +package deployer +package kubernetes + +// Bad: mixed case, plural, verbose +package deployerPackage +package kubernetesResources +``` + +**Variables**: +```go +// Good: camelCase, descriptive +var deploymentID string +var maxRetryCount int + +// Bad: snake_case, abbreviations +var deployment_id string +var max_retry_cnt int +``` + +**Constants**: +```go +// Good: camelCase (not SCREAMING_SNAKE_CASE in Go) +const defaultTimeout = 30 * time.Second +const maxRetries = 3 + +// Exception: Exported constants can be PascalCase +const DefaultNamespace = "default" +``` + +**Functions**: +```go +// Good: PascalCase for exported, camelCase for private +func DeployApplication(ctx context.Context, app *Application) error {} +func validateConfig(cfg *Config) error {} + +// Bad: unclear or abbreviated +func Deploy(a *App) error {} // Too generic +func valCfg(c *Config) error {} // Too abbreviated +``` + +**Interfaces**: +```go +// Good: noun + -er suffix +type Deployer interface {} +type ConfigValidator interface {} + +// Bad: prefix or unclear +type IDeployer interface {} // Don't use "I" prefix +type DeployInterface interface {} // Don't use "Interface" suffix +``` + +### Code Organization + +**File Structure**: +```go +// 1. Package declaration +package deployer + +// 2. Imports (grouped: stdlib, external, internal) +import ( + "context" + "fmt" + + "github.com/pkg/errors" + "go.uber.org/zap" + + "github.com/pipe-cd/pipecd/pkg/model" +) + +// 3. Constants +const ( + defaultRetries = 3 +) + +// 4. Types +type Deployer struct { + logger *zap.Logger +} + +// 5. Constructor +func NewDeployer(logger *zap.Logger) *Deployer { + return &Deployer{logger: logger} +} + +// 6. Public methods +func (d *Deployer) Deploy(ctx context.Context, app *Application) error { + // implementation +} + +// 7. Private methods +func (d *Deployer) validateApp(app *Application) error { + // implementation +} +``` + +### Error Handling + +**Wrap errors with context**: +```go +// Good: provides context +if err := validateConfig(cfg); err != nil { + return fmt.Errorf("failed to validate config: %w", err) +} + +// Bad: loses context +if err := validateConfig(cfg); err != nil { + return err +} +``` + +**Check errors explicitly**: +```go +// Good: handle all errors +result, err := doSomething() +if err != nil { + return err +} + +// Bad: ignore errors +result, _ := doSomething() +``` + +### Comments + +**Package comments**: +```go +// Package deployer provides deployment orchestration functionality +// for managing application deployments across multiple platforms. +package deployer +``` + +**Function comments**: +```go +// DeployApplication initiates a deployment for the specified application. +// It validates the configuration, creates a deployment plan, and executes +// the deployment pipeline stages. +// +// Returns an error if validation fails or deployment cannot be started. +func DeployApplication(ctx context.Context, app *Application) error { + // implementation +} +``` + +**Inline comments**: +```go +// Good: explain WHY, not WHAT +// Wait for rollout to complete to avoid race conditions +time.Sleep(1 * time.Second) + +// Bad: explain WHAT (obvious from code) +// Sleep for 1 second +time.Sleep(1 * time.Second) +``` + +### Context Usage + +```go +// Good: context as first parameter +func ProcessDeployment(ctx context.Context, id string) error { + // Use context for cancellation and timeouts + select { + case <-ctx.Done(): + return ctx.Err() + case result := <-processChan: + return handleResult(result) + } +} + +// Bad: no context support +func ProcessDeployment(id string) error { + // Cannot be cancelled +} +``` + +### Testing + +**Table-driven tests**: +```go +func TestValidateConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + config *Config + expectError bool + errContains string + }{ + { + name: "valid config", + config: &Config{ + Name: "test", + URL: "https://example.com", + }, + expectError: false, + }, + { + name: "missing name", + config: &Config{ + URL: "https://example.com", + }, + expectError: true, + errContains: "name is required", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := validateConfig(tt.config) + + if tt.expectError { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + } else { + require.NoError(t, err) + } + }) + } +} +``` + +## TypeScript/React Style Guide + +### Formatting + +Use Prettier with project configuration: +```bash +yarn --cwd web format +``` + +### Naming Conventions + +**Files**: +``` +// Components: PascalCase +DeploymentList.tsx +ApplicationDetail.tsx + +// Hooks: camelCase with 'use' prefix +useDeployments.ts +useApplications.ts + +// Utilities: camelCase +formatDate.ts +apiClient.ts + +// Types: PascalCase +types.ts +models.ts +``` + +**Variables and Functions**: +```typescript +// Good: camelCase +const deploymentId = "deploy-123"; +const fetchDeployments = async () => {}; + +// Bad: PascalCase or snake_case +const DeploymentId = "deploy-123"; +const fetch_deployments = async () => {}; +``` + +**Types and Interfaces**: +```typescript +// Good: PascalCase, descriptive +interface DeploymentProps { + id: string; + status: DeploymentStatus; +} + +type ApplicationKind = "KUBERNETES" | "TERRAFORM" | "CLOUDRUN"; + +// Bad: prefix or unclear +interface IDeploymentProps {} // No "I" prefix +type AppKind = string; // Not specific enough +``` + +### React Components + +**Functional components with TypeScript**: +```typescript +import { FC } from 'react'; + +interface DeploymentListProps { + projectId: string; + onSelect?: (id: string) => void; +} + +export const DeploymentList: FC = ({ + projectId, + onSelect +}) => { + const [deployments, setDeployments] = useState([]); + + useEffect(() => { + fetchDeployments(projectId).then(setDeployments); + }, [projectId]); + + return ( +
+ {deployments.map(deployment => ( + onSelect?.(deployment.id)} + /> + ))} +
+ ); +}; +``` + +### Hooks + +**Custom hooks**: +```typescript +// Good: starts with 'use', typed return +function useDeployments(projectId: string) { + const [deployments, setDeployments] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + setLoading(true); + fetchDeployments(projectId) + .then(setDeployments) + .catch(setError) + .finally(() => setLoading(false)); + }, [projectId]); + + return { deployments, loading, error }; +} +``` + +### TypeScript Best Practices + +**Avoid `any`**: +```typescript +// Good: specific types +function formatDeployment(deployment: Deployment): string { + return `${deployment.name} - ${deployment.status}`; +} + +// Bad: using any +function formatDeployment(deployment: any): string { + return `${deployment.name} - ${deployment.status}`; +} +``` + +**Use union types**: +```typescript +// Good: explicit union types +type DeploymentStatus = + | "PENDING" + | "RUNNING" + | "SUCCESS" + | "FAILURE" + | "CANCELLED"; + +// Bad: string without constraints +type DeploymentStatus = string; +``` + +## YAML Style Guide + +### Indentation + +```yaml +# Good: 2 spaces +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: my-app + pipeline: + stages: + - name: K8S_SYNC + +# Bad: 4 spaces or tabs +apiVersion: pipecd.dev/v1beta1 +kind: KubernetesApp +spec: + name: my-app +``` + +### Quotes + +```yaml +# Good: use quotes for strings with special characters +name: "my-app" +url: "https://example.com" +description: "This app does X, Y, and Z" + +# Unquoted for simple strings +environment: production +replicas: 3 +``` + +### Comments + +```yaml +# Good: explain non-obvious configurations +spec: + pipeline: + stages: + - name: K8S_SYNC + with: + # Wait for rollout to prevent premature success reporting + waitForRollout: true +``` + +## Shell Script Style Guide + +### Shebang and Options + +```bash +#!/bin/bash +set -euo pipefail # Exit on error, undefined vars, pipe failures + +# Good: strict error handling +do_something || { + echo "Error: something failed" + exit 1 +} +``` + +### Variables + +```bash +# Good: uppercase for constants, lowercase for local +readonly MAX_RETRIES=3 +local deployment_id="deploy-123" + +# Use braces for clarity +echo "Deployment: ${deployment_id}" +``` + +### Functions + +```bash +# Good: descriptive names, local variables +deploy_application() { + local app_name="$1" + local environment="$2" + + echo "Deploying ${app_name} to ${environment}" + # implementation +} +``` + +## Commit Message Style + +Follow [Conventional Commits](https://www.conventionalcommits.org/): + +``` +(): + + + +