diff --git a/.devcontainer/devcontainer.json.tpl b/.devcontainer/devcontainer.json.tpl index f4297af..c0b5dec 100644 --- a/.devcontainer/devcontainer.json.tpl +++ b/.devcontainer/devcontainer.json.tpl @@ -8,9 +8,9 @@ // Use ubuntu-22.04 or ubuntu-18.04 on local arm64/Apple Silicon. "args": { "VARIANT": "ubuntu-22.04" } }, - "mounts": [ - "source=/Users/prabhatsharma/.aws,target=/home/vscode/.aws,type=bind,consistency=cached" - ], + // "mounts": [ + // "source=/Users/prabhatsharma/.aws,target=/home/vscode/.aws,type=bind,consistency=cached" + // ], // Set *default* container specific settings.json values on container create. "settings": {}, diff --git a/docs/.pages b/docs/.pages index fcfb59f..74e5126 100644 --- a/docs/.pages +++ b/docs/.pages @@ -1,5 +1,6 @@ nav: -- Introduction: index.md +- Overview: overview +- Features: features - Getting started: getting-started.md - Quickstart: quickstart.md - Enterprise Edition Installation Guide: openobserve-enterprise-edition-installation-guide.md diff --git a/docs/features/.pages b/docs/features/.pages new file mode 100644 index 0000000..efb183d --- /dev/null +++ b/docs/features/.pages @@ -0,0 +1,10 @@ +nav: + - Log Management: logs.md + - Metric Monitoring: metrics.md + - Distributed Tracing: distributed-tracing.md + - Query & Analysis: query.md + - Frontent Observability: frontend.md + - Storage & Integration: storage.md + - Enterprise Features: enterprise.md + - What's next: upcoming.md + diff --git a/docs/features/deployment.md b/docs/features/deployment.md new file mode 100644 index 0000000..96f4d0e --- /dev/null +++ b/docs/features/deployment.md @@ -0,0 +1,49 @@ +# Logs + +OpenObserve provides powerful log management capabilities for collecting, storing, and analyzing log data from your applications and infrastructure. + +## Overview + +Logs in OpenObserve offer comprehensive observability into your system's behavior, allowing you to track events, debug issues, and monitor application performance. Built with high performance and cost efficiency in mind, OpenObserve handles log ingestion and querying at scale. + +![Logs Page](../images/features/logs-page.png) +*Logs Page view* + +## Key Features + +### Log Ingestion +- **Multiple Protocols**: Support for various log shipping protocols including HTTP, syslog, and popular log shippers +- **Structured & Unstructured**: Handle both JSON structured logs and plain text logs +- **Real-time Processing**: Immediate indexing and availability for search and analysis + +### Search & Query + +- **Field Extraction**: Automatic parsing and extraction of log fields + +![Field Extraction](../images/features/log-feilds-detection.png) + +- **Full-text Search**: Powerful search capabilities across all log fields + +![Full-text Search](../images/features/full-text-search.png) + +- **SQL Queries**: Use familiar SQL syntax for complex log analysis + +![SQL Queries](../images/features/sql-based-log-search.png) + +- **Time-based Filtering**: Efficient time range queries for targeted log exploration + +![Time-based Filtering](../images/features/time-selection.png) + +### Storage & Performance +- **Compressed Storage**: Efficient compression reduces storage costs significantly + +![Compressed Storage](../images/features/data-compression.png) + +- **Fast Retrieval**: Optimized indexing for quick log searches and aggregations + +![Indexing](../images/features/index.png) + +- **Retention Policies**: Configurable data retention to manage storage costs + +![Retention Policies](../images/features/data-retention.png) + diff --git a/docs/features/distributed-tracing.md b/docs/features/distributed-tracing.md new file mode 100644 index 0000000..96f4d0e --- /dev/null +++ b/docs/features/distributed-tracing.md @@ -0,0 +1,49 @@ +# Logs + +OpenObserve provides powerful log management capabilities for collecting, storing, and analyzing log data from your applications and infrastructure. + +## Overview + +Logs in OpenObserve offer comprehensive observability into your system's behavior, allowing you to track events, debug issues, and monitor application performance. Built with high performance and cost efficiency in mind, OpenObserve handles log ingestion and querying at scale. + +![Logs Page](../images/features/logs-page.png) +*Logs Page view* + +## Key Features + +### Log Ingestion +- **Multiple Protocols**: Support for various log shipping protocols including HTTP, syslog, and popular log shippers +- **Structured & Unstructured**: Handle both JSON structured logs and plain text logs +- **Real-time Processing**: Immediate indexing and availability for search and analysis + +### Search & Query + +- **Field Extraction**: Automatic parsing and extraction of log fields + +![Field Extraction](../images/features/log-feilds-detection.png) + +- **Full-text Search**: Powerful search capabilities across all log fields + +![Full-text Search](../images/features/full-text-search.png) + +- **SQL Queries**: Use familiar SQL syntax for complex log analysis + +![SQL Queries](../images/features/sql-based-log-search.png) + +- **Time-based Filtering**: Efficient time range queries for targeted log exploration + +![Time-based Filtering](../images/features/time-selection.png) + +### Storage & Performance +- **Compressed Storage**: Efficient compression reduces storage costs significantly + +![Compressed Storage](../images/features/data-compression.png) + +- **Fast Retrieval**: Optimized indexing for quick log searches and aggregations + +![Indexing](../images/features/index.png) + +- **Retention Policies**: Configurable data retention to manage storage costs + +![Retention Policies](../images/features/data-retention.png) + diff --git a/docs/features/enterprise.md b/docs/features/enterprise.md new file mode 100644 index 0000000..96f4d0e --- /dev/null +++ b/docs/features/enterprise.md @@ -0,0 +1,49 @@ +# Logs + +OpenObserve provides powerful log management capabilities for collecting, storing, and analyzing log data from your applications and infrastructure. + +## Overview + +Logs in OpenObserve offer comprehensive observability into your system's behavior, allowing you to track events, debug issues, and monitor application performance. Built with high performance and cost efficiency in mind, OpenObserve handles log ingestion and querying at scale. + +![Logs Page](../images/features/logs-page.png) +*Logs Page view* + +## Key Features + +### Log Ingestion +- **Multiple Protocols**: Support for various log shipping protocols including HTTP, syslog, and popular log shippers +- **Structured & Unstructured**: Handle both JSON structured logs and plain text logs +- **Real-time Processing**: Immediate indexing and availability for search and analysis + +### Search & Query + +- **Field Extraction**: Automatic parsing and extraction of log fields + +![Field Extraction](../images/features/log-feilds-detection.png) + +- **Full-text Search**: Powerful search capabilities across all log fields + +![Full-text Search](../images/features/full-text-search.png) + +- **SQL Queries**: Use familiar SQL syntax for complex log analysis + +![SQL Queries](../images/features/sql-based-log-search.png) + +- **Time-based Filtering**: Efficient time range queries for targeted log exploration + +![Time-based Filtering](../images/features/time-selection.png) + +### Storage & Performance +- **Compressed Storage**: Efficient compression reduces storage costs significantly + +![Compressed Storage](../images/features/data-compression.png) + +- **Fast Retrieval**: Optimized indexing for quick log searches and aggregations + +![Indexing](../images/features/index.png) + +- **Retention Policies**: Configurable data retention to manage storage costs + +![Retention Policies](../images/features/data-retention.png) + diff --git a/docs/features/frontend.md b/docs/features/frontend.md new file mode 100644 index 0000000..96f4d0e --- /dev/null +++ b/docs/features/frontend.md @@ -0,0 +1,49 @@ +# Logs + +OpenObserve provides powerful log management capabilities for collecting, storing, and analyzing log data from your applications and infrastructure. + +## Overview + +Logs in OpenObserve offer comprehensive observability into your system's behavior, allowing you to track events, debug issues, and monitor application performance. Built with high performance and cost efficiency in mind, OpenObserve handles log ingestion and querying at scale. + +![Logs Page](../images/features/logs-page.png) +*Logs Page view* + +## Key Features + +### Log Ingestion +- **Multiple Protocols**: Support for various log shipping protocols including HTTP, syslog, and popular log shippers +- **Structured & Unstructured**: Handle both JSON structured logs and plain text logs +- **Real-time Processing**: Immediate indexing and availability for search and analysis + +### Search & Query + +- **Field Extraction**: Automatic parsing and extraction of log fields + +![Field Extraction](../images/features/log-feilds-detection.png) + +- **Full-text Search**: Powerful search capabilities across all log fields + +![Full-text Search](../images/features/full-text-search.png) + +- **SQL Queries**: Use familiar SQL syntax for complex log analysis + +![SQL Queries](../images/features/sql-based-log-search.png) + +- **Time-based Filtering**: Efficient time range queries for targeted log exploration + +![Time-based Filtering](../images/features/time-selection.png) + +### Storage & Performance +- **Compressed Storage**: Efficient compression reduces storage costs significantly + +![Compressed Storage](../images/features/data-compression.png) + +- **Fast Retrieval**: Optimized indexing for quick log searches and aggregations + +![Indexing](../images/features/index.png) + +- **Retention Policies**: Configurable data retention to manage storage costs + +![Retention Policies](../images/features/data-retention.png) + diff --git a/docs/features/logs.md b/docs/features/logs.md new file mode 100644 index 0000000..2d97206 --- /dev/null +++ b/docs/features/logs.md @@ -0,0 +1,53 @@ +# Logs + +OpenObserve provides powerful log management capabilities for collecting, storing, and analyzing log data from your applications and infrastructure. + +## Overview + +Logs in OpenObserve offer comprehensive observability into your system's behavior, allowing you to track events, debug issues, and monitor application performance. Built with high performance and cost efficiency in mind, OpenObserve handles log ingestion and querying at scale. + +![Logs Page](../images/features/logs-page.png) +*Logs Page view* + +## Key Features + +### Log Ingestion +- **Multiple Protocols**: Support for various log shipping protocols including HTTP, syslog, and popular log shippers +- **Structured & Unstructured**: Handle both JSON structured logs and plain text logs +- **Real-time Processing**: Immediate indexing and availability for search and analysis + +### Search & Query + +- **Field Extraction**: Automatic parsing and extraction of log fields. + +![Field Extraction](../images/features/log-feilds-detection.png) + +The [Schema Settings](../user-guide/streams/schema-settings.md) tab in the Stream Details panel allows you to inspect and manage the schema used to store and query ingested data. + +- **Full-text Search**: Powerful search capabilities across all log fields + +![Full-text Search](../images/features/full-text-search.png) + +- **SQL Queries**: Use familiar SQL syntax for complex log analysis + +![SQL Queries](../images/features/sql-based-log-search.png) + +- **Time-based Filtering**: Efficient time range queries for targeted log exploration + +![Time-based Filtering](../images/features/time-selection.png) + +### Storage & Performance +- **Compressed Storage**: Efficient compression reduces storage costs significantly + +![Compressed Storage](../images/features/data-compression.png) + +- **Fast Retrieval**: Optimized indexing for quick log searches and aggregations + +![Indexing](../images/features/index.png) + +Know more about [Streams](../user-guide/streams/streams-in-openobserve.md) and its [details](../user-guide/streams/stream-details.md#stream-details) + +- **Retention Policies**: [Configurable data retention](../user-guide/streams/extended-retention.md) to manage storage costs + +![Retention Policies](../images/features/data-retention.png) + diff --git a/docs/features/metrics.md b/docs/features/metrics.md new file mode 100644 index 0000000..b429490 --- /dev/null +++ b/docs/features/metrics.md @@ -0,0 +1,73 @@ +# Metrics + +OpenObserve provides comprehensive metrics collection, storage, and visualization capabilities for monitoring your applications and infrastructure performance in real-time. + +## Overview + +Metrics in OpenObserve enable you to track key performance indicators, monitor system health, and gain insights into your application's behavior over time. Designed for high-throughput environments, OpenObserve efficiently handles metrics ingestion, storage, and querying at scale while maintaining cost-effectiveness. + +![Metrics Page](../images/features/metrics-page.png) +*Metrics Page view* + +## Key Features + +### Metrics Ingestion +- **Multiple Formats**: Support for various metrics formats including Prometheus, InfluxDB, and JSON +- **Push & Pull Models**: Flexible ingestion supporting both push-based and pull-based collection methods +- **High Throughput**: Handle millions of metrics per second with minimal latency +- **Batch Processing**: Efficient batch ingestion for optimal performance + +### Data Types & Structure +- **Time Series Data**: Native support for time-series metrics with timestamp precision +- **Multi-dimensional**: Handle metrics with multiple labels and dimensions for detailed analysis + +![Multi-dimensional Metrics](../images/features/metrics-dimensions.png) + +- **Aggregation Functions**: Built-in support for common aggregation functions (sum, avg, min, max, count) +- **Custom Metrics**: Flexible schema for application-specific metrics and KPIs + +### Query & Analysis + +- **PromQL Support**: Full compatibility with Prometheus Query Language for familiar querying + +![PromQL Queries](../images/features/promql-queries.png) + +- **SQL Interface**: Use SQL syntax for complex metrics analysis and reporting + +- **Time Range Selection**: Flexible time range queries with support for relative and absolute time periods + +![Time Range Selection](../images/features/metrics-time-selection.png) + +- **Mathematical Operations**: Perform calculations and transformations on metrics data + +### Visualization & Dashboards + +- **Real-time Charts**: Interactive time-series visualizations with multiple chart types + +![Metrics Visualization](../images/features/metrics-charts.png) + +- **Custom Dashboards**: Create comprehensive dashboards with multiple metrics panels + +- **Alerting Integration**: Set up alerts based on metrics thresholds and conditions + +![Metrics Alerting](../images/features/metrics-alerts.png) + +### Storage & Performance + +- **Optimized Compression**: Advanced compression algorithms reduce storage costs by up to 90% + +![Metrics Compression](../images/features/metrics-compression.png) + +- **Efficient Indexing**: High-performance indexing for fast query execution across large datasets + +- **Downsampling**: Automatic data downsampling for long-term storage optimization + +- **Retention Policies**: [Configurable retention settings](../user-guide/streams/extended-retention.md) to balance storage costs and data availability + +![Metrics Retention](../images/features/metrics-retention.png) + +### Integration & Compatibility +- **Prometheus Compatible**: Full compatibility with Prometheus ecosystem and exporters +- **Grafana Integration**: Native support for Grafana dashboards and visualization +- **API Access**: RESTful APIs for programmatic access to metrics data +- **Standard Exporters**: Support for popular metrics exporters (Node Exporter, cAdvisor, etc.) diff --git a/docs/features/query.md b/docs/features/query.md new file mode 100644 index 0000000..96f4d0e --- /dev/null +++ b/docs/features/query.md @@ -0,0 +1,49 @@ +# Logs + +OpenObserve provides powerful log management capabilities for collecting, storing, and analyzing log data from your applications and infrastructure. + +## Overview + +Logs in OpenObserve offer comprehensive observability into your system's behavior, allowing you to track events, debug issues, and monitor application performance. Built with high performance and cost efficiency in mind, OpenObserve handles log ingestion and querying at scale. + +![Logs Page](../images/features/logs-page.png) +*Logs Page view* + +## Key Features + +### Log Ingestion +- **Multiple Protocols**: Support for various log shipping protocols including HTTP, syslog, and popular log shippers +- **Structured & Unstructured**: Handle both JSON structured logs and plain text logs +- **Real-time Processing**: Immediate indexing and availability for search and analysis + +### Search & Query + +- **Field Extraction**: Automatic parsing and extraction of log fields + +![Field Extraction](../images/features/log-feilds-detection.png) + +- **Full-text Search**: Powerful search capabilities across all log fields + +![Full-text Search](../images/features/full-text-search.png) + +- **SQL Queries**: Use familiar SQL syntax for complex log analysis + +![SQL Queries](../images/features/sql-based-log-search.png) + +- **Time-based Filtering**: Efficient time range queries for targeted log exploration + +![Time-based Filtering](../images/features/time-selection.png) + +### Storage & Performance +- **Compressed Storage**: Efficient compression reduces storage costs significantly + +![Compressed Storage](../images/features/data-compression.png) + +- **Fast Retrieval**: Optimized indexing for quick log searches and aggregations + +![Indexing](../images/features/index.png) + +- **Retention Policies**: Configurable data retention to manage storage costs + +![Retention Policies](../images/features/data-retention.png) + diff --git a/docs/features/storage.md b/docs/features/storage.md new file mode 100644 index 0000000..96f4d0e --- /dev/null +++ b/docs/features/storage.md @@ -0,0 +1,49 @@ +# Logs + +OpenObserve provides powerful log management capabilities for collecting, storing, and analyzing log data from your applications and infrastructure. + +## Overview + +Logs in OpenObserve offer comprehensive observability into your system's behavior, allowing you to track events, debug issues, and monitor application performance. Built with high performance and cost efficiency in mind, OpenObserve handles log ingestion and querying at scale. + +![Logs Page](../images/features/logs-page.png) +*Logs Page view* + +## Key Features + +### Log Ingestion +- **Multiple Protocols**: Support for various log shipping protocols including HTTP, syslog, and popular log shippers +- **Structured & Unstructured**: Handle both JSON structured logs and plain text logs +- **Real-time Processing**: Immediate indexing and availability for search and analysis + +### Search & Query + +- **Field Extraction**: Automatic parsing and extraction of log fields + +![Field Extraction](../images/features/log-feilds-detection.png) + +- **Full-text Search**: Powerful search capabilities across all log fields + +![Full-text Search](../images/features/full-text-search.png) + +- **SQL Queries**: Use familiar SQL syntax for complex log analysis + +![SQL Queries](../images/features/sql-based-log-search.png) + +- **Time-based Filtering**: Efficient time range queries for targeted log exploration + +![Time-based Filtering](../images/features/time-selection.png) + +### Storage & Performance +- **Compressed Storage**: Efficient compression reduces storage costs significantly + +![Compressed Storage](../images/features/data-compression.png) + +- **Fast Retrieval**: Optimized indexing for quick log searches and aggregations + +![Indexing](../images/features/index.png) + +- **Retention Policies**: Configurable data retention to manage storage costs + +![Retention Policies](../images/features/data-retention.png) + diff --git a/docs/features/upcoming.md b/docs/features/upcoming.md new file mode 100644 index 0000000..96f4d0e --- /dev/null +++ b/docs/features/upcoming.md @@ -0,0 +1,49 @@ +# Logs + +OpenObserve provides powerful log management capabilities for collecting, storing, and analyzing log data from your applications and infrastructure. + +## Overview + +Logs in OpenObserve offer comprehensive observability into your system's behavior, allowing you to track events, debug issues, and monitor application performance. Built with high performance and cost efficiency in mind, OpenObserve handles log ingestion and querying at scale. + +![Logs Page](../images/features/logs-page.png) +*Logs Page view* + +## Key Features + +### Log Ingestion +- **Multiple Protocols**: Support for various log shipping protocols including HTTP, syslog, and popular log shippers +- **Structured & Unstructured**: Handle both JSON structured logs and plain text logs +- **Real-time Processing**: Immediate indexing and availability for search and analysis + +### Search & Query + +- **Field Extraction**: Automatic parsing and extraction of log fields + +![Field Extraction](../images/features/log-feilds-detection.png) + +- **Full-text Search**: Powerful search capabilities across all log fields + +![Full-text Search](../images/features/full-text-search.png) + +- **SQL Queries**: Use familiar SQL syntax for complex log analysis + +![SQL Queries](../images/features/sql-based-log-search.png) + +- **Time-based Filtering**: Efficient time range queries for targeted log exploration + +![Time-based Filtering](../images/features/time-selection.png) + +### Storage & Performance +- **Compressed Storage**: Efficient compression reduces storage costs significantly + +![Compressed Storage](../images/features/data-compression.png) + +- **Fast Retrieval**: Optimized indexing for quick log searches and aggregations + +![Indexing](../images/features/index.png) + +- **Retention Policies**: Configurable data retention to manage storage costs + +![Retention Policies](../images/features/data-retention.png) + diff --git a/docs/images/features/data-compression.png b/docs/images/features/data-compression.png new file mode 100644 index 0000000..abf2518 Binary files /dev/null and b/docs/images/features/data-compression.png differ diff --git a/docs/images/features/data-retention.png b/docs/images/features/data-retention.png new file mode 100644 index 0000000..1a06592 Binary files /dev/null and b/docs/images/features/data-retention.png differ diff --git a/docs/images/features/full-text-search.png b/docs/images/features/full-text-search.png new file mode 100644 index 0000000..ada1fc8 Binary files /dev/null and b/docs/images/features/full-text-search.png differ diff --git a/docs/images/features/index.png b/docs/images/features/index.png new file mode 100644 index 0000000..4c5b8d5 Binary files /dev/null and b/docs/images/features/index.png differ diff --git a/docs/images/features/log-feilds-detection.png b/docs/images/features/log-feilds-detection.png new file mode 100644 index 0000000..25ed246 Binary files /dev/null and b/docs/images/features/log-feilds-detection.png differ diff --git a/docs/images/features/logs-page.png b/docs/images/features/logs-page.png new file mode 100644 index 0000000..5ae93c4 Binary files /dev/null and b/docs/images/features/logs-page.png differ diff --git a/docs/images/features/metrics-alerts.png b/docs/images/features/metrics-alerts.png new file mode 100644 index 0000000..a559907 Binary files /dev/null and b/docs/images/features/metrics-alerts.png differ diff --git a/docs/images/features/metrics-charts.png b/docs/images/features/metrics-charts.png new file mode 100644 index 0000000..d0b4b3d Binary files /dev/null and b/docs/images/features/metrics-charts.png differ diff --git a/docs/images/features/metrics-compression.png b/docs/images/features/metrics-compression.png new file mode 100644 index 0000000..40453a9 Binary files /dev/null and b/docs/images/features/metrics-compression.png differ diff --git a/docs/images/features/metrics-page.png b/docs/images/features/metrics-page.png new file mode 100644 index 0000000..a444017 Binary files /dev/null and b/docs/images/features/metrics-page.png differ diff --git a/docs/images/features/metrics-retention.png b/docs/images/features/metrics-retention.png new file mode 100644 index 0000000..39e71b1 Binary files /dev/null and b/docs/images/features/metrics-retention.png differ diff --git a/docs/images/features/metrics-time-selection.png b/docs/images/features/metrics-time-selection.png new file mode 100644 index 0000000..ca4088a Binary files /dev/null and b/docs/images/features/metrics-time-selection.png differ diff --git a/docs/images/features/promql-queries.png b/docs/images/features/promql-queries.png new file mode 100644 index 0000000..14744cf Binary files /dev/null and b/docs/images/features/promql-queries.png differ diff --git a/docs/images/features/sql-based-log-search.png b/docs/images/features/sql-based-log-search.png new file mode 100644 index 0000000..6786286 Binary files /dev/null and b/docs/images/features/sql-based-log-search.png differ diff --git a/docs/images/features/time-selection.png b/docs/images/features/time-selection.png new file mode 100644 index 0000000..aa7ada7 Binary files /dev/null and b/docs/images/features/time-selection.png differ diff --git a/docs/index.md b/docs/index.md index 9f14a41..27586f2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,59 +1,112 @@ # OpenObserve: Introduction -`OpenObserve` is a cloud native observability platform (`Logs`, `Metrics`, `Traces`) that provides `~140x lower storage costs` (compared to Elasticsearch. YMMV. Could be higher or lower based on data entropy) for real life log data, significantly lower operational cost and ease of use. It can scale to petabytes of data, is highly performant and allows you to sleep better at night 😀. If you are looking for an observability tool for logs, metrics and traces, do evaluate OpenObserve and how its approach towards observability could help you build better software, save money on observability costs, and sleep better. - - -## Project Status, Features and Roadmap - -Following is the list of available features and roadmap. - -| # | Feature | Status | -|---|---------------------------------------------------------------|---------------------| -| 1 | Log search | Available | -| 2 | Highly compressed storage of data | Available | -| 3 | Dynamic evolution of schema | Available | -| 4 | Out of the box authentication | Available | -| 5 | Support of S3, MinIO, GCS, Azure blob for data storage | Available | -| 6 | Advanced GUI | Available | -| 7 | SQL based query language | Available | -| 8 | Support for very high cardinality data | Available | -| 9 | Search-around logs data | Available | -| 10 | User defined Ingest and Query functions (VRL based) | Available | -| 11 | Multi-tenancy | Available | -| 12 | Ingestion API compatibility with Elasticsearch | Available | -| 13 | Search and aggregation API compatibility with Elasticsearch | [Through zPlane](zplane) | -| 14 | Standard alerts (Based on logs) | Available | -| 15 | Real time Alerts (Based on logs) | Available | -| 16 | High Availability (HA) and clustering | Available | -| 17 | Stateless nodes | Available | -| 18 | Localization for multiple languages | Available | -| 19 | Prebuilt binaries for multiple platforms | Available | -| 20 | Prebuilt container images for multiple platforms | Available | -| 21 | Prebuilt container images for with SIMD acceleration | Available | -| 22 | SIMD support for vectorized processing (AVX512 and Neon) | Available | -| 23 | Dashboards | Available | -| 24 | Metrics | Available | -| 25 | PromQL support for metrics | Available (97% PromQL compliant) | -| 26 | Traces | Available | -| 27 | Standard alerts (Based on metrics) | Available | -| 28 | Real time Alerts (Based on metrics) | Available | -| 29 | Template based alert target (Allows alerting to slack, teams and many more) | Available | -| 30 | Send alerts to Prometheus alertmanager | Available | -| 31 | Ingest AWS logs (cloudwatch, VPC flow logs, AWS WAF and more) using Kinesis firehose | Available | -| 32 | Single Sign On(SSO) | Available (Enterprise) | -| 33 | RBAC (Role Based Access Control) | Available (Enterprise) | -| 34 | Front end - Performance analytics | Available | -| 35 | Front end - Session Replay | Available | -| 36 | Front end - Error tracking | Available | -| 37 | Log patterns | To start | -| 38 | Anomaly detection | To start | -| 39 | Correlation between logs, metrics and traces | To start | -| 40 | Dashboard migration from Splunk, Kibana and Grafana | beta. [https://dc.openobserve.ai](https://dc.openobserve.ai) [https://github.com/openobserve/dashboard_converter](https://github.com/openobserve/dashboard_converter) | - - -Please raise any new feature requests via [github issue tracker](https://github.com/openobserve/openobserve/issues). - -You can use either the open source version or [OpenObserve Cloud](https://cloud.openobserve.ai). [OpenObserve Cloud](https://cloud.openobserve.ai) is built on top of open source OpenObserve but has minor differences to account for its SaaS nature. We will highlight the differences in the documentation whenever needed. +## What is OpenObserve? + +**OpenObserve**, also referred to as O2, is a cloud native observability platform that unifies **logs, metrics, and traces**. It provides *~140x lower storage costs* (compared to Elasticsearch. Results can be higher or lower based on data entropy) for real life log data, significantly lower operational cost and ease of use. + +It can scale to **petabytes of data**, is highly performant and allows you to sleep better at night 😀. If you are looking for an observability tool for logs, metrics and traces, do evaluate OpenObserve and understand how its architectural approach can optimize your observability costs and enhance software development. + +## OpenObserve Features + +#### Logs Management +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 1 | Log search | ✅ Available | Advanced search capabilities with SQL-based queries | +| 2 | Highly compressed storage of data | ✅ Available | Efficient data compression reduces storage requirements | +| 3 | Dynamic evolution of schema | ✅ Available | Automatic schema evolution without manual intervention | +| 9 | Search-around logs data | ✅ Available | Contextual log exploration around specific events | +| 8 | Support for very high cardinality data | ✅ Available | Handle datasets with millions of unique values | +| 10 | User defined Ingest and Query functions (VRL based) | ✅ Available | Custom data processing and transformation functions | +| 14 | Standard alerts | ✅ Available | Log-based alerting system | +| 15 | Real time Alerts | ✅ Available | Immediate log-based notifications | + +#### Metrics & Monitoring +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 24 | Metrics | ✅ Available | Native metrics ingestion and storage | +| 25 | PromQL support for metrics | ✅ Available | 97% PromQL compliant for Prometheus compatibility | +| 23 | Dashboards | ✅ Available | Customizable visualization dashboards | +| 27 | Standard alerts | ✅ Available | Metric-based alerting system | +| 28 | Real time Alerts | ✅ Available | Immediate metric-based notifications | + +#### Distributed Tracing +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 26 | Traces | ✅ Available | Complete distributed tracing support | + +#### Query & Analysis +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 7 | SQL based query language | ✅ Available | Standard SQL interface for familiar querying | +| 6 | Advanced GUI | ✅ Available | Built-in interface eliminates need for additional components | + + +#### Deployment & Scaling +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 16 | High Availability (HA) and clustering | ✅ Available | Clustering and failover capabilities | +| 17 | Stateless nodes | ✅ Available | Horizontal scaling without data replication concerns | +| 11 | Multi-tenancy | ✅ Available | Secure data isolation between tenants | +| 22 | SIMD support for vectorized processing (AVX512 and Neon) | ✅ Available | Hardware-accelerated data processing | +| 21 | Prebuilt container images with SIMD acceleration | ✅ Available | Optimized container deployments | + +#### Storage & Integration +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 5 | Support of S3, MinIO, GCS, Azure blob for data storage | ✅ Available | Multi-cloud storage backend support | +| 12 | Ingestion API compatibility with Elasticsearch | ✅ Available | Compatible ingestion APIs for existing tooling | +| 13 | Search and aggregation API compatibility with Elasticsearch | 🔧 Through zPlane | Full Elasticsearch API compatibility via enterprise add-on | +| 4 | Out of the box authentication | ✅ Available | Built-in authentication system | +| 18 | Localization for multiple languages | ✅ Available | Support for multiple languages | + +#### Cloud Integration +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 31 | Ingest AWS logs (cloudwatch, VPC flow logs, AWS WAF and more) using Kinesis firehose | ✅ Available | Native AWS log ingestion | + +### Enterprise Features + +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 32 | Single Sign On (SSO) | ✅ Available (Enterprise) | Integration with enterprise identity providers | +| 33 | RBAC (Role Based Access Control) | ✅ Available (Enterprise) | Role-based access control with granular permissions | + +### Alerting & Notifications + +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 29 | Template based alert target (Allows alerting to slack, teams and many more) | ✅ Available | Flexible alerting to multiple platforms | +| 30 | Send alerts to Prometheus alertmanager | ✅ Available | Integration with Prometheus AlertManager | + +### Frontend Observability + +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 34 | Front end - Performance analytics | ✅ Available | Frontend performance monitoring | +| 35 | Front end - Session Replay | ✅ Available | User session recording and analysis | +| 36 | Front end - Error tracking | ✅ Available | Frontend error monitoring and alerting | + +### Deployment Options + +| # | Feature | Status | Description | +|---|---------|--------|-------------| +| 19 | Prebuilt binaries for multiple platforms | ✅ Available | Cross-platform binary distributions | +| 20 | Prebuilt container images for multiple platforms | ✅ Available | Multi-architecture container support | + +#### Getting Started +For detailed instructions on installation and setting up your first observation, refer to our [Getting Started Guide](getting-started.md). + +### Roadmap + +#### Upcoming Features + +| Feature | Status | Description | +|---------|--------|-------------| +| Log patterns | 🔨 In Development | Automated pattern recognition in log data | +| Anomaly detection | 🔨 In Development | Machine learning-based anomaly identification | +| Cross-signal correlation | 🔨 In Development | Unified correlation between logs, metrics, and traces | + +Please raise any new feature requests via [GitHub issue tracker](https://github.com/openobserve/openobserve/issues). ## Guiding principles @@ -83,23 +136,33 @@ We want to build the best software in the observability category in the world, a Elasticsearch is a general purpose search engine which can be used for app search or log search. OpenObserve is built specifically for log search. If you are looking for a lightweight alternative to Elasticsearch then you should take a look at ZincSearch. -OpenObserve provides ability to index data in multiple ways to make it faster yet keep storage size low. It uses a combination of partitioning, bloom filters, inverted indexes, caching and columnar storage to make search and aggregation queries faster. You can combine these to find the right balance between storage and performance. [Uber found 80% of queries in their production environment to be aggregation queries](https://www.uber.com/en-IN/blog/logging/) and columnar data storage of OpenObserve means that aggregation queries will typically be much faster than Elasticsearch. +#### Technical Advantages +OpenObserve provides ability to index data in multiple ways to make it faster yet keep storage size low. It uses a combination of: -Below is the result when we sent real life log data from our kubernetes cluster to both Elasticsearch and OpenObserve using fluentbit. This only pertains to storage. Cost of EBS volume is [8 cents/GB/Month (GP3)](https://aws.amazon.com/ebs/pricing/), cost of s3 is [2.3 cents/GB/month](https://aws.amazon.com/s3/pricing/). In HA mode in Elasticsearch you generally have 1 primary node and 2 replicas. You don't need to replicate s3 for data durability/availability as [AWS redundantly stores your objects on multiple devices across a minimum of three Availability Zones (AZs) in an Amazon S3 Region](https://aws.amazon.com/s3/faqs/). +- Partitioning +- Bloom filters +- Inverted indexes +- Caching +- Columnar storage -![OpenObserve Vs Elasticsearch storage](./images/zo_vs_es.png) +[Uber found 80% of queries in their production environment to be aggregation queries](https://www.uber.com/en-IN/blog/logging/) and columnar data storage of OpenObserve means that aggregation queries will typically be much faster than Elasticsearch. -OpenObserve offers significant advantage of 140x lower storage costs compared to Elasticsearch in the above scenario (YMMV, you could get higher or lower values based on entropy of data). That does not even consider additional unused EBS volume capacity that needs to be available in order to not run out of disk space and the effort that it requires to keep monitoring disk usage so it is not filled. +#### Storage Cost Comparison -Stateless node architecture allows OpenObserve to scale horizontally without worrying about data replication or corruption challenges. +Below is the result when we sent real life log data from our kubernetes cluster to both Elasticsearch and OpenObserve using fluentbit. This only pertains to storage. Cost of EBS volume is [8 cents/GB/Month (GP3)](https://aws.amazon.com/ebs/pricing/), cost of s3 is [2.3 cents/GB/month](https://aws.amazon.com/s3/pricing/). In HA mode in Elasticsearch you generally have 1 primary node and 2 replicas. You don't need to replicate s3 for data durability/availability as [AWS redundantly stores your objects on multiple devices across a minimum of three Availability Zones (AZs) in an Amazon S3 Region](https://aws.amazon.com/s3/faqs/). -OpenObserve's lack of index mapping and associated challenges provides a hassle-free experience in managing clusters. +![OpenObserve Vs Elasticsearch storage](./images/zo_vs_es.png) -You will typically see much lower operational effort and cost in managing OpenObserve clusters compared to Elasticsearch. +OpenObserve offers significant advantage of 140x lower storage costs compared to Elasticsearch in the above scenario. Your actual results may vary depending on how compressible your specific log data is. This doesn't even consider additional unused EBS volume capacity and monitoring overhead. -The platform's built-in GUI eliminates the need for another component like Kibana, and has awesome performance, thanks to Rust, without the challenges of JVM. +#### Operational Advantages -In contrast to Elasticsearch, which is a general-purpose search engine that doubles as an observability tool, OpenObserve was built from the ground up as an observability tool, with high focus on delivering exceptional observability. +- `Stateless Architecture`: Scale horizontally without data replication or corruption challenges +- `No Index Mapping`: Hassle-free cluster management without index mapping complexities +- `Lower Operational Cost`: Significantly reduced effort in managing clusters +- `Built-in GUI`: Eliminates need for additional components like Kibana +- `Rust Performance`: Awesome performance without JVM challenges +- `Purpose-Built`: Built from ground up as observability tool, not general-purpose search ## Elasticsearch compatibility diff --git a/docs/overview/.pages b/docs/overview/.pages new file mode 100644 index 0000000..ce58dd7 --- /dev/null +++ b/docs/overview/.pages @@ -0,0 +1,8 @@ +nav: + - Introduction: index.md + - Downloads: downloads.md + - Our Principle: guiding-principles.md + - Features: features + - Comparison with Alternatives: comparsion.md + - Architecture: architecture.md + diff --git a/docs/architecture.md b/docs/overview/architecture.md similarity index 100% rename from docs/architecture.md rename to docs/overview/architecture.md diff --git a/docs/overview/comparison.md b/docs/overview/comparison.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/downloads.md b/docs/overview/downloads.md similarity index 100% rename from docs/downloads.md rename to docs/overview/downloads.md diff --git a/docs/overview/guiding-principles.md b/docs/overview/guiding-principles.md new file mode 100644 index 0000000..e1e9821 --- /dev/null +++ b/docs/overview/guiding-principles.md @@ -0,0 +1,56 @@ +# Our Principles + +We are building the best software in the observability category in the world. These principles guide every decision we make: + +## Instant Value +**Get started in minutes, not hours** + +- Install or sign up in under 2 minutes +- Start ingesting data and gaining insights immediately +- Zero configuration required for common use cases +- Immediate ROI on your time investment + +## Effortless Operations +**Systems that manage themselves** + +- Self-healing and automatic recovery from issues +- Simple scaling - as simple as adding nodes in in an autoscaling group (in AWS) or changing the number of replicas (in k8s). +- No specialized database administration skills required +- Worry-free upgrades and maintenance +- Built-in reliability and stability + +## Complete Observability +**Everything you need in one platform** + +- Unified logs, metrics, and traces with correlation +- Intuitive UI and powerful APIs +- Feature-rich yet simple to use +- Designed for real-world workflows + +## Accessible Costs +**Enterprise capabilities without enterprise pricing** + +- Dramatically lower storage and operational costs +- Transparent pricing for both self-hosted and SaaS +- No surprise bills or cost spirals +- Optimize your observability budget + +## Universal Usability +**Built for everyone** + +- Beginners can accomplish complex tasks easily +- Experts can leverage existing skills and knowledge +- Familiar interfaces and standard protocols +- Gentle learning curve with powerful advanced features + +## Real-World Performance +**Optimized for actual usage patterns** + +- High performance for common observability workloads +- Smart tradeoffs that benefit the majority of users +- Hardware-accelerated processing where it matters +- Scales efficiently with your data growth + +--- + +*These principles aren't just words - they're measurable commitments that drive our product development and user experience.* \ No newline at end of file diff --git a/docs/overview/index.md b/docs/overview/index.md new file mode 100644 index 0000000..e69de29 diff --git a/overrides/partials/header.html b/overrides/partials/header.html index 6f62732..a1c339a 100644 --- a/overrides/partials/header.html +++ b/overrides/partials/header.html @@ -42,7 +42,7 @@ {% if config.theme.palette %} {% if not config.theme.palette is mapping %} - {% include "partials/palette.html" %} + {# {% include "partials/palette.html" %} #} {% endif %} {% endif %} {% if not config.theme.palette is mapping %}