diff --git a/docs/_include/links.md b/docs/_include/links.md index ef91a9c2..9b2cb407 100644 --- a/docs/_include/links.md +++ b/docs/_include/links.md @@ -19,6 +19,7 @@ [DynamoDB CDC Relay]: https://cratedb-toolkit.readthedocs.io/io/dynamodb/cdc.html [DynamoDB CDC Relay with AWS Lambda]: https://cratedb-toolkit.readthedocs.io/io/dynamodb/cdc-lambda.html [DynamoDB Table Loader]: https://cratedb-toolkit.readthedocs.io/io/dynamodb/loader.html +[Executable stack with Apache Kafka, Apache Flink, and CrateDB]: https://github.com/crate/cratedb-examples/tree/main/framework/flink/kafka-jdbcsink-java [Geospatial Data Model]: https://cratedb.com/data-model/geospatial [Geospatial Database]: https://cratedb.com/geospatial-spatial-database [HNSW]: https://en.wikipedia.org/wiki/Hierarchical_navigable_small_world diff --git a/docs/conf.py b/docs/conf.py index 75121740..485fb792 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,6 +53,8 @@ r"https://dzone.com/.*", # 504 Client Error: Gateway Timeout for url r"https://web.archive.org/.*", + # 403 Client Error: Forbidden for url + r"https://www.tableau.com/", ] linkcheck_anchors_ignore_for_url += [ diff --git a/docs/connect/df/index.md b/docs/connect/df/index.md index 9ee05087..7313a199 100644 --- a/docs/connect/df/index.md +++ b/docs/connect/df/index.md @@ -1,40 +1,16 @@ (df)= (dataframe)= (dataframes)= +(dataframe-examples)= # CrateDB and DataFrame libraries -Data frame libraries and frameworks which can -be used together with CrateDB. - - -:::::{grid} 1 2 2 2 -:margin: 4 4 0 0 -:padding: 0 -:gutter: 2 - -::::{grid-item-card} {material-outlined}`lightbulb;2em` Tutorials -:link: guide:dataframes -:link-type: ref -Learn how to use CrateDB together with popular open-source data frame -libraries, on behalf of hands-on tutorials and code examples. -+++ -{tag-info}`Dask` {tag-info}`pandas` {tag-info}`Polars` -:::: - -::::{grid-item-card} {material-outlined}`read_more;2em` SQLAlchemy -CrateDB's SQLAlchemy dialect implementation provides fundamental infrastructure -to integrations with Dask, pandas, and Polars. -+++ -[ORM Guides](inv:guide#orm) • -{ref}`ORM Catalog ` -:::: - -::::: - +How to use CrateDB together with popular open-source DataFrame libraries. (dask)= ## Dask +:::{rubric} About +::: [Dask] is a parallel computing library for analytics with task scheduling. It is built on top of the Python programming language, making it easy to scale the Python libraries that you know and love, like NumPy, pandas, and scikit-learn. @@ -56,10 +32,20 @@ the Python libraries that you know and love, like NumPy, pandas, and scikit-lear :style: "clear: both" ``` +:::{rubric} Learn +::: +- [Guide to efficient data ingestion to CrateDB with pandas and Dask] +- [Efficient batch/bulk INSERT operations with pandas, Dask, and SQLAlchemy] +- [Import weather data using Dask] +- [Dask code examples] + (pandas)= ## pandas +:::{rubric} About +::: + ```{div} :style: "float: right" [![](https://pandas.pydata.org/static/img/pandas.svg){w=180px}](https://pandas.pydata.org/) @@ -92,10 +78,21 @@ and operations for manipulating numerical tables and time series. :style: "clear: both" ``` +:::{rubric} Learn +::: +- [Guide to efficient data ingestion to CrateDB with pandas] +- [Importing Parquet files into CrateDB using Apache Arrow and SQLAlchemy] +- [pandas code examples] +- [From data storage to data analysis: Tutorial on CrateDB and pandas] + + (polars)= ## Polars +:::{rubric} About +::: + ```{div} :style: "float: right; margin-left: 0.5em" [![](https://github.com/pola-rs/polars-static/raw/master/logos/polars-logo-dark.svg){w=180px}](https://pola.rs/) @@ -147,13 +144,9 @@ This allows you to easily integrate Polars into your existing data stack. :style: "clear: both" ``` - -```{toctree} -:maxdepth: 1 -:hidden: - -learn -``` +:::{rubric} Learn +::: +- [Polars code examples] [Apache Arrow]: https://arrow.apache.org/ @@ -162,3 +155,13 @@ learn [Dask Futures]: https://docs.dask.org/en/latest/futures.html [pandas]: https://pandas.pydata.org/ [Polars]: https://pola.rs/ + +[Dask code examples]: https://github.com/crate/cratedb-examples/tree/main/by-dataframe/dask +[Efficient batch/bulk INSERT operations with pandas, Dask, and SQLAlchemy]: https://cratedb.com/docs/python/en/latest/by-example/sqlalchemy/dataframe.html +[From data storage to data analysis: Tutorial on CrateDB and pandas]: https://community.cratedb.com/t/from-data-storage-to-data-analysis-tutorial-on-cratedb-and-pandas/1440 +[Guide to efficient data ingestion to CrateDB with pandas]: https://community.cratedb.com/t/guide-to-efficient-data-ingestion-to-cratedb-with-pandas/1541 +[Guide to efficient data ingestion to CrateDB with pandas and Dask]: https://community.cratedb.com/t/guide-to-efficient-data-ingestion-to-cratedb-with-pandas-and-dask/1482 +[Import weather data using Dask]: https://github.com/crate/cratedb-examples/blob/main/topic/timeseries/dask-weather-data-import.ipynb +[Importing Parquet files into CrateDB using Apache Arrow and SQLAlchemy]: https://community.cratedb.com/t/importing-parquet-files-into-cratedb-using-apache-arrow-and-sqlalchemy/1161 +[pandas code examples]: https://github.com/crate/cratedb-examples/tree/main/by-dataframe/pandas +[Polars code examples]: https://github.com/crate/cratedb-examples/tree/main/by-dataframe/polars diff --git a/docs/connect/df/learn.md b/docs/connect/df/learn.md deleted file mode 100644 index 6eb0c722..00000000 --- a/docs/connect/df/learn.md +++ /dev/null @@ -1,29 +0,0 @@ -(dataframe-examples)= - -# DataFrame Examples - -How to use CrateDB together with popular open-source dataframe libraries. - -## Dask -- [Guide to efficient data ingestion to CrateDB with pandas and Dask] -- [Efficient batch/bulk INSERT operations with pandas, Dask, and SQLAlchemy] -- [Import weather data using Dask] -- [Dask code examples] - -## pandas -- [Guide to efficient data ingestion to CrateDB with pandas] -- [Importing Parquet files into CrateDB using Apache Arrow and SQLAlchemy] -- [pandas code examples] - -## Polars -- [Polars code examples] - - -[Dask code examples]: https://github.com/crate/cratedb-examples/tree/main/by-dataframe/dask -[Efficient batch/bulk INSERT operations with pandas, Dask, and SQLAlchemy]: https://cratedb.com/docs/python/en/latest/by-example/sqlalchemy/dataframe.html -[Guide to efficient data ingestion to CrateDB with pandas]: https://community.cratedb.com/t/guide-to-efficient-data-ingestion-to-cratedb-with-pandas/1541 -[Guide to efficient data ingestion to CrateDB with pandas and Dask]: https://community.cratedb.com/t/guide-to-efficient-data-ingestion-to-cratedb-with-pandas-and-dask/1482 -[Import weather data using Dask]: https://github.com/crate/cratedb-examples/blob/main/topic/timeseries/dask-weather-data-import.ipynb -[Importing Parquet files into CrateDB using Apache Arrow and SQLAlchemy]: https://community.cratedb.com/t/importing-parquet-files-into-cratedb-using-apache-arrow-and-sqlalchemy/1161 -[pandas code examples]: https://github.com/crate/cratedb-examples/tree/main/by-dataframe/pandas -[Polars code examples]: https://github.com/crate/cratedb-examples/tree/main/by-dataframe/polars diff --git a/docs/connect/index.md b/docs/connect/index.md index f05713bf..68fd4db6 100644 --- a/docs/connect/index.md +++ b/docs/connect/index.md @@ -151,9 +151,9 @@ Python configure Drivers -CLI -DataFrame -ORM +CLI programs +DataFrame libraries +ORM libraries ``` ```{toctree} diff --git a/docs/domain/index.md b/docs/domain/index.md index 67b5ee1d..666d0dad 100644 --- a/docs/domain/index.md +++ b/docs/domain/index.md @@ -23,7 +23,7 @@ telemetry/index analytics/index industrial/index timeseries/index -ml/index +Machine Learning ``` diff --git a/docs/domain/ml/index.md b/docs/domain/ml/index.md index bc070378..5ae7d738 100644 --- a/docs/domain/ml/index.md +++ b/docs/domain/ml/index.md @@ -1,13 +1,16 @@ (ml)= +(ml-tools)= (machine-learning)= - -# Machine Learning +# Machine Learning with CrateDB :::{include} /_include/links.md ::: :::{include} /_include/styles.html ::: +Machine learning applications and frameworks +which can be used together with CrateDB. + Integrate CrateDB with machine learning frameworks and tools, for MLOps and vector database operations. @@ -60,6 +63,27 @@ generation (RAG), and other applications. (mlflow)= ### MLflow +:::{rubric} About +::: +```{div} +:style: "float: right; margin-left: 1em" +[![](https://github.com/crate/crate-clients-tools/assets/453543/d1d4f4ac-1b44-46b8-ba6f-4a82607c57d3){w=180px}](https://mlflow.org/) +``` + +[MLflow] is an open source platform to manage the whole ML lifecycle, including +experimentation, reproducibility, deployment, and a central model registry. + +The [MLflow adapter for CrateDB], available through the [mlflow-cratedb] package, +provides support to use CrateDB as a storage database for the [MLflow Tracking] +subsystem, which is about recording and querying experiments, across code, data, +config, and results. + +```{div} +:style: "clear: both" +``` + +:::{rubric} Learn +::: Tutorials and Notebooks about using [MLflow] together with CrateDB. ::::{info-card} @@ -109,6 +133,28 @@ prediction using NumPy, Salesforce Merlion, and Matplotlib. (pycaret)= ### PyCaret +:::{rubric} About +::: +```{div} +:style: "float: right; margin-left: 1em" +[![](https://github.com/crate/crate-clients-tools/assets/453543/b17a59e2-6801-4f53-892f-ff472491095f){w=180px}](https://pycaret.org/) +``` + +[PyCaret] is an open-source, low-code machine learning library for Python that +automates machine learning workflows. + +It is a high-level interface and AutoML wrapper on top of your loved machine learning +libraries like scikit-learn, xgboost, ray, lightgbm, and many more. PyCaret provides a +universal interface to utilize these libraries without needing to know the details +of the underlying model architectures and parameters. + +```{div} +:style: "clear: both" +``` + +:::{rubric} Learn +::: + Tutorials and Notebooks about using [PyCaret] together with CrateDB. ::::{info-card} @@ -154,9 +200,50 @@ How to train time series forecasting models using PyCaret and CrateDB. :::: +(iris-r)= +### R + +Use R with CrateDB. + +:::::{info-card} +::::{grid-item} +:columns: 9 +**Statistical analysis and visualization on huge datasets** + +Details about how to create a machine learning pipeline +using R and CrateDB. + +:::{toctree} +:maxdepth: 1 + +r +::: + +:::: +::::{grid-item} +:columns: 3 +{tags-primary}`Fundamentals` +:::: +::::: + + (scikit-learn)= ### scikit-learn +:::{rubric} About +::: +```{div} +:style: "float: right; margin-left: 1em" +[![](https://upload.wikimedia.org/wikipedia/commons/thumb/0/05/Scikit_learn_logo_small.svg/240px-Scikit_learn_logo_small.svg.png){w=180px}](https://scikit-learn.org/) + +[![](https://pandas.pydata.org/static/img/pandas.svg){w=180px}](https://pandas.pydata.org/) + +[![](https://jupyter.org/assets/logos/rectanglelogo-greytext-orangebody-greymoons.svg){w=180px}](https://jupyter.org/) +``` + +:::{rubric} Learn +::: + Use [scikit-learn] with CrateDB. ::::{info-card} @@ -283,7 +370,10 @@ tensorflow [Machine Learning and CrateDB: Getting Started With Jupyter]: https://cratedb.com/blog/machine-learning-cratedb-jupyter [Machine Learning and CrateDB: Experiment Design & Linear Regression]: https://cratedb.com/blog/machine-learning-and-cratedb-part-three-experiment-design-and-linear-regression [MLflow]: https://mlflow.org/ +[MLflow adapter for CrateDB]: https://github.com/crate/mlflow-cratedb [MLflow and CrateDB]: https://github.com/crate/cratedb-examples/tree/main/topic/machine-learning/mlflow +[mlflow-cratedb]: https://pypi.org/project/mlflow-cratedb/ +[MLflow Tracking]: https://mlflow.org/docs/latest/tracking.html [MLOps]: https://en.wikipedia.org/wiki/MLOps [pandas]: https://pandas.pydata.org/ [PyCaret]: https://www.pycaret.org diff --git a/docs/integrate/visualize/r.rst b/docs/domain/ml/r.rst similarity index 100% rename from docs/integrate/visualize/r.rst rename to docs/domain/ml/r.rst diff --git a/docs/integrate/apache-airflow/index.md b/docs/integrate/apache-airflow/index.md new file mode 100644 index 00000000..e8db6077 --- /dev/null +++ b/docs/integrate/apache-airflow/index.md @@ -0,0 +1,98 @@ +(airflow)= +(apache-airflow)= +(astronomer)= +# Apache Airflow / Astronomer + +:::{rubric} About +::: + +```{div} +:style: "float: right" +[![](https://19927462.fs1.hubspotusercontent-na1.net/hub/19927462/hubfs/Partner%20Logos/392x140/Apache-Airflow-Logo-392x140.png?width=784&height=280&name=Apache-Airflow-Logo-392x140.png){w=180px}](https://airflow.apache.org/) + +[![](https://logowik.com/content/uploads/images/astronomer2824.jpg){w=180px}](https://www.astronomer.io/) +``` +[Apache Airflow] is an open source software platform to programmatically author, +schedule, and monitor workflows, written in Python. +[Astronomer] offers managed Airflow services on the cloud of your choice, in +order to run Airflow with less overhead. + +Airflow has a modular architecture and uses a message queue to orchestrate an +arbitrary number of workers. Pipelines are defined in Python, allowing for +dynamic pipeline generation and on-demand, code-driven pipeline invocation. + +Pipeline parametrization is using the powerful Jinja templating engine. +To extend the system, you can define your own operators and extend libraries +to fit the level of abstraction that suits your environment. +```{div} +:style: "clear: both" +``` + +```{seealso} +[CrateDB and Apache Airflow] +``` + + +:::{dropdown} **Managed Airflow** + +```{div} +:style: "float: right" +[![](https://logowik.com/content/uploads/images/astronomer2824.jpg){w=180px}](https://www.astronomer.io/) +``` + +[Astro][Astronomer] is the best managed service in the market for teams on any step of their data +journey. Spend time where it counts. + +- Astro runs on the cloud of your choice. Astro manages Airflow and gives you all the + features you need to focus on what really matters – your data. All while connecting + securely to any service in your network. +- Create Airflow environments with a click of a button. +- Protect production DAGs with easy Airflow upgrades and custom high-availability configs. +- Get visibility into what’s running with analytics views and easy interfaces for logs + and alerts. Across environments. +- Take down tech-debt and learn how to drive Airflow best practices from the experts + behind the project. Get world-class support, fast-tracked bug fixes, and same-day + access to new Airflow versions. + +```{div} +:style: "clear: both" +``` +::: + + +:::{rubric} Learn +::: + +A set of starter tutorials. + +- [Automating the import of Parquet files with Apache Airflow] +- [Updating stock market data automatically with CrateDB and Apache Airflow] +- [Automating stock data collection and storage with CrateDB and Apache Airflow] + +A set of elaborated tutorials, including blueprint implementations. + +- [Automating export of CrateDB data to S3 using Apache Airflow] +- [Implementing a data retention policy in CrateDB using Apache Airflow] +- [CrateDB and Apache Airflow: Building a data ingestion pipeline] +- [Building a hot and cold storage data retention policy in CrateDB with Apache Airflow] + +Tutorials and resources about configuring the managed variants, Astro and CrateDB Cloud. + +- [ETL with Astro and CrateDB Cloud in 30min - fully up in the cloud] +- [ETL pipeline using Apache Airflow with CrateDB (Source)] +- [Run an ETL pipeline with CrateDB and data quality checks] + + + +[Apache Airflow]: https://airflow.apache.org/ +[Automating export of CrateDB data to S3 using Apache Airflow]: https://community.cratedb.com/t/cratedb-and-apache-airflow-automating-data-export-to-s3/901 +[Automating stock data collection and storage with CrateDB and Apache Airflow]: https://community.cratedb.com/t/automating-stock-data-collection-and-storage-with-cratedb-and-apache-airflow/990 +[Automating the import of Parquet files with Apache Airflow]: https://community.cratedb.com/t/automating-the-import-of-parquet-files-with-apache-airflow/1247 +[Building a hot and cold storage data retention policy in CrateDB with Apache Airflow]: https://community.cratedb.com/t/cratedb-and-apache-airflow-building-a-hot-cold-storage-data-retention-policy/934 +[CrateDB and Apache Airflow]: https://cratedb.com/integrations/cratedb-and-apache-airflow +[CrateDB and Apache Airflow: Building a data ingestion pipeline]: https://community.cratedb.com/t/cratedb-and-apache-airflow-building-a-data-ingestion-pipeline/926 +[ETL pipeline using Apache Airflow with CrateDB (Source)]: https://github.com/astronomer/astro-cratedb-blogpost +[ETL with Astro and CrateDB Cloud in 30min - fully up in the cloud]: https://www.astronomer.io/blog/run-etlelt-with-airflow-and-cratedb/ +[Implementing a data retention policy in CrateDB using Apache Airflow]: https://community.cratedb.com/t/implementing-a-data-retention-policy-in-cratedb-using-apache-airflow/913 +[Run an ETL pipeline with CrateDB and data quality checks]: https://registry.astronomer.io/dags/etl_pipeline/ +[Updating stock market data automatically with CrateDB and Apache Airflow]: https://community.cratedb.com/t/updating-stock-market-data-automatically-with-cratedb-and-apache-airflow/1304 diff --git a/docs/integrate/apache-flink/index.md b/docs/integrate/apache-flink/index.md index f3d73b88..34744e6e 100644 --- a/docs/integrate/apache-flink/index.md +++ b/docs/integrate/apache-flink/index.md @@ -1,8 +1,10 @@ (apache-flink)= (flink)= - # Apache Flink +:::{include} /_include/links.md +::: + ```{div} :style: "float: right" [![](https://flink.apache.org/flink-header-logo.svg){w=180px}](https://flink.apache.org/) @@ -127,9 +129,10 @@ JdbcSink.sink( :::{rubric} Development ::: +:::{div} - [Executable stack with Apache Kafka, Apache Flink, and CrateDB] (Java) - [Streaming data with Apache Kafka, Apache Flink and CrateDB] (Python) - +::: :::{rubric} Webinars ::: @@ -200,9 +203,7 @@ The webinar includes a live demo of Apache Flink with CrateDB as source or sink. [Community Day: Stream processing with Apache Flink and CrateDB]: https://cratedb.com/event/cratedb-community-day-2023 [Confluent Streaming Data Pipelines]: https://www.confluent.io/streaming-data-pipelines/ [CrateDB Community Day 2nd Edition: Summary and Highlights]: https://cratedb.com/blog/cratedb-community-day-2nd-edition-summary-and-highlights -[Executable stack with Apache Kafka, Apache Flink, and CrateDB]: https://github.com/crate/cratedb-examples/tree/main/framework/flink/kafka-jdbcsink-java [Flink managed by Confluent]: https://www.datanami.com/2023/05/17/confluents-new-cloud-capabilities-address-data-streaming-hurdles/ [Immerok Cloud]: https://web.archive.org/web/20230602085618/https://www.immerok.io/product [JdbcSink]: https://nightlies.apache.org/flink/flink-docs-release-1.20/docs/connectors/datastream/jdbc/ -[PostgreSQL JDBC Driver]: https://jdbc.postgresql.org/ [Streaming data with Apache Kafka, Apache Flink and CrateDB]: https://github.com/crate/cratedb-examples/tree/main/framework/flink/kafka-jdbcsink-python diff --git a/docs/integrate/apache-hop/index.md b/docs/integrate/apache-hop/index.md new file mode 100644 index 00000000..446699b8 --- /dev/null +++ b/docs/integrate/apache-hop/index.md @@ -0,0 +1,44 @@ +(apache-hop)= +# Apache Hop + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 0.3em" +[![](https://hop.apache.org/img/hop-logo.svg){w=180px}](https://hop.apache.org/) +``` + +[Apache Hop] aims to be a modern, open source data integration platform that is +easy to use, fast, and flexible. It facilitates all aspects of data and metadata +orchestration. + +- **Visual development** enables developers to be more productive than they can + be through code. + +- **Workflows and pipelines** can be designed in the Hop Gui and run on the Hop + native engine (local or remote), or on Spark, Flink, Google Dataflow or AWS EMR + through Beam. _Design once, run anywhere._ + +- **Lifecycle Management** enables developers and administrators to switch between + projects, environments and purposes without leaving your train of thought. + +![](https://github.com/crate/crate-clients-tools/assets/453543/da6baf11-8430-4a0f-b2df-55717ce02802){h=120px} +![](https://github.com/crate/crate-clients-tools/assets/453543/60cfc82a-db0a-49f1-8e26-a37b774b3614){h=120px} +![](https://github.com/crate/crate-clients-tools/assets/453543/2bd59577-b664-45ae-a71e-36a130d36739){h=120px} + + +:::{rubric} Learn +::: + + +- [Using Apache Hop with CrateDB] +- [CrateDB dialect for Apache Hop] +- [CrateDB Apache Hop Bulk Loader transform] + + + +[Apache Hop]: https://hop.apache.org/ +[CrateDB Apache Hop Bulk Loader transform]: https://hop.apache.org/manual/latest/pipeline/transforms/cratedb-bulkloader.html +[CrateDB dialect for Apache Hop]: https://hop.apache.org/manual/latest/database/databases/cratedb.html +[Using Apache Hop with CrateDB]: https://community.cratedb.com/t/using-apache-hop-with-cratedb/1754 diff --git a/docs/integrate/apache-kafka/index.md b/docs/integrate/apache-kafka/index.md new file mode 100644 index 00000000..c2a4b254 --- /dev/null +++ b/docs/integrate/apache-kafka/index.md @@ -0,0 +1,61 @@ +(apache-kafka)= +# Apache Kafka + +:::{include} /_include/links.md +::: + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 2em" +[![](https://kafka.apache.org/logos/kafka_logo--simple.png){w=180px}](https://kafka.apache.org/) +``` + +[Apache Kafka] is an open-source distributed event streaming platform used by +thousands of companies for high-performance data pipelines, streaming analytics, +data integration, and mission-critical applications. + + +:::{dropdown} **Managed Kafka** +Several companies provide managed Kafka services (see the [overview of managed Kafka offerings] +for a more complete list). + +- [Aiven for Apache Kafka] +- [Amazon Managed Streaming for Apache Kafka (MSK)] +- [Apache Kafka on Azure] +- [Azure Event Hubs for Apache Kafka] +- [Confluent Cloud] +- [DoubleCloud Managed Service for Apache Kafka] +::: + + +:::{rubric} Learn +::: + +:::{div} +- {ref}`kafka-connect` +- [Replicating data to CrateDB with Debezium and Kafka] +- [Executable stack with Apache Kafka, Apache Flink, and CrateDB] +::: + +```{toctree} +:hidden: + +kafka-connect +``` + +```{seealso} +[CrateDB and Apache Kafka] +``` + + +[Aiven for Apache Kafka]: https://aiven.io/kafka +[Amazon Managed Streaming for Apache Kafka (MSK)]: https://aws.amazon.com/msk/ +[Apache Kafka]: https://kafka.apache.org/ +[Apache Kafka on Azure]: https://azuremarketplace.microsoft.com/marketplace/consulting-services/canonical.0001-com-ubuntu-managed-kafka +[Azure Event Hubs for Apache Kafka]: https://learn.microsoft.com/en-us/azure/event-hubs/azure-event-hubs-kafka-overview +[Confluent Cloud]: https://www.confluent.io/confluent-cloud/ +[CrateDB and Apache Kafka]: https://cratedb.com/integrations/cratedb-and-kafka +[DoubleCloud Managed Service for Apache Kafka]: https://double.cloud/services/managed-kafka/ +[overview of managed Kafka offerings]: https://keen.io/blog/managed-apache-kafka-vs-diy/ diff --git a/docs/integrate/etl/kafka-connect.md b/docs/integrate/apache-kafka/kafka-connect.md similarity index 100% rename from docs/integrate/etl/kafka-connect.md rename to docs/integrate/apache-kafka/kafka-connect.md diff --git a/docs/integrate/apache-nifi/index.md b/docs/integrate/apache-nifi/index.md new file mode 100644 index 00000000..389c417e --- /dev/null +++ b/docs/integrate/apache-nifi/index.md @@ -0,0 +1,40 @@ +(apache-nifi)= +# Apache NiFi + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 2em" +[![](https://nifi.apache.org/images/apache-nifi-drop-logo.svg){w=180px}](https://nifi.apache.org/) +``` + +[Apache NiFi] is an easy to use, powerful, and reliable system to process and +distribute data. NiFi automates cybersecurity, observability, event streams, +and generative AI data pipelines and distribution for thousands of companies +worldwide across every industry. + +- **Data provenance tracking**: Complete lineage of information from beginning to end. +- **Extensive configuration**: Loss-tolerant and guaranteed delivery, Low latency and + high throughput, Dynamic prioritization, Runtime modification of flow configuration, + Back pressure control. +- **Browser-based user interface**: Seamless experience for design, control, feedback, + and monitoring +- **Secure communication**: HTTPS with configurable authentication strategies, + Multi-tenant authorization and policy management, Standard protocols for encrypted + communication including TLS and SSH. + +![](https://github.com/crate/crate-clients-tools/assets/453543/ba6973dd-2eec-4f1f-a436-96aac7eb9892){h=120px} +![](https://github.com/crate/crate-clients-tools/assets/453543/7fd4d2e7-98bc-44ee-b441-e1835016ab4d){h=120px} +![](https://github.com/crate/crate-clients-tools/assets/453543/ccfa4ac7-0d60-432f-b952-2b50789cd325){h=120px} + + +:::{rubric} Learn +::: + +- [Connecting to CrateDB from Apache NiFi] + + + +[Apache NiFi]: https://nifi.apache.org/ +[Connecting to CrateDB from Apache NiFi]: https://community.cratedb.com/t/connecting-to-cratedb-from-apache-nifi/647 diff --git a/docs/integrate/bi/index.md b/docs/integrate/bi/index.md index 5d7eb475..3ec9b23c 100644 --- a/docs/integrate/bi/index.md +++ b/docs/integrate/bi/index.md @@ -8,32 +8,27 @@ Integrations of CrateDB with other tools, specifically related to business analytics and intelligence software. -(powerbi)= ## PowerBI ```{toctree} :maxdepth: 1 -powerbi-desktop -powerbi-gateway +../powerbi/index ``` -(rill)= ## Rill -- [Introducing Rill and BI as Code with CrateDB Cloud] +```{toctree} +:maxdepth: 1 +../rill/index +``` -(tableau)= -## Tableau -- [Connecting to CrateDB from Tableau with JDBC] and [Using CrateDB with Tableau] - will guide you through the process of setting it Tableau correctly with CrateDB. -- The [CrateDB Tableau Connector] unlocks advanced SQL functionality and - resolves compatibility issues beyond standard usage. +## Tableau +```{toctree} +:maxdepth: 1 -[Connecting to CrateDB from Tableau with JDBC]: https://cratedb.com/blog/connecting-to-cratedb-from-tableau-with-jdbc -[CrateDB Tableau Connector]: https://github.com/crate/cratedb-tableau-connector -[Introducing Rill and BI as Code with CrateDB Cloud]: https://community.cratedb.com/t/introducing-rill-and-bi-as-code-with-cratedb-cloud/1718 -[Using CrateDB with Tableau]: https://community.cratedb.com/t/using-cratedb-with-tableau/1192 +../tableau/index +``` diff --git a/docs/integrate/cdc/index.md b/docs/integrate/cdc/index.md index d4cba7c1..249c426c 100644 --- a/docs/integrate/cdc/index.md +++ b/docs/integrate/cdc/index.md @@ -54,18 +54,12 @@ Kinesis Stream, and consume that from an adapter to a consolidation database. - See: [](#cdc-dynamodb) ::: -(debezium)= ## Debezium -Debezium is an open source distributed platform for change data capture (CDC). -It is built on top of Apache Kafka, a distributed streaming platform. It allows -to capture changes on a source database system, mostly OLTP, and replicate them -to another system, mostly to run OLAP workloads on the data. -Debezium provides connectors for MySQL/MariaDB, MongoDB, PostgreSQL, Oracle, -SQL Server, IBM DB2, Cassandra, Vitess, Spanner, JDBC, and Informix. -:::{div} -- Tutorial: [Replicating data to CrateDB with Debezium and Kafka] -- Webinar: [How to replicate data from other databases to CrateDB with Debezium and Kafka] +:::{toctree} +:maxdepth: 1 + +../debezium/index ::: (cdc-dynamodb)= @@ -107,5 +101,4 @@ lives. -[How to replicate data from other databases to CrateDB with Debezium and Kafka]: https://cratedb.com/resources/webinars/lp-wb-debezium-kafka [StreamSets Data Collector]: https://www.softwareag.com/en_corporate/platform/integration-apis/data-collector-engine.html diff --git a/docs/integrate/cluvio/index.md b/docs/integrate/cluvio/index.md new file mode 100644 index 00000000..52385cba --- /dev/null +++ b/docs/integrate/cluvio/index.md @@ -0,0 +1,32 @@ +(cluvio)= +# Cluvio + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 1em" +[![cluvio-logo-full_color-on_dark.svg ](https://github.com/crate/crate-clients-tools/assets/453543/cac142ef-412a-4a67-a63f-bf9d1ce92c84){w=180px}](https://www.cluvio.com/) +``` + +[Cluvio] is a programmable and interactive dashboarding platform — your analytics +cockpit. Run queries, filter your results, choose the most vivid way to display them, +and share them with your colleagues and partners without efforts. + +Cluvio dashboards are interactive, so you can easily change aggregation, select a +specific timerange or filter dashboards by any individual attribute of your data. + +Use SQL and R to analyze your data and create beautiful, interactive dashboards for +your entire company in few minutes. + +![custom-filters.png](https://github.com/crate/crate-clients-tools/assets/453543/49ca6a35-239e-4915-951c-db6649fd35a4){h=200px} +![report-creator.png](https://github.com/crate/crate-clients-tools/assets/453543/844a5ffd-0b92-4c77-8cdd-0b5cc5b392b1){h=200px} + +:::{rubric} Learn +::: + +- [Data Analysis with Cluvio and CrateDB] + + +[Cluvio]: https://www.cluvio.com/ +[Data Analysis with Cluvio and CrateDB]: https://community.cratedb.com/t/data-analysis-with-cluvio-and-cratedb/1571 diff --git a/docs/integrate/debezium/index.md b/docs/integrate/debezium/index.md new file mode 100644 index 00000000..09355493 --- /dev/null +++ b/docs/integrate/debezium/index.md @@ -0,0 +1,37 @@ +(debezium)= +# Debezium + +:::{include} /_include/links.md +::: + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 2em" +[![](https://debezium.io/assets/images/color_white_debezium_type_600px.svg){w=180px}](https://debezium.io/) +``` + +[Debezium] is an open source distributed platform for change data capture. After +pointing it at your databases, you can subscribe to the event stream of +all database update operations. + +Debezium is an open source distributed platform for change data capture (CDC). +It is built on top of Apache Kafka, a distributed streaming platform. It allows +to capture changes on a source database system, mostly OLTP, and replicate them +to another system, mostly to run OLAP workloads on the data. + +Debezium provides connectors for MySQL/MariaDB, MongoDB, PostgreSQL, Oracle, +SQL Server, IBM DB2, Cassandra, Vitess, Spanner, JDBC, and Informix. + +:::{rubric} Learn +::: + +:::{div} +- Tutorial: [Replicating data to CrateDB with Debezium and Kafka] +- Webinar: [How to replicate data from other databases to CrateDB with Debezium and Kafka] +::: + + +[Debezium]: https://debezium.io/ +[How to replicate data from other databases to CrateDB with Debezium and Kafka]: https://cratedb.com/resources/webinars/lp-wb-debezium-kafka diff --git a/docs/integrate/etl/index.md b/docs/integrate/etl/index.md index 93d070b0..6999c7fa 100644 --- a/docs/integrate/etl/index.md +++ b/docs/integrate/etl/index.md @@ -17,29 +17,13 @@ to use them optimally. Please also have a look at support for [](#cdc) solutions. -(airflow)= -(apache-airflow)= ## Apache Airflow / Astronomer -A set of starter tutorials. - -- [Automating the import of Parquet files with Apache Airflow] -- [Updating stock market data automatically with CrateDB and Apache Airflow] -- [Automating stock data collection and storage with CrateDB and Apache Airflow] - -A set of elaborated tutorials, including blueprint implementations. - -- [Automating export of CrateDB data to S3 using Apache Airflow] -- [Implementing a data retention policy in CrateDB using Apache Airflow] -- [CrateDB and Apache Airflow: Building a data ingestion pipeline] -- [Building a hot and cold storage data retention policy in CrateDB with Apache Airflow] - -Tutorials and resources about configuring the managed variants, Astro and CrateDB Cloud. - -- [ETL with Astro and CrateDB Cloud in 30min - fully up in the cloud] -- [ETL pipeline using Apache Airflow with CrateDB (Source)] -- [Run an ETL pipeline with CrateDB and data quality checks] +:::{toctree} +:maxdepth: 1 +../apache-airflow/index +::: ## Apache Flink @@ -49,13 +33,13 @@ Tutorials and resources about configuring the managed variants, Astro and CrateD ../apache-flink/index ::: - ## Apache Hop -- [Using Apache Hop with CrateDB] -- [CrateDB dialect for Apache Hop] -- [CrateDB Apache Hop Bulk Loader transform] +:::{toctree} +:maxdepth: 1 +../apache-hop/index +::: ## Apache Iceberg / RisingWave :::{div} @@ -68,24 +52,21 @@ Tutorials and resources about configuring the managed variants, Astro and CrateD iceberg-risingwave ``` - ## Apache Kafka -:::{div} -- {ref}`kafka-connect` -- [Replicating data to CrateDB with Debezium and Kafka] -- [Executable stack with Apache Kafka, Apache Flink, and CrateDB] -::: -```{toctree} -:hidden: +:::{toctree} +:maxdepth: 1 -kafka-connect -``` +../apache-kafka/index +::: ## Apache NiFi -- [Connecting to CrateDB from Apache NiFi] +:::{toctree} +:maxdepth: 1 +../apache-nifi/index +::: ## AWS DMS @@ -168,10 +149,21 @@ azure-functions ../influxdb/index ``` - ## Kestra -- [Setting up data pipelines with CrateDB and Kestra] +:::{toctree} +:maxdepth: 1 + +../kestra/index +::: + +## Meltano + +:::{toctree} +:maxdepth: 1 + +../meltano/index +::: ## MongoDB :::{div} @@ -196,12 +188,13 @@ mongodb mysql ``` - ## Node-RED -- [Ingesting MQTT messages into CrateDB using Node-RED] -- [Automating recurrent CrateDB queries using Node-RED] +:::{toctree} +:maxdepth: 1 +../node-red/index +::: ## RisingWave @@ -211,22 +204,13 @@ mysql ../risingwave/index ::: - -## Singer / Meltano - -- [meltano-target-cratedb] -- [meltano-tap-cratedb] -- [Examples about working with CrateDB and Meltano] - -🚧 _Please note these adapters are a work in progress._ 🚧 - - ## SQL Server Integration Services -A demo project which uses SSIS and ODBC to read and write data from CrateDB: - -- [Using SQL Server Integration Services with CrateDB] +:::{toctree} +:maxdepth: 1 +../sql-server/index +::: ## StreamSets @@ -237,28 +221,3 @@ A demo project which uses SSIS and ODBC to read and write data from CrateDB: streamsets ``` - - - -[Automating recurrent CrateDB queries using Node-RED]: https://community.cratedb.com/t/automating-recurrent-cratedb-queries/788 -[Automating export of CrateDB data to S3 using Apache Airflow]: https://community.cratedb.com/t/cratedb-and-apache-airflow-automating-data-export-to-s3/901 -[Automating stock data collection and storage with CrateDB and Apache Airflow]: https://community.cratedb.com/t/automating-stock-data-collection-and-storage-with-cratedb-and-apache-airflow/990 -[Automating the import of Parquet files with Apache Airflow]: https://community.cratedb.com/t/automating-the-import-of-parquet-files-with-apache-airflow/1247 -[Building a hot and cold storage data retention policy in CrateDB with Apache Airflow]: https://community.cratedb.com/t/cratedb-and-apache-airflow-building-a-hot-cold-storage-data-retention-policy/934 -[Connecting to CrateDB from Apache NiFi]: https://community.cratedb.com/t/connecting-to-cratedb-from-apache-nifi/647 -[CrateDB and Apache Airflow: Building a data ingestion pipeline]: https://community.cratedb.com/t/cratedb-and-apache-airflow-building-a-data-ingestion-pipeline/926 -[CrateDB Apache Hop Bulk Loader transform]: https://hop.apache.org/manual/latest/pipeline/transforms/cratedb-bulkloader.html -[CrateDB dialect for Apache Hop]: https://hop.apache.org/manual/latest/database/databases/cratedb.html -[ETL pipeline using Apache Airflow with CrateDB (Source)]: https://github.com/astronomer/astro-cratedb-blogpost -[ETL with Astro and CrateDB Cloud in 30min - fully up in the cloud]: https://www.astronomer.io/blog/run-etlelt-with-airflow-and-cratedb/ -[Examples about working with CrateDB and Meltano]: https://github.com/crate/cratedb-examples/tree/amo/meltano/framework/singer-meltano -[Executable stack with Apache Kafka, Apache Flink, and CrateDB]: https://github.com/crate/cratedb-examples/tree/main/framework/flink/kafka-jdbcsink-java -[Implementing a data retention policy in CrateDB using Apache Airflow]: https://community.cratedb.com/t/implementing-a-data-retention-policy-in-cratedb-using-apache-airflow/913 -[Ingesting MQTT messages into CrateDB using Node-RED]: https://community.cratedb.com/t/ingesting-mqtt-messages-into-cratedb-using-node-red/803 -[meltano-tap-cratedb]: https://github.com/crate-workbench/meltano-tap-cratedb -[meltano-target-cratedb]: https://github.com/crate-workbench/meltano-target-cratedb -[Run an ETL pipeline with CrateDB and data quality checks]: https://registry.astronomer.io/dags/etl_pipeline/ -[Setting up data pipelines with CrateDB and Kestra]: https://community.cratedb.com/t/setting-up-data-pipelines-with-cratedb-and-kestra-io/1400 -[Updating stock market data automatically with CrateDB and Apache Airflow]: https://community.cratedb.com/t/updating-stock-market-data-automatically-with-cratedb-and-apache-airflow/1304 -[Using Apache Hop with CrateDB]: https://community.cratedb.com/t/using-apache-hop-with-cratedb/1754 -[Using SQL Server Integration Services with CrateDB]: https://github.com/crate/cratedb-examples/tree/main/application/microsoft-ssis diff --git a/docs/integrate/explo/index.md b/docs/integrate/explo/index.md new file mode 100644 index 00000000..173ef2cd --- /dev/null +++ b/docs/integrate/explo/index.md @@ -0,0 +1,36 @@ +(explo)= +# Explo + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 1em" +[![](https://uploads-ssl.webflow.com/62f681c18d50329187681754/62f681c18d5032d0bd681785_logo.svg){w=180px}](https://www.explo.co/) +``` + +[Explo] is a software platform for personalized and real-time customer facing +analytics. Organizations use Explo’s platform services "Explore", "Host", "Report +builder", and "Email", to activate and share data with their customers. + +[Explo Explore] integrates directly into your web portal or application and provides +your customers with a complete self-service data toolkit, which can also be used to +run white-labeled data portals. + +```{div} +:style: "clear: both" +``` + +![](https://cratedb.com/hs-fs/hubfs/Screenshot%202023-07-21%20at%2013.17.45.png?width=2948&height=2312&name=Screenshot%202023-07-21%20at%2013.17.45.png){h=200px} +![](https://cratedb.com/hs-fs/hubfs/Screenshot%202023-07-21%20at%2013.24.01.png?width=2932&height=1716&name=Screenshot%202023-07-21%20at%2013.24.01.png){h=200px} + + +:::{rubric} Learn +::: + +- [Introduction to Time Series Visualization in CrateDB and Explo] + + +[Explo]: https://www.explo.co/ +[Explo Explore]: https://www.explo.co/products/explore +[Introduction to Time Series Visualization in CrateDB and Explo]: https://cratedb.com/blog/introduction-to-time-series-visualization-in-cratedb-and-explo diff --git a/docs/integrate/framework.md b/docs/integrate/framework.md new file mode 100644 index 00000000..d4d3e4f1 --- /dev/null +++ b/docs/integrate/framework.md @@ -0,0 +1,33 @@ +(framework)= +# Using Programming Frameworks with CrateDB + +Using application programming frameworks together with CrateDB. + +Many of them are built on top of the Python programming language, making it easy +to use the Python libraries that you know and love. + +:::::{grid} 1 2 2 2 +:margin: 4 4 0 0 +:padding: 0 +:gutter: 2 + +::::{grid-item-card} {material-outlined}`lightbulb;2em` Examples + +A few quick examples about how to use selected frameworks together with CrateDB. + +- [cratedb-examples/framework](https://github.com/crate/cratedb-examples/tree/main/framework) ++++ +{tag-info}`dbt` {tag-info}`dlt` {tag-info}`Flink` {tag-info}`Gradio` {tag-info}`Graphene` {tag-info}`MCP` {tag-info}`records` {tag-info}`Streamlit` +:::: + +::::: + + +:::{toctree} +:maxdepth: 1 + +plotly/index +gradio/index +pyviz/index +streamlit/index +::: diff --git a/docs/integrate/gradio/index.md b/docs/integrate/gradio/index.md new file mode 100644 index 00000000..7f606554 --- /dev/null +++ b/docs/integrate/gradio/index.md @@ -0,0 +1,27 @@ +(gradio)= +# Gradio + +```{div} +:style: "float: right; margin-left: 0.5em" +[![](https://raw.githubusercontent.com/gradio-app/gradio/main/readme_files/gradio.svg){w=180px}](https://www.gradio.app/) +``` + +[Gradio] is an open source programming framework for quickly creating and sharing +machine learning model demo applications, written in Python. + +- Creating a user interface only requires adding a couple lines of code to your project. +- It does not require any experience with HTML/JS/CSS, or web hosting. +- Gradio can be embedded in Python notebooks, or presented as a web application. +- Once you've created an interface, you can permanently host it on [Hugging Face]. + +```{div} +:style: "clear: both" +``` + + + +_How to Build Machine Learning APIs Using Gradio._ + + +[Gradio]: https://www.gradio.app/ +[Hugging Face]: https://en.wikipedia.org/wiki/Hugging_Face diff --git a/docs/integrate/grafana/index.md b/docs/integrate/grafana/index.md new file mode 100644 index 00000000..bf34e025 --- /dev/null +++ b/docs/integrate/grafana/index.md @@ -0,0 +1,65 @@ +(grafana)= +# Grafana + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 1em" +[![](https://cratedb.com/hs-fs/hubfs/Imported_Blog_Media/grafana-logo-1-520x126.png?width=1040&height=252&name=grafana-logo-1-520x126.png){w=180px}](https://grafana.com/grafana/) +``` + +[Grafana OSS] is the leading open-source metrics visualization tool that helps you +build real-time dashboards, graphs, and many other sorts of data visualizations. +[Grafana Cloud] is a fully-managed service offered by [Grafana Labs]. + +Grafana complements CrateDB in monitoring and visualizing large volumes of machine +data in real-time. + +Connecting to a CrateDB cluster will use the Grafana PostgreSQL data source adapter. +The following tutorials outline how to configure Grafana to connect to CrateDB, and +how to run a database query. + +![image](https://github.com/crate/cratedb-guide/raw/a9c8c03384/docs/_assets/img/integrations/grafana/grafana-connection.png){h=200px} +![image](https://github.com/crate/cratedb-guide/raw/a9c8c03384/docs/_assets/img/integrations/grafana/grafana-panel1.png){h=200px} + + +:::{dropdown} **Managed Grafana** +```{div} +:style: "float: right" +[![](https://cratedb.com/hs-fs/hubfs/Imported_Blog_Media/grafana-logo-1-520x126.png?width=1040&height=252&name=grafana-logo-1-520x126.png){w=180px}](https://grafana.com/grafana/) +``` + +Get Grafana fully managed with [Grafana Cloud]. + +- Offered as a fully managed service, Grafana Cloud is the fastest way to adopt + Grafana and includes a scalable, managed backend for metrics, logs, and traces. +- Managed and administered by Grafana Labs with free and paid options for + individuals, teams, and large enterprises. +- Includes a robust free tier with access to 10k metrics, 50GB logs, 50GB traces, + 50GB profiles, and 500VUh of k6 testing for 3 users. + +```{div} +:style: "clear: both" +``` +::: + +:::{rubric} Learn +::: + +:::{toctree} +:maxdepth: 1 + +learn +::: + + +```{seealso} +[CrateDB and Grafana] +``` + + +[CrateDB and Grafana]: https://cratedb.com/integrations/cratedb-and-grafana +[Grafana Cloud]: https://grafana.com/grafana/ +[Grafana Labs]: https://grafana.com/about/team/ +[Grafana OSS]: https://grafana.com/oss/grafana/ diff --git a/docs/integrate/visualize/grafana.rst b/docs/integrate/grafana/learn.rst similarity index 100% rename from docs/integrate/visualize/grafana.rst rename to docs/integrate/grafana/learn.rst diff --git a/docs/integrate/index.md b/docs/integrate/index.md index 7b5179c9..89370f83 100644 --- a/docs/integrate/index.md +++ b/docs/integrate/index.md @@ -13,11 +13,12 @@ optimally. :maxdepth: 2 ide +Frameworks etl/index cdc/index mcp/index -metrics/index -visualize/index +Monitoring and Metrics +Data Visualization bi/index lineage/index testing diff --git a/docs/integrate/kestra/index.md b/docs/integrate/kestra/index.md new file mode 100644 index 00000000..f377d28a --- /dev/null +++ b/docs/integrate/kestra/index.md @@ -0,0 +1,43 @@ +(kestra)= +# Kestra + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 2em" +[![](https://kestra.io/logo.svg){w=180px}](https://kestra.io/) +``` + +[Kestra] is an open source workflow automation and orchestration toolkit with a rich +plugin ecosystem. It enables users to automate and manage complex workflows in a +streamlined and efficient manner, defining them both declaratively, or imperatively +using any scripting language like Python, Bash, or JavaScript. + +Kestra comes with a user-friendly web-based interface including a live-updating DAG +view, allowing users to create, modify, and manage scheduled and event-driven flows +without the need for any coding skills. + +Plugins are at the core of Kestra's extensibility. Many plugins are available from +the Kestra core team, and creating your own is easy. With plugins, you can add new +functionality to Kestra. + +![](https://kestra.io/landing/features/declarative1.svg){h=120px} +![](https://kestra.io/landing/features/flowable.svg){h=120px} +![](https://kestra.io/landing/features/monitor.svg){h=120px} + +:::{rubric} Learn +::: + +- [Setting up data pipelines with CrateDB and Kestra] + + +```{seealso} +[CrateDB and Kestra] +``` + + + +[CrateDB and Kestra]: https://cratedb.com/integrations/cratedb-and-kestra +[Kestra]: https://kestra.io/ +[Setting up data pipelines with CrateDB and Kestra]: https://community.cratedb.com/t/setting-up-data-pipelines-with-cratedb-and-kestra-io/1400 diff --git a/docs/integrate/meltano/index.md b/docs/integrate/meltano/index.md new file mode 100644 index 00000000..001daeea --- /dev/null +++ b/docs/integrate/meltano/index.md @@ -0,0 +1,45 @@ +(meltano)= +# Meltano + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 0.3em" +[![](https://www.singer.io/img/singer_logo_full_black.svg){w=180px}](https://www.singer.io/) + +[![](https://github.com/crate/crate-clients-tools/assets/453543/0c01e995-d7c2-4a4d-8e90-c6697fe2a85d){w=180px}](https://meltano.com/) +``` + +[Meltano] is a declarative code-first data integration engine adhering to the Singer +specification. +[Singer] is a composable open source ETL framework and specification, including +powerful data extraction and consolidation elements. + +[Meltano Hub] is the single source of truth to find any Meltano plugins as well +as Singer taps and targets. + +```{div} +:style: "clear: both" +``` + +:::{rubric} Learn +::: + +- [Examples about working with CrateDB and Meltano] + +:::{rubric} Resources +::: + +- [meltano-tap-cratedb] +- [meltano-target-cratedb] + +🚧 _Please note these adapters are a work in progress._ 🚧 + + +[Examples about working with CrateDB and Meltano]: https://github.com/crate/cratedb-examples/tree/amo/meltano/framework/singer-meltano +[Meltano]: https://meltano.com/ +[Meltano Hub]: https://hub.meltano.com/ +[meltano-tap-cratedb]: https://github.com/crate-workbench/meltano-tap-cratedb +[meltano-target-cratedb]: https://github.com/crate-workbench/meltano-target-cratedb +[Singer]: https://www.singer.io/ diff --git a/docs/integrate/metabase/index.md b/docs/integrate/metabase/index.md new file mode 100644 index 00000000..90ca0641 --- /dev/null +++ b/docs/integrate/metabase/index.md @@ -0,0 +1,65 @@ +# Metabase + +```{div} +:style: "float: right; margin-left: 1em" +[![](https://www.metabase.com/images/logo.svg){w=180px}](https://www.metabase.com/cloud/) +``` + +[Metabase] is the ultimate data analysis and visualization tool that unlocks the full +potential of your data. Built for data and made for everyone, Metabase can be leveraged +with no SQL required. + +Fast analytics with the friendly UX and integrated tooling to let your company explore +data on their own. + +![image](https://github.com/crate/cratedb-guide/raw/a9c8c03384/docs/_assets/img/integrations/metabase/metabase-question.png){h=140px} +![image](https://github.com/crate/cratedb-guide/raw/a9c8c03384/docs/_assets/img/integrations/metabase/metabase-dashboard.png){h=140px} + + +:::{dropdown} **Managed Metabase** +```{div} +:style: "float: right" +[![](https://www.metabase.com/images/logo.svg){w=180px}](https://www.metabase.com/) +``` + +With [Metabase Cloud], you will get a fast, reliable, and secure deployment +with none of the work or hidden costs that come with self-hosting. + +- **Save the time** needed for setup and maintenance of the platform, focusing only on the insights we can get from our data. +- **Trustworthy, production-grade deployment** by people who do this stuff for a living. + With the infrastructure, specialists, and thousands of Metabases in our cloud, we've put a lot of thought and resources into optimizing hosting. +- **Upgrades:** Automatically upgrade to the current version, so you're always getting the latest and greatest of Metabase. +- **Backups:** The way they should be: there when you need them, out of sight and out of mind when you don't. +- **SMTP server:** Even your alerts and dashboard subscriptions covered with a preconfigured and managed SMTP server. + + +```{div} +:style: "clear: both" +``` +::: + + +:::{rubric} Learn +::: + +- {ref}`integrations-metabase` +- [Real-time data analytics with Metabase and CrateDB] +- https://github.com/paoliniluis/metabase-cratedb + +:::{toctree} +:maxdepth: 1 +:hidden: + +learn +::: + +```{seealso} +[CrateDB and Metabase] +``` + + +[CrateDB and Metabase]: https://cratedb.com/integrations/cratedb-and-metabase +[Metabase]: https://www.metabase.com/ +[Metabase Cloud]: https://www.metabase.com/cloud/ +[Real-time data analytics with Metabase and CrateDB]: https://www.metabase.com/community_posts/real-time-data-analytics-with-metabase-and-cratedb +[Using Metabase with CrateDB Cloud]: #integrations-metabase diff --git a/docs/integrate/visualize/metabase.rst b/docs/integrate/metabase/learn.rst similarity index 100% rename from docs/integrate/visualize/metabase.rst rename to docs/integrate/metabase/learn.rst diff --git a/docs/integrate/metrics/index.md b/docs/integrate/metrics/index.md index ecf5a538..9549e106 100644 --- a/docs/integrate/metrics/index.md +++ b/docs/integrate/metrics/index.md @@ -1,23 +1,27 @@ +(metrics)= (integrate-metrics)= - -# System Metrics +# Monitoring and Metrics with CrateDB CrateDB integrations with metrics collection agents, brokers, and stores. - This documentation section lists applications and daemons which can -be used together with CrateDB, and outlines how to use them optimally. +be used together with CrateDB, and educates about how to use them optimally. +Storing metrics data for the long term is a common need in systems monitoring +scenarios. CrateDB offers corresponding integration adapters. ## Prometheus -- [CrateDB as a long term metrics store for Prometheus](#metrics-store-prometheus) -- [Webinar: Using Prometheus and Grafana with CrateDB Cloud] +:::{toctree} +:maxdepth: 1 +../prometheus/index +::: -## Telegraf -- [Use CrateDB With Telegraf, an Agent for Collecting & Reporting Metrics] +## Telegraf +:::{toctree} +:maxdepth: 1 -[Use CrateDB With Telegraf, an Agent for Collecting & Reporting Metrics]: https://cratedb.com/blog/use-cratedb-with-telegraf-an-agent-for-collecting-reporting-metrics -[Webinar: Using Prometheus and Grafana with CrateDB Cloud]: https://cratedb.com/resources/webinars/lp-wb-prometheus-grafana +../telegraf/index +::: diff --git a/docs/integrate/node-red/index.md b/docs/integrate/node-red/index.md new file mode 100644 index 00000000..ec7835ca --- /dev/null +++ b/docs/integrate/node-red/index.md @@ -0,0 +1,72 @@ +(node-red)= +# Node-RED + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 1em" +[![](https://upload.wikimedia.org/wikipedia/commons/2/2b/Node-red-icon.png){w=180px}](https://nodered.org/) +``` + +[Node-RED] is a programming tool for wiring together hardware devices, APIs +and online services within a low-code programming environment for event-driven +applications. It allows orchestrating message flows and transformations through +an intuitive web interface. + +It provides a browser-based editor that makes it easy to wire together flows +using the wide range of elements called "nodes" from the palette that can be +deployed to its runtime in a single-click. + +:::{dropdown} **Managed Node-RED** +```{div} +:style: "float: right; margin: 0.5em" +[![](https://github.com/crate/crate-clients-tools/assets/453543/200d1a92-1e38-453a-89bf-d8b727451fab){w=180px}](https://flowfuse.com/) +``` + +With [FlowFuse], and [FlowFuse Cloud], essentially unmanaged and managed DevOps +for Node-RED, you can reliably deliver Node-RED applications in a continuous, +collaborative, and secure manner. + +- **Collaborative Development:** FlowFuse adds team collaboration to Node-RED, + allowing multiple developers to work together on a single instance. With + FlowFuse, developers can easily share projects, collaborate in real-time and + streamline workflow for faster development and deployment. +- **Manage Remote Deployments:** Many organizations deploy Node-RED instances to + remote servers or edge devices. FlowFuse automates this process by creating + snapshots on Node-RED instances that can be deployed to multiple remote targets. + It also allows for rollback in cases where something might not have gone correctly. +- **Streamline Application Delivery:** FlowFuse simplifies the software development + lifecycle of Node-RED applications. You can now set up DevOps delivery pipelines + to support development, test and production environments for Node-RED application + delivery, see [Introduction to FlowFuse]. +- **Flexible FlowFuse Deployment:** We want to make it easy for you to use FlowFuse, + so we offer FlowFuse Cloud, a managed cloud service, or a self-hosted solution. + You have the freedom to choose the deployment method that works best for your + organization. + +```{div} +:style: "clear: both" +``` +::: + + +:::{rubric} Learn +::: + +- [Ingesting MQTT messages into CrateDB using Node-RED] +- [Automating recurrent CrateDB queries using Node-RED] + + +```{seealso} +[CrateDB and Node-RED] +``` + + +[Automating recurrent CrateDB queries using Node-RED]: https://community.cratedb.com/t/automating-recurrent-cratedb-queries/788 +[CrateDB and Node-RED]: https://cratedb.com/integrations/cratedb-and-node-red +[FlowFuse]: https://flowfuse.com/ +[FlowFuse Cloud]: https://app.flowforge.com/ +[Ingesting MQTT messages into CrateDB using Node-RED]: https://community.cratedb.com/t/ingesting-mqtt-messages-into-cratedb-using-node-red/803 +[Introduction to FlowFuse]: https://flowfuse.com/webinars/2023/introduction-to-flowforge/ +[Node-RED]: https://nodered.org/ diff --git a/docs/integrate/plotly/index.md b/docs/integrate/plotly/index.md new file mode 100644 index 00000000..f4ca43cf --- /dev/null +++ b/docs/integrate/plotly/index.md @@ -0,0 +1,99 @@ +# Plotly and Dash + +:::{include} /_include/links.md +::: + +(plotly)= +## Plotly + +```{div} +:style: "float: right" +[![](https://github.com/crate/crate-clients-tools/assets/453543/8b679c0b-2740-4dcc-88f0-1106aee7fa95){w=180px}](https://plotly.com/) +``` + +[Plotly] Open Source Graphing Libraries make interactive, publication-quality graphs. +Line plots, scatter plots, area charts, bar charts, error bars, box plots, histograms, +heatmaps, subplots, multiple-axes, polar charts, bubble charts, and maps. + +The supported programming languages / libraries / frameworks are Python, R, Julia, +JavaScript, ggplot2, F#, MATLAB®, and Dash. + +Based on Plotly, [Dash] is a low-code framework for rapidly building data apps in Python. + +```{div} +:style: "clear: both" +``` + +![](https://github.com/crate/crate-clients-tools/assets/453543/380114a8-7984-4966-929b-6e6d52ddd48a){h=200px} +![](https://github.com/crate/crate-clients-tools/assets/453543/f6a99ae7-b730-4587-bd23-499e1be02c92){h=200px} + + +(dash)= +## Dash + +```{div} +:style: "float: right" +[![](https://github.com/crate/crate-clients-tools/assets/453543/8b679c0b-2740-4dcc-88f0-1106aee7fa95){w=180px}](https://dash.plotly.com/) +``` + +[Dash] is a low-code framework for rapidly building data apps in Python, +based on [Plotly]. Built on top of Plotly.js, React and Flask, Dash ties +modern UI elements like dropdowns, sliders, and graphs, directly to your +analytical Python code. + +Dash is a trusted Python framework for building ML & data science web apps. Many +specialized open-source Dash libraries exist that are tailored for building +domain-specific Dash components and applications. + +```{div} +:style: "clear: both" +``` +::: + +![](https://github.com/crate/crate-clients-tools/assets/453543/cc538982-e351-437b-97ec-f1fc6ca34948){h=200px} +![](https://github.com/crate/crate-clients-tools/assets/453543/24908861-f0ad-43f3-b229-b2bfcc61596d){h=200px} + +:::{dropdown} **Dash Enterprise** +```{div} +:style: "float: right" +[![](https://github.com/crate/crate-clients-tools/assets/453543/8b679c0b-2740-4dcc-88f0-1106aee7fa95){w=180px}](https://plotly.com/dash/) +``` + +Dash Enterprise is Plotly’s paid product for building, testing, deploying, managing, +and scaling Dash applications organization-wide, advertised as the Premier Data App +Platform for Python. + +When building Dash apps in a business setting, Dash Enterprise supports you to deploy +and scale them, plus integrate them with IT infrastructure such as authentication and +VPC services, in order to deliver faster and more impactful business outcomes on AI +and data science initiatives. + +Dash Enterprise enables the rapid development of production-grade data apps within your +business. Python has taken over the world, and traditional BI dashboards no longer +cut it in today’s AI and ML driven world. Production-grade, low-code Python data apps +are needed to visualize the sophisticated data analytics and data pipelines that run +modern businesses. + +```{div} +:style: "clear: both" +``` +![](https://github.com/crate/crate-clients-tools/assets/453543/161a9b73-25eb-4ec4-aa3e-5fa73757b440){h=200px} +![](https://github.com/crate/crate-clients-tools/assets/453543/d199b9c9-8be0-4ff7-a7b5-835dc122cc6d){h=200px} +::: + + + +_Plotly Dash Course - Session 1._ + + +## Learn + +:::{include} /_include/card/timeseries-explore.md +::: + +Alternatively, you are welcome to explore the canonical [Dash Examples]. + + +[Dash]: https://plotly.com/dash/ +[Dash Examples]: https://plotly.com/examples/ +[Plotly]: https://plotly.com/graphing-libraries/ diff --git a/docs/integrate/bi/powerbi-desktop.rst b/docs/integrate/powerbi/desktop.rst similarity index 100% rename from docs/integrate/bi/powerbi-desktop.rst rename to docs/integrate/powerbi/desktop.rst diff --git a/docs/integrate/bi/powerbi-gateway.rst b/docs/integrate/powerbi/gateway.rst similarity index 100% rename from docs/integrate/bi/powerbi-gateway.rst rename to docs/integrate/powerbi/gateway.rst diff --git a/docs/integrate/powerbi/index.md b/docs/integrate/powerbi/index.md new file mode 100644 index 00000000..15097989 --- /dev/null +++ b/docs/integrate/powerbi/index.md @@ -0,0 +1,48 @@ +(powerbi)= +# Microsoft Power BI + +:::{rubric} About +::: + +```{div} +:style: "float: right" +[![](https://upload.wikimedia.org/wikipedia/en/thumb/2/20/Power_BI_logo.svg/192px-Power_BI_logo.svg.png?20200923233425){w=180px}](https://powerbi.microsoft.com/en-us/desktop/) +``` + +[Power BI Desktop] is a powerful business intelligence tool that provides a set of +data analytics and visualizations. Using Power BI Desktop, users can create reports +and dashboards from large datasets. + +For connecting to CrateDB with Power BI, you can use the [Power Query PostgreSQL connector]. +Earlier versions used the [PostgreSQL ODBC driver]. [](project:#powerbi-desktop) walks +you through the process of configuring that correctly. + +[Power BI Service] is an online data analysis and visualization tool, making it +possible to publish your dashboards, in order to share them with others. +[](project:#powerbi-service) has a corresponding tutorial. + +![](https://cratedb.com/docs/crate/howtos/en/latest/_images/powerbi-table-navigator.png){h=160px} +![](https://cratedb.com/docs/crate/howtos/en/latest/_images/powerbi-pie-chart.png){h=160px} +![](https://cratedb.com/docs/crate/howtos/en/latest/_images/powerbi-publish-success.png){h=160px} + + +:::{rubric} Learn +::: + +```{toctree} +:maxdepth: 1 + +desktop +gateway +``` + + +```{seealso} +[CrateDB and Power BI] +``` + +[CrateDB and Power BI]: https://cratedb.com/integrations/cratedb-and-power-bi +[PostgreSQL ODBC driver]: https://odbc.postgresql.org/ +[Power BI Desktop]: https://powerbi.microsoft.com/en-us/desktop/ +[Power BI Service]: https://powerbi.microsoft.com/en-us/ +[Power Query PostgreSQL connector]: https://learn.microsoft.com/en-us/power-query/connectors/postgresql diff --git a/docs/integrate/prometheus/index.md b/docs/integrate/prometheus/index.md new file mode 100644 index 00000000..cf40adda --- /dev/null +++ b/docs/integrate/prometheus/index.md @@ -0,0 +1,68 @@ +(prometheus)= +# Prometheus + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 0.3em" +[![](https://github.com/crate/crate-clients-tools/assets/453543/8ddb109f-b45f-46b0-8103-30ba491f7142){w=180px}](https://prometheus.io/) +``` + +[Prometheus] is an open-source systems monitoring and alerting toolkit +for collecting metrics data from applications and infrastructures. + +Prometheus collects and stores its metrics as time series data, i.e. +metrics information is stored with the timestamp at which it was recorded, +alongside optional key-value pairs called labels. + +:::{rubric} Features +::: +Prometheus's main features are: + +- a multi-dimensional data model with time series data identified by metric name and key/value pairs +- PromQL, a flexible query language to leverage this dimensionality +- no reliance on distributed storage; single server nodes are autonomous +- time series collection happens via a pull model over HTTP +- pushing time series is supported via an intermediary gateway +- targets are discovered via service discovery or static configuration +- multiple modes of graphing and dashboarding support + + +:::{rubric} Remote Endpoints and Storage +::: +The [Prometheus remote endpoints and storage] subsystem, based on its +[remote write] and [remote read] features, allows to transparently +send and receive metric samples. It is primarily intended for long term +storage. + +This is where CrateDB comes into place. Using the [CrateDB Prometheus +Adapter], one can easily store the collected metrics data in CrateDB and +take advantage of its high ingestion and query speed and friendly UI to +massively scale-out Prometheus. + +![](https://github.com/crate/crate-clients-tools/assets/453543/26b47686-889a-4137-a87f-d6a6b38d56d2){h=200px} + +```{div} +:style: "clear: both" +``` + +:::{rubric} Learn +::: + +- [CrateDB as a long term metrics store for Prometheus](#metrics-store-prometheus) +- [Webinar: Using Prometheus and Grafana with CrateDB Cloud] + +```{seealso} +- [CrateDB and Prometheus] +- [CrateDB Prometheus Adapter] +``` + + +[CrateDB and Prometheus]: https://cratedb.com/integrations/cratedb-and-prometheus +[CrateDB Prometheus Adapter]: https://github.com/crate/cratedb-prometheus-adapter +[Prometheus remote endpoints and storage]: https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage +[Prometheus]: https://prometheus.io/ +[remote read]: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read +[remote write]: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write +[Webinar: Using Prometheus and Grafana with CrateDB Cloud]: https://cratedb.com/resources/webinars/lp-wb-prometheus-grafana diff --git a/docs/integrate/pyviz/index.md b/docs/integrate/pyviz/index.md new file mode 100644 index 00000000..fb58c910 --- /dev/null +++ b/docs/integrate/pyviz/index.md @@ -0,0 +1,63 @@ +(pyviz)= +# PyViz + +:::{include} /_include/links.md +::: + +The PyViz.org website is an open platform for helping users decide on the best +open-source (OSS) Python data visualization tools. + +(hvplot)= +(datashader)= +## hvPlot and Datashader + +```{div} +:style: "float: right; margin-left: 1em" +[![](https://hvplot.holoviz.org/_static/logo_horizontal.svg){w=220px}](https://hvplot.holoviz.org/) + +[![](https://datashader.org/_static/logo_horizontal.svg){w=220px}](https://datashader.org/) +``` + +[hvPlot] is a familiar and high-level API for data exploration and visualization. +[Datashader][Datashader Homepage] is a graphics pipeline system for creating +meaningful representations of large datasets quickly and flexibly. + +It is used on behalf of the [hvPlot] package, which is based on [HoloViews], from the +family of [HoloViz] packages of the [PyViz] ecosystem. + +With Datashader, you can "just plot" large datasets and explore them instantly, with no +parameter tweaking, magic numbers, subsampling, or approximation, up to the resolution +of the display. + +:::{div} +[hvPlot] sources its power in the [HoloViz] ecosystem. With [HoloViews], you get the +ability to easily layout and overlay plots, with [Panel], you can get more interactive +control of your plots with widgets, with [DataShader][Datashader Homepage], you can +visualize and interactively explore very large data, and with [GeoViews], you can +create geographic plots. +::: + +```{div} +:style: "clear: both" +``` + +![](https://github.com/crate/crate-clients-tools/assets/453543/7f38dff6-04bc-429e-9d31-6beeb9289c4b){h=200px} +![](https://github.com/crate/crate-clients-tools/assets/453543/23561a87-fb4f-4154-9891-1b3068e40579){h=200px} + + + + +_hvPlot and Panel: Visualize all your data easily, from notebooks to dashboards | SciPy 2023._ + +## Learn + +:::{include} /_include/card/timeseries-datashader.md +::: + + +[Datashader Homepage]: https://datashader.org/ +[GeoViews]: https://geoviews.org/ +[HoloViz]: https://holoviz.org/ +[hvPlot]: https://hvplot.holoviz.org/ +[Panel]: https://panel.holoviz.org/ +[PyViz]: https://pyviz.org/ diff --git a/docs/integrate/rill/index.md b/docs/integrate/rill/index.md new file mode 100644 index 00000000..cecf21d9 --- /dev/null +++ b/docs/integrate/rill/index.md @@ -0,0 +1,54 @@ +# Rill + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 0.5em" +[![](https://github.com/rilldata/rill/raw/main/docs/static/img/rill-logo-light.svg){w=180px}](https://www.rilldata.com/) +``` + +[Rill] is an open-source operational BI framework for effortlessly transforming +data sets into powerful, opinionated dashboards using SQL. + +Unlike most BI tools, Rill comes with its own embedded in-memory database. Data +and compute are co-located, and queries return in milliseconds. So you can pivot, +slice, and drill-down into your data instantly. + +Rill takes a modern approach to Business Intelligence (BI), which is starting to +leverage software engineering principles by implementing the concept of BI as +code. + +This methodology allows for versioning and tracking, thus improving collaboration +on BI projects using code, which is more efficient and scalable than traditional +BI tools, also breaking down information and knowledge barriers. + +:::{rubric} Rill's design principles +::: + +- **Feels good to use** – powered by Sveltekit & DuckDB = conversation-fast, not + wait-ten-seconds-for-result-set fast +- **Works with your local and remote datasets** – imports and exports Parquet and + CSV (s3, gcs, https, local) +- **No more data analysis "side-quests"** – helps you build intuition about your + dataset through automatic profiling +- **No "run query" button required** – responds to each keystroke by re-profiling + the resulting dataset +- **Radically simple interactive dashboards** – thoughtful, opinionated, interactive + dashboard defaults to help you quickly derive insights from your data +- **Dashboards as code** – each step from data to dashboard has versioning, Git + sharing, and easy project rehydration + +![](https://cdn.prod.website-files.com/659ddac460dbacbdc813b204/65b83308971b2f12202ae0fa_b2a470f529fc0f7d9b66de4d75742674.gif){h=200px} +![](https://cdn.prod.website-files.com/659ddac460dbacbdc813b204/65b835371c75806184829601_BI-as-code%20(1)-p-3200.webp){h=200px} + + +:::{rubric} Learn +::: + +- [Introducing Rill and BI as Code with CrateDB Cloud] + +[Introducing Rill and BI as Code with CrateDB Cloud]: https://community.cratedb.com/t/introducing-rill-and-bi-as-code-with-cratedb-cloud/1718 + + +[Rill]: https://www.rilldata.com/ diff --git a/docs/integrate/sql-server/index.md b/docs/integrate/sql-server/index.md new file mode 100644 index 00000000..068b9fce --- /dev/null +++ b/docs/integrate/sql-server/index.md @@ -0,0 +1,48 @@ +(sql-server)= +# SQL Server + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 1em" +[![](https://github.com/crate/crate-clients-tools/assets/453543/a93a0fdb-1a1e-451e-abcb-8f705e2b03f4){w=180px}](https://www.microsoft.com/) + +[![](https://github.com/crate/crate-clients-tools/assets/453543/6317965a-0b69-4d8e-bc77-e12dfc8ed338){w=180px}](https://learn.microsoft.com/en-us/sql/) +``` + +Microsoft [SQL Server Integration Services] (SSIS) is a component of the Microsoft +SQL Server database software that can be used to perform a broad range of data +migration tasks. + +[SSIS] is a platform for data integration and workflow applications. It features a +data warehousing tool used for data extraction, transformation, and loading (ETL). +The tool may also be used to automate maintenance of SQL Server databases and +updates to multidimensional cube data. + +Integration Services can extract and transform data from a wide variety of sources +such as XML data files, flat files, and relational data sources, and then load the +data into one or more destinations. + +Integration Services includes a rich set of built-in [tasks][ssis-tasks] and +[transformations][ssis-transformations], graphical tools for building packages, and +an SSIS Catalog database to store, run, and manage packages. + +```{div} +:style: "clear: both" +``` + + +:::{rubric} Learn +::: + +A demo project which uses SSIS and ODBC to read and write data from CrateDB: + +- [Using SQL Server Integration Services with CrateDB] + + +[SQL Server Integration Services]: https://learn.microsoft.com/en-us/sql/integration-services/sql-server-integration-services +[SSIS]: https://en.wikipedia.org/wiki/SQL_Server_Integration_Services +[ssis-tasks]: https://learn.microsoft.com/en-us/sql/integration-services/control-flow/integration-services-tasks +[ssis-transformations]: https://learn.microsoft.com/en-us/sql/integration-services/data-flow/transformations/integration-services-transformations +[Using SQL Server Integration Services with CrateDB]: https://github.com/crate/cratedb-examples/tree/main/application/microsoft-ssis diff --git a/docs/integrate/streamlit/index.md b/docs/integrate/streamlit/index.md new file mode 100644 index 00000000..ee3e84c9 --- /dev/null +++ b/docs/integrate/streamlit/index.md @@ -0,0 +1,29 @@ +(streamlit)= +# Streamlit + +```{div} +:style: "float: right; margin-left: 0.5em" +[![](https://github.com/crate/crate-clients-tools/assets/453543/0fffb2d4-1d17-49c9-96e3-fd6ae42a39c4){w=180px}](https://streamlit.io/) +``` + +[Streamlit] is an open source application programming framework for quickly sketching +out Python data applications. It provides fast, interactive prototyping, and live editing. + +- Build dashboards, generate reports, or create chat apps using beautiful, easy-to-read code. +- No in-depth knowledge of HTML/JS/CSS needed, the framework offers elegant default + styling, which can be adjusted when applicable. +- Transform Python scripts into interactive web apps in minutes, instead of weeks. +- Build upon a range of [Streamlit components](https://streamlit.io/components). +- Optionally use their [Community Cloud platform](https://streamlit.io/cloud) to deploy, + manage, and share your application. + +```{div} +:style: "clear: both" +``` + + + +_Streamlit 101 - A faster way to build and share data applications._ + + +[Streamlit]: https://streamlit.io/ diff --git a/docs/integrate/tableau/index.md b/docs/integrate/tableau/index.md new file mode 100644 index 00000000..3e445962 --- /dev/null +++ b/docs/integrate/tableau/index.md @@ -0,0 +1,36 @@ +(tableau)= +# Tableau + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 0.5em" +[![](https://upload.wikimedia.org/wikipedia/en/thumb/0/06/Tableau_logo.svg/500px-Tableau_logo.svg.png?20200509180027){w=180px}](https://www.tableau.com/) +``` + +[Tableau] is a visual business intelligence and analytics software platform. It expresses +data by translating drag-and-drop actions into data queries through an intuitive interface. + +![](https://cratedb.com/hs-fs/hubfs/08-index.png?width=1536&name=08-index.png){h=200px} + + +:::{rubric} Learn +::: + +- [Connecting to CrateDB from Tableau with JDBC] and [Using CrateDB with Tableau] + will guide you through the process of setting up Tableau correctly with CrateDB. +- The [CrateDB Tableau Connector] unlocks advanced SQL functionality and + resolves compatibility issues beyond standard usage. + + +```{seealso} +[CrateDB and Tableau] +``` + + +[Connecting to CrateDB from Tableau with JDBC]: https://cratedb.com/blog/connecting-to-cratedb-from-tableau-with-jdbc +[CrateDB and Tableau]: https://cratedb.com/integrations/cratedb-and-tableau +[CrateDB Tableau Connector]: https://github.com/crate/cratedb-tableau-connector +[Tableau]: https://www.tableau.com/ +[Using CrateDB with Tableau]: https://community.cratedb.com/t/using-cratedb-with-tableau/1192 diff --git a/docs/integrate/telegraf/index.md b/docs/integrate/telegraf/index.md new file mode 100644 index 00000000..e1df987a --- /dev/null +++ b/docs/integrate/telegraf/index.md @@ -0,0 +1,54 @@ +(telegraf)= +# Telegraf + +:::{rubric} About +::: + +```{div} +:style: "float: right; margin-left: 0.3em" +[![](https://github.com/crate/crate-clients-tools/assets/453543/3f0b4525-5344-42fe-bae6-1b0970fa0540){w=180px}](https://www.influxdata.com/time-series-platform/telegraf/) +``` + +[Telegraf] is a leading open source server agent to help you collect metrics +from your stacks, sensors, and systems. More than 200 adapters to connect +to other systems leaves nothing to be desired. + +Telegraf is a server-based agent for collecting and sending all metrics and +events from databases, systems, and IoT sensors. Telegraf is written in Go +and compiles into a single binary with no external dependencies, and requires +a very minimal memory footprint. + +:::{rubric} Overview +::: + +- **IoT sensors**: Collect critical stateful data (pressure levels, temperature + levels, etc.) with popular protocols like MQTT, ModBus, OPC-UA, and Kafka. + +- **DevOps Tools and frameworks**: Gather metrics from cloud platforms, + containers, and orchestrators like GitHub, Kubernetes, CloudWatch, Prometheus, + and more. + +- **System telemetry**: Metrics from system telemetry like iptables, Netstat, + NGINX, and HAProxy help provide a full stack view of your apps. + +![](https://www.influxdata.com/wp-content/uploads/Main-Diagram_06.01.2022v1.png){h=200px} + +```{div} +:style: "clear: both" +``` + +:::{rubric} Learn +::: + +- [Use CrateDB With Telegraf, an Agent for Collecting & Reporting Metrics] + + +```{seealso} +[CrateDB and Telegraf] +``` + + + +[CrateDB and Telegraf]: https://cratedb.com/integrations/cratedb-and-telegraf +[Telegraf]: https://www.influxdata.com/time-series-platform/telegraf/ +[Use CrateDB With Telegraf, an Agent for Collecting & Reporting Metrics]: https://cratedb.com/blog/use-cratedb-with-telegraf-an-agent-for-collecting-reporting-metrics diff --git a/docs/integrate/visualize/index.md b/docs/integrate/visualize/index.md index b03f5753..e3ca2d7c 100644 --- a/docs/integrate/visualize/index.md +++ b/docs/integrate/visualize/index.md @@ -1,86 +1,21 @@ +(visualize)= (visualization)= +# Visualize data in CrateDB -# Data Visualization - -Guidelines about data analysis and visualization with CrateDB. +Data visualization applications and toolkits for +visualizing data stored inside CrateDB, mostly dashboarding. :::{include} /_include/links.md ::: - :::{toctree} :maxdepth: 1 ../apache-superset/index +../cluvio/index +../explo/index +../grafana/index +../pyviz/index +../metabase/index +../plotly/index ::: - - -## Cluvio - -- [Data Analysis with Cluvio and CrateDB] - - -## Explo - -- [Introduction to Time Series Visualization in CrateDB and Explo] - - -## Grafana - -- {ref}`integrations-grafana` - - -(datashader)= -## hvPlot and Datashader - -:::{include} /_include/card/timeseries-datashader.md -::: - - -## Metabase - -- {ref}`integrations-metabase` -- [Real-time data analytics with Metabase and CrateDB] -- https://github.com/paoliniluis/metabase-cratedb - - -## pandas - -- [From data storage to data analysis\: Tutorial on CrateDB and pandas] - - -(plotly)= -## Plotly / Dash - -:::{include} /_include/card/timeseries-explore.md -::: - -Alternatively, you are welcome to explore the canonical [Dash Examples]. - - - -## R - -```{toctree} -:maxdepth: 1 - -r -``` - - -```{toctree} -:hidden: - -grafana -metabase -``` - - -[Dash Examples]: https://plotly.com/examples/ -[Data Analysis with Cluvio and CrateDB]: https://community.cratedb.com/t/data-analysis-with-cluvio-and-cratedb/1571 -[From data storage to data analysis\: Tutorial on CrateDB and pandas]: https://community.cratedb.com/t/from-data-storage-to-data-analysis-tutorial-on-cratedb-and-pandas/1440 -[Introduction to Time Series Visualization in CrateDB and Explo]: https://cratedb.com/blog/introduction-to-time-series-visualization-in-cratedb-and-explo -[Real-time data analytics with Metabase and CrateDB]: https://www.metabase.com/community_posts/real-time-data-analytics-with-metabase-and-cratedb -[Time Series with CrateDB]: https://github.com/crate/cratedb-examples/tree/main/topic/timeseries/explore -[Using Grafana with CrateDB Cloud]: #integrations-grafana -[Using Metabase with CrateDB Cloud]: #integrations-metabase