add redis sink and source

Signed-off-by: shumin <[email protected]>

Author: shumin1027

.github/workflows/integration-test.yml

@@ -313,6 +313,24 @@ jobs:
       - run: make slim-builds
       - run: make test-integration-splunk
 
+  test-integration-redis:
+    name: Integration - Linux, Redis
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - run: make ci-sweep
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh
+      - run: bash scripts/environment/prepare.sh
+      - run: echo "::add-matcher::.github/matchers/rust.json"
+      - run: make slim-builds
+      - run: make test-integration-redis
+
   master-failure:
     name: master-failure
     if: failure() && github.ref == 'refs/heads/master'
@@ -329,6 +347,7 @@ jobs:
       - test-integration-loki
       - test-integration-pulsar
       - test-integration-splunk
+      - test-integration-redis
     runs-on: ubuntu-20.04
     steps:
     - name: Discord notification

Cargo.lock

@@ -222,6 +222,7 @@ typetag = "0.1.6"
 url = "2.2.0"
 uuid = { version = "0.8", features = ["serde", "v4"], optional = true }
 warp = { version = "0.2.5", default-features = false, optional = true }
+redis = { version = "0.20.0", features = ["connection-manager","async-std-comp"], optional = true }
 
 # For WASM
 async-stream = "0.3.0"
@@ -361,6 +362,7 @@ sources-metrics = [
   "sources-prometheus",
   "sources-statsd",
   "sources-vector",
+  "sources-redis",
 ]
 
 sources-apache_metrics = []
@@ -394,6 +396,7 @@ sources-utils-tls = []
 sources-utils-udp = ["socket2"]
 sources-utils-unix = []
 sources-vector = ["listenfd", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls"]
+sources-redis = ["redis"]
 
 # Transforms
 transforms = ["transforms-logs", "transforms-metrics"]
@@ -512,7 +515,8 @@ sinks-metrics = [
   "sinks-prometheus",
   "sinks-sematext",
   "sinks-statsd",
-  "sinks-vector"
+  "sinks-vector",
+  "sinks-redis",
 ]
 
 sinks-aws_cloudwatch_logs = ["rusoto", "rusoto_logs"]
@@ -547,6 +551,7 @@ sinks-splunk_hec = ["bytesize"]
 sinks-statsd = ["sinks-utils-udp", "tokio-util/udp"]
 sinks-utils-udp = ["socket2"]
 sinks-vector = ["sinks-utils-udp"]
+sinks-redis = ["redis"]
 
 # Identifies that the build is a nightly build
 nightly = []
@@ -571,6 +576,7 @@ all-integration-tests = [
   "prometheus-integration-tests",
   "pulsar-integration-tests",
   "splunk-integration-tests",
+  "redis-integration-tests",
 ]
 
 aws-integration-tests = [
@@ -609,6 +615,7 @@ postgresql_metrics-integration-tests = ["sources-postgresql_metrics"]
 prometheus-integration-tests = ["bytesize", "sinks-prometheus", "sources-prometheus"]
 pulsar-integration-tests = ["sinks-pulsar"]
 splunk-integration-tests = ["sinks-splunk_hec", "warp"]
+redis-integration-tests = ["sources-redis", "sinks-redis"]
 
 disable-resolv-conf = []
 shutdown-tests = ["rdkafka", "sinks-blackhole", "sinks-console", "sinks-prometheus", "sources", "transforms-log_to_metric", "transforms-lua", "unix"]

Cargo.toml

@@ -299,7 +299,7 @@ test-behavior: ## Runs behaviorial test
 
 .PHONY: test-integration
 test-integration: ## Runs all integration tests
-test-integration: test-integration-aws test-integration-clickhouse test-integration-docker-logs test-integration-elasticsearch
+test-integration: test-integration-aws  test-integration-docker-logs test-integration-elasticsearch
 test-integration: test-integration-gcp test-integration-humio test-integration-influxdb test-integration-kafka
 test-integration: test-integration-loki test-integration-mongodb_metrics test-integration-nats
 test-integration: test-integration-nginx test-integration-postgresql_metrics test-integration-prometheus test-integration-pulsar
@@ -317,7 +317,7 @@ ifeq ($(AUTODESPAWN), true)
 	@scripts/setup_integration_env.sh aws stop
 endif
 
-.PHONY: test-integration-clickhouse
+.PHONY:
 test-integration-clickhouse: ## Runs Clickhouse integration tests
 ifeq ($(AUTOSPAWN), true)
 	@scripts/setup_integration_env.sh clickhouse stop
@@ -492,6 +492,18 @@ ifeq ($(AUTODESPAWN), true)
 	@scripts/setup_integration_env.sh splunk stop
 endif
 
+.PHONY: test-integration-redis
+test-integration-redis: ## Runs Redis integration tests
+ifeq ($(AUTOSPAWN), true)
+	@scripts/setup_integration_env.sh redis stop
+	@scripts/setup_integration_env.sh redis start
+	sleep 10 # Many services are very slow... Give them a sec..
+endif
+	${MAYBE_ENVIRONMENT_EXEC} cargo test --no-fail-fast --no-default-features --features redis-integration-tests --lib ::redis:: -- --nocapture
+ifeq ($(AUTODESPAWN), true)
+	@scripts/setup_integration_env.sh redis stop
+endif
+
 .PHONY: test-e2e-kubernetes
 test-e2e-kubernetes: ## Runs Kubernetes E2E tests (Sorry, no `ENVIRONMENT=true` support)
 	@scripts/test-e2e-kubernetes.sh