Skip to content

Commit 7d2b723

Browse files
fe2sfe2s
authored
Merge pull request #333 from RedisLabs/acl-auth
issue #328: add user/password authentication (Redis 6 ACL feature)
2 parents 1b72192 + ddb174a commit 7d2b723

23 files changed

+372
-95
lines changed

‎.circleci/config.yml

Lines changed: 109 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,25 @@
11
version: 2.1
2+
23
commands:
4+
5+
abort_for_docs:
6+
steps:
7+
- run:
8+
name: Avoid tests for docs
9+
command: |
10+
if [[ $CIRCLE_BRANCH == *docs ]]; then
11+
echo "Identifies as documents PR, no testing required"
12+
circleci step halt
13+
fi
14+
abort_for_noci:
15+
steps:
16+
- run:
17+
name: Ignore CI for specific branches
18+
command: |
19+
if [[ $CIRCLE_BRANCH == *noci ]]; then
20+
echo "Identifies as actively ignoring CI, no testing required."
21+
circleci step halt
22+
fi
323
early_return_for_forked_pull_requests:
424
description: >-
525
If this build is from a fork, stop executing the current job and return success.
@@ -12,80 +32,140 @@ commands:
1232
echo "Nothing to do for forked PRs, so marking this step successful"
1333
circleci step halt
1434
fi
35+
jobs:
1536

16-
executors:
17-
linux-8-jdk:
37+
build:
1838
docker:
1939
- image: circleci/openjdk:8-jdk
2040

21-
jobs:
22-
build:
23-
parameters:
24-
os:
25-
type: executor
26-
27-
executor: << parameters.os >>
28-
29-
working_directory: ~/repo
30-
3141
environment:
3242
JVM_OPTS: -Xmx3200m
3343
TERM: dumb
3444

3545
steps:
46+
- abort_for_docs
47+
- abort_for_noci
3648
- checkout
3749

3850
# Download and cache dependencies
3951
- restore_cache:
4052
keys:
4153
- spark-redis-{{ checksum "pom.xml" }}
4254

43-
- run: mvn dependency:go-offline
55+
- run: mvn dependency:go-offline
4456

57+
- run:
58+
name: install Redis
59+
command: |
60+
sudo apt-get -y update
61+
sudo apt-get install -y libssl-dev
62+
wget http://download.redis.io/releases/redis-6.0.10.tar.gz
63+
tar -xzvf redis-6.0.10.tar.gz
64+
make -C redis-6.0.10 -j`nproc` BUILD_TLS=yes
65+
export PATH=$PWD/redis-6.0.10/src:$PATH
66+
- run:
67+
name: Run Test
68+
command: |
69+
export PATH=$PWD/redis-6.0.10/src:$PATH
70+
make test
71+
- run:
72+
name: switch scala to 2.12
73+
command: |
74+
sleep 5s # let redis exit gracefully (we use kill, not kill -9 in makefile)
75+
ps aux | grep redis
76+
./dev/change-scala-version.sh 2.12 # switch to scala 2.12
77+
- run:
78+
name: Run Test with scala 2.12
79+
command: |
80+
export PATH=$PWD/redis-6.0.10/src:$PATH
81+
make test
4582
- save_cache:
4683
paths:
4784
- ~/.m2
4885
key: spark-redis-{{ checksum "pom.xml" }}
4986

50-
- run:
87+
- early_return_for_forked_pull_requests
88+
89+
- run: bash <(curl -s https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t ${CODECOV_TOKEN}
90+
91+
build-and-publish:
92+
docker:
93+
- image: circleci/openjdk:8-jdk
94+
95+
environment:
96+
JVM_OPTS: -Xmx3200m
97+
TERM: dumb
98+
99+
steps:
100+
- abort_for_docs
101+
- abort_for_noci
102+
- checkout
103+
104+
# Download and cache dependencies
105+
- restore_cache:
106+
keys:
107+
- spark-redis-{{ checksum "pom.xml" }}
108+
109+
- run: mvn dependency:go-offline
110+
111+
- run:
51112
name: install Redis
52113
command: |
53114
sudo apt-get -y update
54-
sudo apt-get install -y gcc-8 g++-8 libssl-dev
115+
sudo apt-get install -y libssl-dev
55116
wget http://download.redis.io/releases/redis-6.0.10.tar.gz
56117
tar -xzvf redis-6.0.10.tar.gz
57-
make -C redis-6.0.10 -j4 BUILD_TLS=yes
118+
make -C redis-6.0.10 -j`nproc` BUILD_TLS=yes
58119
export PATH=$PWD/redis-6.0.10/src:$PATH
59-
60120
- run:
61121
name: Run Test
62122
command: |
63123
export PATH=$PWD/redis-6.0.10/src:$PATH
64124
make test
65-
66125
- run:
67-
name: switch scala to 2.12
126+
name: switch scala to 2.12
68127
command: |
69128
sleep 5s # let redis exit gracefully (we use kill, not kill -9 in makefile)
70129
ps aux | grep redis
71130
./dev/change-scala-version.sh 2.12 # switch to scala 2.12
72-
73-
- run:
74-
name: Run Test with scala 2.12
131+
- run:
132+
name: Run Test with scala 2.12
75133
command: |
76134
export PATH=$PWD/redis-6.0.10/src:$PATH
77135
make test
78-
136+
- save_cache:
137+
paths:
138+
- ~/.m2
139+
key: spark-redis-{{ checksum "pom.xml" }}
140+
79141
- early_return_for_forked_pull_requests
80142

81-
- run: bash <(curl -s https://codecov.io/bash) -t ${CODECOV_TOKEN}
82-
143+
- run: bash <(curl -s https://raw.githubusercontent.com/codecov/codecov-bash/master/codecov) -t ${CODECOV_TOKEN}
144+
83145
- run: mvn -s .circleci.settings.xml -DskipTests deploy
84146

147+
on-master: &on-master
148+
filters:
149+
branches:
150+
only: master
151+
tags:
152+
ignore: /.*/
153+
154+
not-on-master: &not-on-master
155+
filters:
156+
branches:
157+
ignore: master
158+
tags:
159+
ignore: /.*/
160+
85161
workflows:
86-
all-jdks:
162+
commit:
87163
jobs:
88164
- build:
89-
matrix:
90-
parameters:
91-
os: [linux-8-jdk]
165+
<<: *not-on-master
166+
context:
167+
- common
168+
- build-and-publish:
169+
<<: *on-master
170+
context:
171+
- common

‎Makefile

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
# user
2+
USER_ACL = user alice on >p1pp0 ~* +@all
3+
14
# STANDALONE REDIS NODE
25
define REDIS_STANDALONE_NODE_CONF
36
daemonize yes
@@ -7,6 +10,7 @@ logfile /tmp/redis_standalone_node_for_spark-redis.log
710
save ""
811
appendonly no
912
requirepass passwd
13+
$(USER_ACL)
1014
endef
1115

1216
# STANDALONE REDIS NODE WITH SSL
@@ -18,6 +22,7 @@ logfile /tmp/redis_standalone_node_ssl_for_spark-redis.log
1822
save ""
1923
appendonly no
2024
requirepass passwd
25+
$(USER_ACL)
2126
tls-auth-clients no
2227
tls-port 6380
2328
tls-cert-file ./src/test/resources/tls/redis.crt
@@ -30,6 +35,7 @@ endef
3035
define REDIS_CLUSTER_NODE1_CONF
3136
daemonize yes
3237
port 7379
38+
$(USER_ACL)
3339
pidfile /tmp/redis_cluster_node1_for_spark-redis.pid
3440
logfile /tmp/redis_cluster_node1_for_spark-redis.log
3541
save ""
@@ -41,6 +47,7 @@ endef
4147
define REDIS_CLUSTER_NODE2_CONF
4248
daemonize yes
4349
port 7380
50+
$(USER_ACL)
4451
pidfile /tmp/redis_cluster_node2_for_spark-redis.pid
4552
logfile /tmp/redis_cluster_node2_for_spark-redis.log
4653
save ""
@@ -52,6 +59,7 @@ endef
5259
define REDIS_CLUSTER_NODE3_CONF
5360
daemonize yes
5461
port 7381
62+
$(USER_ACL)
5563
pidfile /tmp/redis_cluster_node3_for_spark-redis.pid
5664
logfile /tmp/redis_cluster_node3_for_spark-redis.log
5765
save ""

‎doc/configuration.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ The supported configuration parameters are:
55
* `spark.redis.host` - host or IP of the initial node we connect to. The connector will read the cluster
66
topology from the initial node, so there is no need to provide the rest of the cluster nodes.
77
* `spark.redis.port` - the initial node's TCP redis port.
8+
* `spark.redis.user` - the initial node's AUTH user
89
* `spark.redis.auth` - the initial node's AUTH password
910
* `spark.redis.db` - optional DB number. Avoid using this, especially in cluster mode.
1011
* `spark.redis.timeout` - connection timeout in ms, 2000 ms by default

‎doc/dataframe.md

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -330,22 +330,23 @@ root
330330

331331
## DataFrame options
332332

333-
| Name | Description | Type | Default |
334-
| -----------------------| ------------------------------------------------------------------------------------------| --------------------- | ------- |
335-
| model | defines the Redis model used to persist DataFrame, see [Persistence model](#persistence-model)| `enum [binary, hash]` | `hash` |
336-
| filter.keys.by.type | make sure the underlying data structures match persistence model | `Boolean` | `false` |
337-
| partitions.number | number of partitions (applies only when reading DataFrame) | `Int` | `3` |
333+
| Name | Description | Type | Default |
334+
|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| --------------------- | ------- |
335+
| model | defines the Redis model used to persist DataFrame, see [Persistence model](#persistence-model) | `enum [binary, hash]` | `hash` |
336+
| filter.keys.by.type | make sure the underlying data structures match persistence model | `Boolean` | `false` |
337+
| partitions.number | number of partitions (applies only when reading DataFrame) | `Int` | `3` |
338338
| key.column | when writing - specifies unique column used as a Redis key, by default a key is auto-generated <br/> when reading - specifies column name to store hash key | `String` | - |
339-
| ttl | data time to live in `seconds`. Data doesn't expire if `ttl` is less than `1` | `Int` | `0` |
340-
| infer.schema | infer schema from random row, all columns will have `String` type | `Boolean` | `false` |
341-
| max.pipeline.size | maximum number of commands per pipeline (used to batch commands) | `Int` | 100 |
342-
| scan.count | count option of SCAN command (used to iterate over keys) | `Int` | 100 |
343-
| iterator.grouping.size | the number of items to be grouped when iterating over underlying RDD partition | `Int` | 1000 |
344-
| host | overrides `spark.redis.host` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `String` | `localhost` |
345-
| port | overrides `spark.redis.port` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `Int` | `6379` |
346-
| auth | overrides `spark.redis.auth` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `String` | - |
347-
| dbNum | overrides `spark.redis.db` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `Int` | `0` |
348-
| timeout | overrides `spark.redis.timeout` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `Int` | `2000` |
339+
| ttl | data time to live in `seconds`. Data doesn't expire if `ttl` is less than `1` | `Int` | `0` |
340+
| infer.schema | infer schema from random row, all columns will have `String` type | `Boolean` | `false` |
341+
| max.pipeline.size | maximum number of commands per pipeline (used to batch commands) | `Int` | 100 |
342+
| scan.count | count option of SCAN command (used to iterate over keys) | `Int` | 100 |
343+
| iterator.grouping.size | the number of items to be grouped when iterating over underlying RDD partition | `Int` | 1000 |
344+
| host | overrides `spark.redis.host` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `String` | `localhost` |
345+
| port | overrides `spark.redis.port` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `Int` | `6379` |
346+
| user | overrides `spark.redis.user` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `String` | - |
347+
| auth | overrides `spark.redis.auth` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `String` | - |
348+
| dbNum | overrides `spark.redis.db` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `Int` | `0` |
349+
| timeout | overrides `spark.redis.timeout` configured in SparkSession (if set, any other connection setting from SparkSession is ignored ) | `Int` | `2000` |
349350

350351

351352
## Known limitations

‎pom.xml

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>com.redislabs</groupId>
55
<artifactId>spark-redis_2.11</artifactId>
6-
<version>2.6.0-SNAPSHOT</version>
6+
<version>2.7.0-SNAPSHOT</version>
77
<name>Spark-Redis</name>
88
<description>A Spark library for Redis</description>
99
<url>http://github.com/RedisLabs/spark-redis</url>
@@ -49,7 +49,7 @@
4949
<java.version>1.8</java.version>
5050
<scala.major.version>2.11</scala.major.version>
5151
<scala.complete.version>${scala.major.version}.12</scala.complete.version>
52-
<jedis.version>3.2.0</jedis.version>
52+
<jedis.version>3.8.0</jedis.version>
5353
<spark.version>2.4.1</spark.version>
5454
<plugins.scalatest.version>1.0</plugins.scalatest.version>
5555
</properties>
@@ -271,11 +271,6 @@
271271
</plugins>
272272
</build>
273273
<dependencies>
274-
<dependency>
275-
<groupId>org.apache.commons</groupId>
276-
<artifactId>commons-pool2</artifactId>
277-
<version>2.0</version>
278-
</dependency>
279274
<dependency>
280275
<groupId>redis.clients</groupId>
281276
<artifactId>jedis</artifactId>

‎src/main/scala/com/redislabs/provider/redis/ConnectionPool.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package com.redislabs.provider.redis
22

3-
import redis.clients.jedis.{JedisPoolConfig, Jedis, JedisPool}
43
import redis.clients.jedis.exceptions.JedisConnectionException
4+
import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig}
55

6+
import java.time.Duration
67
import java.util.concurrent.ConcurrentHashMap
7-
88
import scala.collection.JavaConversions._
99

1010

@@ -21,11 +21,11 @@ object ConnectionPool {
2121
poolConfig.setTestOnBorrow(false)
2222
poolConfig.setTestOnReturn(false)
2323
poolConfig.setTestWhileIdle(false)
24-
poolConfig.setMinEvictableIdleTimeMillis(60000)
25-
poolConfig.setTimeBetweenEvictionRunsMillis(30000)
24+
poolConfig.setSoftMinEvictableIdleTime(Duration.ofMinutes(1))
25+
poolConfig.setTimeBetweenEvictionRuns(Duration.ofSeconds(30))
2626
poolConfig.setNumTestsPerEvictionRun(-1)
2727

28-
new JedisPool(poolConfig, re.host, re.port, re.timeout, re.auth, re.dbNum, re.ssl)
28+
new JedisPool(poolConfig, re.host, re.port, re.timeout, re.user, re.auth, re.dbNum, re.ssl)
2929
}
3030
)
3131
var sleepTime: Int = 4

0 commit comments

Comments
 (0)