Merge pull request #174 from peter-wangxu/feature/support_external_db

Add mysql support(only for python3 now)

.circleci/config.yml

@@ -1,13 +1,33 @@
-version: 2
+version: 2.1
 jobs:
   py27:
     docker:
       # Primary container image where all steps run.
       - image: circleci/python:2.7.17
-    environment:
-      - TOXENV: py27
+        environment:
+          TOXENV: py27
+      # MySQL env for mysql queue tests
+      - image: circleci/mysql:8.0
+        environment:
+          MYSQL_ROOT_PASSWORD: rootpw
+          MYSQL_DATABASE: testqueue
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
+          MYSQL_HOST: '%'
+
     steps: &common_steps
       - checkout
+      - run:
+          # Our primary container isn't MYSQL so run a sleep command until it's ready.
+          name: Waiting for MySQL to be ready
+          command: |
+            for i in `seq 1 10`;
+            do
+              nc -z 127.0.0.1 3306 && echo Success && exit 0
+              echo -n .
+              sleep 5
+            done
+            echo Failed waiting for MySQL && exit 1
       - run:
           command: |
             sudo pip install tox
@@ -19,7 +39,7 @@ jobs:
       - run:
           command: |
             mkdir -p /tmp/core_dumps
-            cp core.* /tmp/core_dumps
+            ls core.* && cp core.* /tmp/core_dumps
           when: on_fail
       - store_artifacts:
           # collect core dumps
@@ -35,57 +55,106 @@ jobs:
     docker:
       # Primary container image where all steps run.
       - image: circleci/python:3.4.10
-    environment:
-      - TOXENV: py34
+        environment:
+          TOXENV: py34
+      # MySQL env for mysql queue tests
+      - image: circleci/mysql:8.0
+        environment:
+          MYSQL_ROOT_PASSWORD: 123456
+          MYSQL_DATABASE: testqueue
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
     steps: *common_steps
 
   py35:
     docker:
       # Primary container image where all steps run.
       - image: circleci/python:3.5.9
-    environment:
-      - TOXENV: py35
+        environment:
+          TOXENV: py35
+      # MySQL env for mysql queue tests
+      - image: circleci/mysql:8.0
+        environment:
+          MYSQL_ROOT_PASSWORD: 123456
+          MYSQL_DATABASE: testqueue
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
     steps: *common_steps
 
   py36:
     docker:
       # Primary container image where all steps run.
       - image: circleci/python:3.6.10
-    environment:
-      - TOXENV: py36
+        environment:
+          TOXENV: py36
+      # MySQL env for mysql queue tests
+      - image: circleci/mysql:8.0
+        environment:
+          MYSQL_ROOT_PASSWORD: 123456
+          MYSQL_DATABASE: testqueue
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
     steps: *common_steps
 
   py37:
     docker:
       # Primary container image where all steps run.
       - image: circleci/python:3.7.7
-    environment:
-      - TOXENV: py37
+        environment:
+          TOXENV: py37
+      # MySQL env for mysql queue tests
+      - image: circleci/mysql:8.0
+        environment:
+          MYSQL_ROOT_PASSWORD: 123456
+          MYSQL_DATABASE: testqueue
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
     steps: *common_steps
 
   py38:
     docker:
       # Primary container image where all steps run.
       - image: circleci/python:3.8.2
-    environment:
-      - TOXENV: py38
+        environment:
+          TOXENV: py38
+      # MySQL env for mysql queue tests
+      - image: circleci/mysql:8.0
+        environment:
+          MYSQL_ROOT_PASSWORD: 123456
+          MYSQL_DATABASE: testqueue
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
     steps: *common_steps
 
   pep8:
     docker:
       # Primary container image where all steps run.
       - image: circleci/python:3.5.9
-    environment:
-      - TOXENV: pep8
+        environment:
+          TOXENV: pep8
+      # MySQL env for mysql queue tests
+      - image: circleci/mysql:8.0
+        environment:
+          MYSQL_ROOT_PASSWORD: rootpw
+          MYSQL_DATABASE: testqueue
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
     steps: *common_steps
 
 
   cover:
     docker:
       # Primary container image where all steps run.
       - image: circleci/python:3.5.9
-    environment:
-      - TOXENV: cover
+        environment:
+          TOXENV: cover
+      # MySQL env for mysql queue tests
+      - image: circleci/mysql:8.0
+        environment:
+          MYSQL_ROOT_PASSWORD: 123456
+          MYSQL_DATABASE: testqueue
+          MYSQL_USER: user
+          MYSQL_PASSWORD: passw0rd
     steps: *common_steps
 
 workflows:
@@ -94,7 +163,7 @@ workflows:
     jobs:
       - pep8
       - py27
-      - py34
+#      - py34
       - py35
       - py36
       - py37

README.rst

@@ -41,7 +41,7 @@ Join `persist-queue <https://join.slack
 
 Requirements
 ------------
-* Python 2.7 or Python 3.x.
+* Python 2.7 or Python 3.x (refer to `Deprecation`_ for future plan)
 * Full support for Linux.
 * Windows support (with `Caution`_ if ``persistqueue.Queue`` is used).
 
@@ -53,7 +53,12 @@ Features
 - Both filed based queues and sqlite3 based queues are supported
 - Filed based queue: multiple serialization protocol support: pickle(default), msgpack, json
 
-
+Deprecation
+-----------
+- `Python 3.4 release has reached end of life <https://www.python.org/downloads/release/python-3410/>`_ and
+  `DBUtils <https://webwareforpython.github.io/DBUtils/changelog.html>`_ ceased support for `Python 3.4`, `persist queue` drops the support for python 3.4 since version 0.8.0.
+  other queue implementations such as file based queue and sqlite3 based queue are still workable.
+- `Python 2 was sunset on January 1, 2020 <https://www.python.org/doc/sunset-python-2/>`_, `persist-queue` will drop any Python 2 support in future version `1.0.0`, no new feature will be developed under Python 2.
 
 Installation
 ------------
@@ -64,7 +69,7 @@ from pypi
 .. code-block:: console
 
     pip install persist-queue
-    # for msgpack support, use following command
+    # for msgpack and mysql support, use following command
     pip install persist-queue[extra]
 
 
@@ -426,6 +431,42 @@ multi-thread usage for **Queue**
 
     q.join()       # block until all tasks are done
 
+Example usage with a MySQL based queue
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*Available since: v0.8.0*
+
+.. code-block:: python
+
+    >>> import persistqueue
+    >>> db_conf = {
+    >>>     "host": "127.0.0.1",
+    >>>     "user": "user",
+    >>>     "passwd": "passw0rd",
+    >>>     "db_name": "testqueue",
+    >>>     # "name": "",
+    >>>     "port": 3306
+    >>> }
+    >>> q = persistqueue.MySQLQueue(name="testtable", **db_conf)
+    >>> q.put('str1')
+    >>> q.put('str2')
+    >>> q.put('str3')
+    >>> q.get()
+    'str1'
+    >>> del q
+
+
+Close the console, and then recreate the queue:
+
+.. code-block:: python
+
+   >>> import persistqueue
+   >>> q = persistqueue.MySQLQueue(name="testtable", **db_conf)
+   >>> q.get()
+   'str2'
+   >>>
+
+
 
 **note**
 

appveyor.yml

@@ -1,3 +1,6 @@
+services:
+  - mysql
+
 environment:
 
   matrix:
@@ -42,6 +45,10 @@ test_script:
   # Note that you must use the environment variable %PYTHON% to refer to
   # the interpreter you're using - Appveyor does not do anything special
   # to put the Python evrsion you want to use on PATH.
+  - ps: |
+      $env:MYSQL_PWD="Password12!"
+      $cmd = '"C:\Program Files\MySQL\MySQL Server 5.7\bin\mysql" -e "create database testqueue;" --user=root'
+      iex "& $cmd"
   - "%PYTHON%\\Scripts\\tox.exe"
 
 #on_success:

extra-requirements.txt

@@ -1 +1,3 @@
-msgpack>=0.5.6
+msgpack>=0.5.6
+PyMySQL
+DBUtils<3.0.0 # since 3.0.0 no longer supports Python2.x

persistqueue/__init__.py

@@ -1,7 +1,7 @@
 # coding=utf-8
 __author__ = 'Peter Wang'
 __license__ = 'BSD'
-__version__ = '0.7.0'
+__version__ = '0.8.0-alpha0'
 
 from .exceptions import Empty, Full  # noqa
 from .queue import Queue  # noqa
@@ -11,12 +11,13 @@
     from .sqlqueue import SQLiteQueue, FIFOSQLiteQueue, FILOSQLiteQueue, \
         UniqueQ  # noqa
     from .sqlackqueue import SQLiteAckQueue, UniqueAckQ
+    from .mysqlqueue import MySQLQueue
 except ImportError:
     import logging
 
     log = logging.getLogger(__name__)
     log.info("No sqlite3 module found, sqlite3 based queues are not available")
 
 __all__ = ["Queue", "SQLiteQueue", "FIFOSQLiteQueue", "FILOSQLiteQueue",
-           "UniqueQ", "PDict", "SQLiteAckQueue", "UniqueAckQ", "Empty", "Full",
-           "__author__", "__license__", "__version__"]
+           "UniqueQ", "PDict", "SQLiteAckQueue", "UniqueAckQ", "MySQLQueue",
+           "Empty", "Full", "__author__", "__license__", "__version__"]

persistqueue/common.py

@@ -1,4 +1,4 @@
-#! coding = utf-8
+# coding=utf-8
 
 import logging
 import pickle

persistqueue/exceptions.py

@@ -1,4 +1,4 @@
-#! coding = utf-8
+# coding=utf-8
 try:
     from queue import (
         Empty as StdEmpty,