From c548317f6a8eabd50babba372b38d9d23772726b Mon Sep 17 00:00:00 2001 From: Ash Vardanian <1983160+ashvardanian@users.noreply.github.com> Date: Fri, 11 Aug 2023 17:12:55 +0400 Subject: [PATCH] Refactor: Capitalization and Black formatting --- .github/workflows/prerelease.yml | 32 ++--- setup.py | 196 +++++++++++++++---------------- 2 files changed, 110 insertions(+), 118 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index baaeb36b..aa1cfbb7 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -23,7 +23,7 @@ jobs: build_ustore: - name: Build UStore + name: Build Libraries runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -36,7 +36,7 @@ jobs: - name: Prepare CMake, Conan and PyArrow run: python -m pip install --force-reinstall numpy pyarrow cmake conan==1.60.1 - - name: Install UStore dependencies using conan + - name: Install Conan dependencies run: | wget -q https://github.com/unum-cloud/ustore-deps/releases/download/v0.1.3/ustore_deps_x86_linux.tar.gz conan profile new --detect default @@ -45,7 +45,7 @@ jobs: conan install ustore_deps/0.12.1@unum/x86_linux -g cmake -s compiler.version=11 rm -rf ./ustore_deps_x86_linux.tar.gz - - name: Configure cmake + - name: Configure CMake run: | cmake -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \ -DUSTORE_BUILD_ENGINE_UCSET=1 \ @@ -66,8 +66,8 @@ jobs: path: build_release/build/bin/* - test_ustore: - name: Test UStore + test_cpp: + name: Test C++ runs-on: ubuntu-latest needs: build_ustore steps: @@ -81,10 +81,10 @@ jobs: with: name: binaries - - name: Change Permissions + - name: Change permissions run: chmod +x /home/runner/work/ustore/ustore/* - - name: Run Embeded servers unit tests + - name: Run embeded tests run: | mkdir -p ./tmp/ustore/ export USTORE_TEST_PATH="./tmp/" @@ -93,12 +93,12 @@ jobs: timeout -v --kill-after=5 300 $test done - - name: Run flight client test + - name: Run Arrow Flight tests run: /home/runner/work/ustore/ustore/test_units_ustore_flight_client - build_test_wheel: - name: Build and test Python wheels + test_python: + name: Test Python 3.9 & 3.10 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -109,15 +109,15 @@ jobs: - name: Prepare CMake, Conan and PyArrow run: python -m pip install --force-reinstall numpy pyarrow cmake conan==1.60.1 - - name: Build wheel + - name: Build Wheels run: | pip install cibuildwheel twine CIBW_BUILD="cp39-*" cibuildwheel --platform linux CIBW_BUILD="cp310-*" cibuildwheel --platform linux - build_test_java: - name: Build and test java + test_java: + name: Test Java runs-on: ubuntu-latest steps: - name: Checkout @@ -129,12 +129,12 @@ jobs: - name: Prepare CMake, Conan and PyArrow run: python -m pip install --force-reinstall numpy pyarrow cmake conan==1.60.1 - - name: Install default jdk + - name: Install JDK run: | sudo apt update sudo apt install default-jdk -y - - name: Install UStore dependencies using conan + - name: Install Conan dependencies run: | wget -q https://github.com/unum-cloud/ustore-deps/releases/download/v0.1.3/ustore_deps_x86_linux.tar.gz conan profile new --detect default @@ -143,7 +143,7 @@ jobs: conan install ustore_deps/0.12.1@unum/x86_linux -g cmake -s compiler.version=11 rm -rf ./ustore_deps_x86_linux.tar.gz - - name: Build and test java + - name: Test Java run: | mkdir -p ./tmp export USTORE_TEST_PATH="./tmp/" diff --git a/setup.py b/setup.py index b79e0106..98d4b010 100644 --- a/setup.py +++ b/setup.py @@ -12,17 +12,17 @@ from setuptools import setup, Extension from setuptools.command.build_ext import build_ext -__version__ = open('VERSION', 'r').read().strip() -__lib_name__ = 'ukv' +__version__ = open("VERSION", "r").read().strip() +__lib_name__ = "ustore" this_directory = os.path.abspath(dirname(__file__)) -with open(os.path.join(this_directory, 'README.md')) as f: +with open(os.path.join(this_directory, "README.md")) as f: long_description = f.read() class CMakeExtension(Extension): - def __init__(self, name, source_dir=''): + def __init__(self, name, source_dir=""): Extension.__init__(self, name, sources=[]) self.source_dir = os.path.abspath(source_dir) @@ -30,178 +30,170 @@ def __init__(self, name, source_dir=''): class CMakeBuild(build_ext): def build_extension(self, ext): self.parallel = multiprocessing.cpu_count() // 2 - extension_dir = os.path.abspath(dirname( - self.get_ext_fullpath(ext.name))) + extension_dir = os.path.abspath(dirname(self.get_ext_fullpath(ext.name))) # required for auto-detection & inclusion of auxiliary 'native' libs if not extension_dir.endswith(os.path.sep): extension_dir += os.path.sep - if 'USTORE_DEBUG_PYTHON' in os.environ: + if "USTORE_DEBUG_PYTHON" in os.environ: os.makedirs(extension_dir, exist_ok=True) - copy_tree('./build/lib/', extension_dir) + copy_tree("./build/lib/", extension_dir) return cmake_args = [ - f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extension_dir}', - f'-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={extension_dir}', - f'-DPYTHON_EXECUTABLE={sys.executable}', - '-DUSTORE_BUILD_ENGINE_UCSET=1', - '-DUSTORE_BUILD_ENGINE_LEVELDB=1', - '-DUSTORE_BUILD_ENGINE_ROCKSDB=1', - '-DUSTORE_BUILD_API_FLIGHT_CLIENT=1', - '-DUSTORE_BUILD_SDK_PYTHON=1', - '-DUSTORE_BUILD_TESTS=0', - '-DUSTORE_USE_PYARROW=1', - '-DUSE_CONAN=1' + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extension_dir}", + f"-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={extension_dir}", + f"-DPYTHON_EXECUTABLE={sys.executable}", + "-DUSTORE_BUILD_ENGINE_UCSET=1", + "-DUSTORE_BUILD_ENGINE_LEVELDB=1", + "-DUSTORE_BUILD_ENGINE_ROCKSDB=1", + "-DUSTORE_BUILD_API_FLIGHT_CLIENT=1", + "-DUSTORE_BUILD_SDK_PYTHON=1", + "-DUSTORE_BUILD_TESTS=0", + "-DUSTORE_USE_PYARROW=1", + "-DUSE_CONAN=1", ] # Adding CMake arguments set as environment variable # (needed e.g. to build for ARM OSx on conda-forge) - if 'CMAKE_ARGS' in os.environ: + if "CMAKE_ARGS" in os.environ: + cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] + elif "CMAKE_ARGS_F" in os.environ: cmake_args += [ - item for item in os.environ['CMAKE_ARGS'].split(' ') if item] - elif 'CMAKE_ARGS_F' in os.environ: - cmake_args += [item.strip() - for item in open(os.environ['CMAKE_ARGS_F']).read().split(' ') if item] + item.strip() + for item in open(os.environ["CMAKE_ARGS_F"]).read().split(" ") + if item + ] - if sys.platform.startswith('darwin'): + if sys.platform.startswith("darwin"): # Cross-compile support for macOS - respect ARCHFLAGS if set - archs = re.findall(r'-arch (\S+)', os.environ.get('ARCHFLAGS', '')) + archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", "")) if archs: - cmake_args += [ - '-DCMAKE_OSX_ARCHITECTURES={}'.format(';'.join(archs))] + cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level # across all generators. build_args = [] - if 'CMAKE_BUILD_PARALLEL_LEVEL' not in os.environ: + if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: # self.parallel is a Python 3 only way to set parallel jobs by hand # using -j in the build_ext call, not supported by pip or PyPA-build. - if hasattr(self, 'parallel') and self.parallel: - build_args += [f'-j{self.parallel}'] + if hasattr(self, "parallel") and self.parallel: + build_args += [f"-j{self.parallel}"] - subprocess.check_call(['cmake', ext.source_dir] + cmake_args) + subprocess.check_call(["cmake", ext.source_dir] + cmake_args) subprocess.check_call( - ['cmake', '--build', '.', '--target', ext.name.replace('ustore.', 'py_')] + build_args) + ["cmake", "--build", ".", "--target", ext.name.replace("ustore.", "py_")] + + build_args + ) def run(self): build_ext.run(self) - if 'USTORE_DEBUG_PYTHON' not in os.environ: - self.run_command('build_pyi') + if "USTORE_DEBUG_PYTHON" not in os.environ: + self.run_command("build_pyi") class BuildPyi(Command): - command_name = 'build_pyi' - description = 'Generates pyi files from built extensions' + command_name = "build_pyi" + description = "Generates pyi files from built extensions" def initialize_options(self): self.build_lib = None def finalize_options(self): - self.set_undefined_options('build', ('build_lib', 'build_lib')) + self.set_undefined_options("build", ("build_lib", "build_lib")) def run(self): # Gather information for needed stubs - data = {'mapping': {}, 'stubs': []} + data = {"mapping": {}, "stubs": []} env = os.environ.copy() # Requires information from build_ext to work - build_ext = self.distribution.get_command_obj('build_ext') + build_ext = self.distribution.get_command_obj("build_ext") if build_ext.inplace: - inst_command = self.distribution.get_command_obj('install') + inst_command = self.distribution.get_command_obj("install") inst_command.ensure_finalized() - data['out'] = inst_command.install_platlib + data["out"] = inst_command.install_platlib else: - data['out'] = self.build_lib + data["out"] = self.build_lib - wrappers = ['ustore'] + wrappers = ["ustore"] # Ensure that the associated packages can always be found locally for wrapper in wrappers: - pkgdir = wrapper.split('.') - init_py = abspath(join(self.build_lib, *pkgdir, '__init__.py')) - data['mapping'][wrapper] = init_py + pkgdir = wrapper.split(".") + init_py = abspath(join(self.build_lib, *pkgdir, "__init__.py")) + data["mapping"][wrapper] = init_py if not exists(init_py): - open(init_py, 'w').close() + open(init_py, "w").close() for ext in build_ext.extensions: fname = build_ext.get_ext_filename(ext.name) - data['mapping'][ext.name] = abspath(join(self.build_lib, fname)) - data['stubs'].append(ext.name) + data["mapping"][ext.name] = abspath(join(self.build_lib, fname)) + data["stubs"].append(ext.name) data_json = json.dumps(data) # Execute in a subprocess in case it crashes - args = [sys.executable, '-m', 'python.py11stubs', data_json] + args = [sys.executable, "-m", "python.py11stubs", data_json] proc = subprocess.run(args, env=env, capture_output=True) - print('exit status:', proc.returncode) - print('stdout:', proc.stdout.decode()) - print('stderr:', proc.stderr.decode()) + print("exit status:", proc.returncode) + print("stdout:", proc.stdout.decode()) + print("stderr:", proc.stderr.decode()) if proc.returncode != 0: raise RuntimeError( - 'Failed to generate .pyi file via %s' % (args,) + "Failed to generate .pyi file via %s" % (args,) ) from None # Create a py.typed for PEP 561 - with open(join(data['out'], 'ustore', 'py.typed'), 'w'): + with open(join(data["out"], "ustore", "py.typed"), "w"): pass setup( name=__lib_name__, version=__version__, - packages=['ustore'], - package_dir={'ustore': 'python/ustore'}, - - author='Ashot Vardanian', - author_email='info@unum.cloud', - url='https://github.com/unum-cloud/ustore', - description='Python bindings for Unum\'s UStore.', + packages=["ustore"], + package_dir={"ustore": "python/ustore"}, + author="Ash Vardanian", + author_email="info@unum.cloud", + url="https://github.com/unum-cloud/ustore", + description="Python bindings for Unum's UStore.", long_description=long_description, - long_description_content_type='text/markdown', - license='Apache-2.0', - + long_description_content_type="text/markdown", + license="Apache-2.0", classifiers=[ - 'Development Status :: 5 - Production/Stable', - - 'Natural Language :: English', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'License :: OSI Approved :: Apache Software License', - - 'Operating System :: Unix', - 'Operating System :: POSIX', - 'Operating System :: POSIX :: Linux', - - 'Programming Language :: C', - 'Programming Language :: C++', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - - 'Topic :: Database', - 'Topic :: Database :: Database Engines/Servers', - 'Topic :: Database :: Front-Ends', + "Development Status :: 5 - Production/Stable", + "Natural Language :: English", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "License :: OSI Approved :: Apache Software License", + "Operating System :: Unix", + "Operating System :: POSIX", + "Operating System :: POSIX :: Linux", + "Programming Language :: C", + "Programming Language :: C++", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Database", + "Topic :: Database :: Database Engines/Servers", + "Topic :: Database :: Front-Ends", ], - ext_modules=[ - CMakeExtension('ustore.ucset'), - CMakeExtension('ustore.rocksdb'), - CMakeExtension('ustore.leveldb'), - CMakeExtension('ustore.flight_client'), + CMakeExtension("ustore.ucset"), + CMakeExtension("ustore.rocksdb"), + CMakeExtension("ustore.leveldb"), + CMakeExtension("ustore.flight_client"), ], cmdclass={ - 'build_ext': CMakeBuild, - 'build_pyi': BuildPyi, + "build_ext": CMakeBuild, + "build_pyi": BuildPyi, }, zip_safe=False, - install_requires=[ - 'numpy>=1.16', - 'pyarrow>=10.0.1,<11' - ], - extras_require={'test': 'pytest'}, - python_requires='>=3.7', + install_requires=["numpy>=1.16", "pyarrow>=10.0.1,<11"], + extras_require={"test": "pytest"}, + python_requires=">=3.7", )