diff --git a/.gitignore b/.gitignore index b4f4030d5..d1163052b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ target/ .idea_modules/ *.versionsBackup shims-sources/ -env.sh \ No newline at end of file +env.sh +build/ +*.egg-info/ diff --git a/conf/env.sh.template b/conf/env.sh.template index 7eed7b84c..b12a0bc8a 100644 --- a/conf/env.sh.template +++ b/conf/env.sh.template @@ -16,7 +16,8 @@ export SPARK_HOME=$SPARK_HOME # Set OAP MLlib source code root directory SCRIPT_DIR=$( cd $(dirname ${BASH_SOURCE[0]}) && pwd ) -export OAP_MLLIB_ROOT=$(cd $SCRIPT_DIR/.. && pwd) +export OAP_MLLIB_ROOT:=$(cd $SCRIPT_DIR/.. && pwd) +export OAP_MLLIB_PATH:=$OAP_MLLIB_ROOT/mllib-dal/target # export OAP_MLLIB_ROOT=/path/to/oap-mllib/home # Set HDFS Root, should be hdfs://xxx or file://xxx @@ -33,7 +34,7 @@ export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop # Set JAR name & path OAP_MLLIB_JAR_NAME=oap-mllib-$OAP_MLLIB_VERSION.jar -OAP_MLLIB_JAR=$OAP_MLLIB_ROOT/mllib-dal/target/$OAP_MLLIB_JAR_NAME +OAP_MLLIB_JAR=$OAP_MLLIB_PATH/$OAP_MLLIB_JAR_NAME # Set Spark driver & executor classpaths # YARN mode: use absolute path for driver, relative path for executors # Standalone mode: use absolute path for both driver and executors diff --git a/jars_path.sh b/jars_path.sh new file mode 100644 index 000000000..161cc2ac2 --- /dev/null +++ b/jars_path.sh @@ -0,0 +1,2 @@ +export OAP_MLLIB_PATH=$(python -c "import oap_mllib +print(oap_mllib.get_oap_path())") diff --git a/python/oap_mllib/__init__.py b/python/oap_mllib/__init__.py new file mode 100644 index 000000000..1e936b031 --- /dev/null +++ b/python/oap_mllib/__init__.py @@ -0,0 +1,17 @@ +import os +import sys + + +def get_oap_path(): + current_module = sys.modules[__name__] + current_path = os.path.dirname(current_module.__file__) + return current_path + +def get_jars_path(): + current_path = get_oap_path() + jar_path = current_path + '/jars' + if os.path.exists(jar_path): + return jar_path + else: + print("error: can not find jars path") + return None diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 000000000..06e2527f5 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import importlib.util +import glob +import os +import sys +import subprocess + +from setuptools import setup +from setuptools.command.install import install +from shutil import copyfile, copytree, rmtree + +# A temporary path so we can access above the Python project root and fetch scripts and jars we need +TEMP_PATH = "deps" +OAP_HOME = os.path.abspath("../") + +# Provide guidance about how to use setup.py +incorrect_invocation_message = """ +If you are installing oap_mllib from source, you must first build oap_mllib and +run sdist. + After Building source code: + cd python + python setup.py sdist + pip install dist/*.tar.gz""" + +# Figure out is the jar compiled. +JAR_PATH = os.path.join(OAP_HOME, "mllib-dal/target") +EXAMPLES_PATH = os.path.join(OAP_HOME, "examples") + +JARS_TARGET = os.path.join(TEMP_PATH, "jars") +EXAMPLES_TARGET = os.path.join(TEMP_PATH, "examples") + + +try: + copytree(JAR_PATH, JARS_TARGET) + copytree(EXAMPLES_PATH, EXAMPLES_TARGET) + + with open('../README.md') as f: + long_description = f.read() + + VERSION = "1.6.0" + + setup( + name='oap_mllib', + version=VERSION, + description='OAP MLlib', + long_description=long_description, + long_description_content_type="text/markdown", + packages=['oap_mllib', + 'oap_mllib.jars', + 'oap_mllib.examples'], + install_requires=[], + package_dir={ + 'oap_mllib': 'oap_mllib', + 'oap_mllib.jars': 'deps/jars', + 'oap_mllib.examples': 'deps/examples' + }, + package_data={ + 'oap_mllib': ['*'], + 'oap_mllib.jars': ['*.jar'], + 'oap_mllib.examples': ['*'], + }, + python_requires='>=3.6', + ) +finally: + rmtree(os.path.join(TEMP_PATH, "jars")) + rmtree(os.path.join(TEMP_PATH, "examples")) + os.rmdir(TEMP_PATH)