Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add init_env as utility to set environment and connect to HANA db #14

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add init_env as utility to set environment and connect to HANA db #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b7927b4
Add an alternative to install python packages
michael1011101 Sep 23, 2024
a8e79cc
Add init_env as utility to
michael1011101 Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions exercises/03-setup-python-environment.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ source ~/projects/generative-ai-codejam/env/bin/activate
![venv](images/venv.png)

👉 Install the Generative AI Hub [Python SDK](https://pypi.org/project/generative-ai-hub-sdk/) (and the other packages listed below) using the following `pip install` commands.
(Or you can use `pip install -r requirements.txt` to install all required python packages.)

```bash
pip install --require-virtualenv generative-ai-hub-sdk[all]
Expand Down
24 changes: 4 additions & 20 deletions exercises/04-prompt-llm.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,6 @@
"That way the Generative AI Hub [Python SDK](https://pypi.org/project/generative-ai-hub-sdk/) will connect to Generative AI Hub."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"\n",
"with open('/home/user/projects/generative-ai-codejam/.aicore-config.json', 'r') as config_file:\n",
" config_data = json.load(config_file)\n",
"\n",
"os.environ[\"AICORE_AUTH_URL\"]=config_data[\"url\"]+\"/oauth/token\"\n",
"os.environ[\"AICORE_CLIENT_ID\"]=config_data[\"clientid\"]\n",
"os.environ[\"AICORE_CLIENT_SECRET\"]=config_data[\"clientsecret\"]\n",
"os.environ[\"AICORE_BASE_URL\"]=config_data[\"serviceurls\"][\"AI_API_URL\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -43,12 +25,14 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import init_env\n",
"import variables\n",
"os.environ[\"AICORE_RESOURCE_GROUP\"]=variables.RESOURCE_GROUP"
"\n",
"init_env.set_environment_variables()"
]
},
{
Expand Down
17 changes: 4 additions & 13 deletions exercises/05-create-embeddings.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,10 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import init_env\n",
"import variables\n",
"\n",
"with open('/home/user/projects/generative-ai-codejam/.aicore-config.json', 'r') as config_file:\n",
" config_data = json.load(config_file)\n",
"\n",
"os.environ[\"AICORE_AUTH_URL\"]=config_data[\"url\"]+\"/oauth/token\"\n",
"os.environ[\"AICORE_CLIENT_ID\"]=config_data[\"clientid\"]\n",
"os.environ[\"AICORE_CLIENT_SECRET\"]=config_data[\"clientsecret\"]\n",
"os.environ[\"AICORE_BASE_URL\"]=config_data[\"serviceurls\"][\"AI_API_URL\"]\n",
"\n",
"os.environ[\"AICORE_RESOURCE_GROUP\"]=variables.RESOURCE_GROUP\n",
"init_env.set_environment_variables()\n",
"\n",
"from gen_ai_hub.proxy.native.openai import embeddings"
]
Expand All @@ -49,8 +40,8 @@
"source": [
"def get_embedding(input_text):\n",
" response = embeddings.create(\n",
" input=input_text, \n",
" model_name=\"text-embedding-ada-002\"\n",
" input=input_text,\n",
" model_name=variables.EMBEDDING_MODEL_NAME\n",
" )\n",
" embedding = response.data[0].embedding\n",
" return embedding\n"
Expand Down
35 changes: 3 additions & 32 deletions exercises/06-store-embeddings-hana.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,10 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import init_env\n",
"import variables\n",
"\n",
"with open('/home/user/projects/generative-ai-codejam/.aicore-config.json', 'r') as config_file:\n",
" config_data = json.load(config_file)\n",
"\n",
"os.environ[\"AICORE_AUTH_URL\"]=config_data[\"url\"]+\"/oauth/token\"\n",
"os.environ[\"AICORE_CLIENT_ID\"]=config_data[\"clientid\"]\n",
"os.environ[\"AICORE_CLIENT_SECRET\"]=config_data[\"clientsecret\"]\n",
"os.environ[\"AICORE_BASE_URL\"]=config_data[\"serviceurls\"][\"AI_API_URL\"]\n",
"\n",
"os.environ[\"AICORE_RESOURCE_GROUP\"]=variables.RESOURCE_GROUP\n",
"init_env.set_environment_variables()\n",
"\n",
"# OpenAIEmbeddings to create text embeddings\n",
"#from gen_ai_hub.proxy.native.openai import OpenAIEmbeddings\n",
Expand Down Expand Up @@ -67,33 +58,13 @@
"```"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# connect to HANA instance\n",
"from hdbcli import dbapi\n",
"import configparser"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"config = configparser.ConfigParser()\n",
"config.read('/home/user/projects/generative-ai-codejam/.user.ini')\n",
"connection = dbapi.connect(\n",
" address=config.get('hana', 'url'), \n",
" port=config.get('hana', 'port'), \n",
" user=config.get('hana', 'user'),\n",
" password=config.get('hana', 'passwd'),\n",
" autocommit=True,\n",
" sslValidateCertificate=False\n",
")\n",
"connection = init_env.connect_to_hana_db()\n",
"\n",
"connection.isconnected()"
]
Expand Down
26 changes: 4 additions & 22 deletions exercises/07-RAG.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,10 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import configparser\n",
"import init_env\n",
"import variables\n",
"\n",
"with open('/home/user/projects/generative-ai-codejam/.aicore-config.json', 'r') as config_file:\n",
" config_data = json.load(config_file)\n",
"\n",
"os.environ[\"AICORE_AUTH_URL\"]=config_data[\"url\"]+\"/oauth/token\"\n",
"os.environ[\"AICORE_CLIENT_ID\"]=config_data[\"clientid\"]\n",
"os.environ[\"AICORE_CLIENT_SECRET\"]=config_data[\"clientsecret\"]\n",
"os.environ[\"AICORE_BASE_URL\"]=config_data[\"serviceurls\"][\"AI_API_URL\"]\n",
"\n",
"os.environ[\"AICORE_RESOURCE_GROUP\"]=variables.RESOURCE_GROUP\n",
"init_env.set_environment_variables()\n",
"\n",
"from gen_ai_hub.proxy.langchain.openai import ChatOpenAI\n",
"from gen_ai_hub.proxy.langchain.openai import OpenAIEmbeddings\n",
Expand Down Expand Up @@ -59,16 +49,8 @@
"metadata": {},
"outputs": [],
"source": [
"config = configparser.ConfigParser()\n",
"config.read('/home/user/projects/generative-ai-codejam/.user.ini')\n",
"connection = dbapi.connect(\n",
" address=config.get('hana', 'url'), \n",
" port=config.get('hana', 'port'), \n",
" user=config.get('hana', 'user'),\n",
" password=config.get('hana', 'passwd'),\n",
" autocommit=True,\n",
" sslValidateCertificate=False\n",
")"
"# Connect to HANA\n",
"connection = init_env.connect_to_hana_db()"
]
},
{
Expand Down
24 changes: 3 additions & 21 deletions exercises/08-semantic-chunking.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,10 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import init_env\n",
"import variables\n",
"\n",
"with open('/home/user/projects/generative-ai-codejam/.aicore-config.json', 'r') as config_file:\n",
" config_data = json.load(config_file)\n",
"\n",
"os.environ[\"AICORE_AUTH_URL\"]=config_data[\"url\"]+\"/oauth/token\"\n",
"os.environ[\"AICORE_CLIENT_ID\"]=config_data[\"clientid\"]\n",
"os.environ[\"AICORE_CLIENT_SECRET\"]=config_data[\"clientsecret\"]\n",
"os.environ[\"AICORE_BASE_URL\"]=config_data[\"serviceurls\"][\"AI_API_URL\"]\n",
"\n",
"os.environ[\"AICORE_RESOURCE_GROUP\"]=variables.RESOURCE_GROUP\n",
"init_env.set_environment_variables()\n",
"\n",
"from langchain_community.document_loaders import PyPDFDirectoryLoader\n",
"from gen_ai_hub.proxy.langchain.openai import OpenAIEmbeddings\n",
Expand All @@ -47,16 +38,7 @@
"outputs": [],
"source": [
"# Connect to HANA\n",
"config = configparser.ConfigParser()\n",
"config.read('/home/user/projects/generative-ai-codejam/.user.ini')\n",
"connection = dbapi.connect(\n",
" address=config.get('hana', 'url'), \n",
" port=config.get('hana', 'port'), \n",
" user=config.get('hana', 'user'),\n",
" password=config.get('hana', 'passwd'),\n",
" autocommit=True,\n",
" sslValidateCertificate=False\n",
")"
"connection = init_env.connect_to_hana_db()"
]
},
{
Expand Down
14 changes: 3 additions & 11 deletions exercises/09-use-multimodal-models.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,12 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import base64\n",
"import variables\n",
"\n",
"with open('/home/user/projects/generative-ai-codejam/.aicore-config.json', 'r') as config_file:\n",
" config_data = json.load(config_file)\n",
"\n",
"os.environ[\"AICORE_AUTH_URL\"]=config_data[\"url\"]+\"/oauth/token\"\n",
"os.environ[\"AICORE_CLIENT_ID\"]=config_data[\"clientid\"]\n",
"os.environ[\"AICORE_CLIENT_SECRET\"]=config_data[\"clientsecret\"]\n",
"os.environ[\"AICORE_BASE_URL\"]=config_data[\"serviceurls\"][\"AI_API_URL\"]\n",
"import init_env\n",
"import variables\n",
"\n",
"os.environ[\"AICORE_RESOURCE_GROUP\"]=variables.RESOURCE_GROUP\n",
"init_env.set_environment_variables()\n",
"\n",
"from gen_ai_hub.proxy.langchain.openai import ChatOpenAI"
]
Expand Down
34 changes: 34 additions & 0 deletions exercises/init_env.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import os
import json
import configparser

from hdbcli import dbapi

import variables

ROOT_PATH_DIR = os.path.dirname(os.getcwd())
AICORE_CONFIG_FILENAME = '.aicore-config.json'
USER_CONFIG_FILENAME = '.user.ini'

def set_environment_variables() -> None:
with open(os.path.join(ROOT_PATH_DIR, AICORE_CONFIG_FILENAME), 'r') as config_file:
config_data = json.load(config_file)

os.environ["AICORE_AUTH_URL"]=config_data["url"]+"/oauth/token"
os.environ["AICORE_CLIENT_ID"]=config_data["clientid"]
os.environ["AICORE_CLIENT_SECRET"]=config_data["clientsecret"]
os.environ["AICORE_BASE_URL"]=config_data["serviceurls"]["AI_API_URL"]

os.environ["AICORE_RESOURCE_GROUP"]=variables.RESOURCE_GROUP

def connect_to_hana_db() -> dbapi.Connection:
config = configparser.ConfigParser()
config.read(os.path.join(ROOT_PATH_DIR, USER_CONFIG_FILENAME))
return dbapi.connect(
address=config.get('hana', 'url'),
port=config.get('hana', 'port'),
user=config.get('hana', 'user'),
password=config.get('hana', 'passwd'),
autocommit=True,
sslValidateCertificate=False
)
1 change: 1 addition & 0 deletions exercises/variables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
RESOURCE_GROUP = "YOUR RESOURCE GROUP GOES HERE"
EMBEDDING_MODEL_NAME = "text-embedding-ada-002" # default is "text-embedding-ada-002"
EMBEDDING_DEPLOYMENT_ID = "YOUR DEPLOYMENT ID GOES HERE" # e.g. d7b8e46fc3d5c25
LLM_DEPLOYMENT_ID = "YOUR DEPLOYMENT ID GOES HERE" # e.g. d01dff41125cfa27
EMBEDDING_TABLE = "EMBEDDINGS_CODEJAM_NORA"
Expand Down
14 changes: 14 additions & 0 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# The Generative AI Hub Python SDK
generative-ai-hub-sdk[all]

# The HANA client for Python
hdbcli

# The SciPy package
scipy

# The pypdf package
pypdf

# The LangChain Experimental
langchain-experimental