Merge pull request #68 from briefercloud/fix-oracle

fix: upgrade SQLAlchemy so that it recognizes oracle dialect

apps/api/jupyter-requirements.txt

@@ -18,7 +18,7 @@ vegafusion-python-embed==1.5.0
 vl-convert-python==1.2.0
 tiktoken==0.5.2
 polars==0.19.19
-SQLAlchemy==1.4.50
+SQLAlchemy==2.0.34
 google-api-core==2.15.0
 google-api-python-client==1.6.7
 google-api-support==0.1.4
@@ -46,7 +46,8 @@ db-dtypes==1.2.0
 fastparquet==2024.2.0
 oracledb==2.2.0
 redshift-connector==2.0.917
-sqlalchemy-redshift==0.8.14
+# sqlalchemy-redshift==0.8.14
+git+https://github.com/briefercloud/sqlalchemy-redshift.git#egg=sqlalchemy-redshift
 trino==0.329.0
 duckdb==1.0.0
 openpyxl==3.1.2

apps/api/jupyter.Dockerfile

@@ -2,10 +2,6 @@ FROM python:3.9-slim
 
 WORKDIR /usr/src/app
 
-ARG JUPYTER_REQUIREMENTS_FILE=jupyter-requirements.txt
-
-COPY $JUPYTER_REQUIREMENTS_FILE ./requirements.txt
-
 RUN apt-get update && \
     apt-get install -y \
         libpq-dev \
@@ -27,6 +23,9 @@ RUN apt-get update && \
 ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
 ENV C_INCLUDE_PATH=/usr/include/gdal
 
+ARG JUPYTER_REQUIREMENTS_FILE=jupyter-requirements.txt
+COPY $JUPYTER_REQUIREMENTS_FILE ./requirements.txt
+
 RUN pip install --upgrade pip
 RUN pip install --no-cache-dir jupyter_server
 RUN pip install --no-cache-dir ipykernel

apps/api/package.json

@@ -49,7 +49,6 @@
     "lru-cache": "^10.2.2",
     "mysql2": "^3.10.1",
     "node-fetch": "^3.3.2",
-    "oracledb": "^6.5.1",
     "p-all": "^5.0.0",
     "p-queue": "^8.0.1",
     "parse-duration": "^1.1.0",
@@ -85,7 +84,6 @@
     "@types/multer": "^1.4.10",
     "@types/node": "^20.9.0",
     "@types/node-fetch": "^2.6.9",
-    "@types/oracledb": "^6.5.0",
     "@types/ramda": "^0.29.9",
     "@types/semver": "^7.5.8",
     "@types/split2": "^4.2.3",

apps/api/src/datasources/oracle.ts

@@ -1,82 +1,30 @@
 import config from '../config/index.js'
-import prisma, {
-  DataSource,
-  OracleDataSource,
-  getOraclePassword,
-} from '@briefer/database'
-import oracle from 'oracledb'
-import { logger } from '../logger.js'
+import prisma, { DataSource, OracleDataSource } from '@briefer/database'
 import { DataSourceStatus } from './index.js'
-import { DataSourceConnectionError } from '@briefer/types'
-
-async function getConnectionAttributes(ds: OracleDataSource) {
-  const password = await getOraclePassword(
-    ds,
-    config().DATASOURCES_ENCRYPTION_KEY
-  )
-
-  let connectData = ''
-  if (ds.serviceName) {
-    connectData += `(service_name=${ds.serviceName})`
-  }
-  if (ds.sid) {
-    connectData += `(sid=${ds.sid})`
-  }
-
-  if (connectData === '' && ds.database) {
-    return {
-      user: ds.username,
-      password,
-      connectString: `${ds.host}:${ds.port}/${ds.database}`,
-    }
-  }
-
-  if (connectData !== '') {
-    connectData = `(connect_data=${connectData})`
-  }
-
-  return {
-    user: ds.username,
-    password,
-    connectString: `(description=(retry_count=3)(retry_delay=3)(address=(protocol=tcps)(port=${ds.port})(host=${ds.host}))${connectData}(security=(ssl_server_dn_match=yes)))`,
-  }
-}
+import { getOracleSchema, pingOracle } from '../python/query/oracle.js'
+import { logger } from '../logger.js'
+import { DataSourceStructure } from '@briefer/types'
 
-export async function ping(datasource: OracleDataSource): Promise<DataSource> {
+export async function ping(ds: OracleDataSource): Promise<DataSource> {
   const lastConnection = new Date()
+  const err = await pingOracle(ds, config().DATASOURCES_ENCRYPTION_KEY)
 
-  try {
-    const attrs = await getConnectionAttributes(datasource)
-    const connection = await oracle.getConnection(attrs)
-
-    await connection.ping()
+  logger.error({ err }, 'ping error')
 
-    return updateConnStatus(datasource, {
+  if (!err) {
+    return updateConnStatus(ds, {
       connStatus: 'online',
       lastConnection,
     })
-  } catch (err) {
-    logger.info({ err, id: datasource.id }, 'Error pinging Oracle')
-    const parsedErr = DataSourceConnectionError.safeParse(err)
-    if (!parsedErr.success) {
-      logger.error(
-        { err, id: datasource.id },
-        'Error parsing Oracle connection error'
-      )
-      return updateConnStatus(datasource, {
-        connStatus: 'offline',
-        connError: {
-          name: 'UnknownError',
-          message: 'Unknown error',
-        },
-      })
-    }
-
-    return updateConnStatus(datasource, {
-      connStatus: 'offline',
-      connError: parsedErr.data,
-    })
   }
+
+  return updateConnStatus(ds, { connStatus: 'offline', connError: err })
+}
+
+export async function getSchema(
+  ds: OracleDataSource
+): Promise<DataSourceStructure> {
+  return getOracleSchema(ds, config().DATASOURCES_ENCRYPTION_KEY)
 }
 
 export async function updateConnStatus(

apps/api/src/datasources/structure.ts

@@ -5,6 +5,7 @@ import * as psql from './psql.js'
 import * as athena from './athena.js'
 import * as mysql from './mysql.js'
 import * as trino from './trino.js'
+import * as oracle from './oracle.js'
 import { DataSourceStructure, jsonString } from '@briefer/types'
 import { logger } from '../logger.js'
 import { z } from 'zod'
@@ -202,7 +203,7 @@ async function fetchStructure(
         return await trino.getSchema(ds.data)
       }
       case 'oracle': {
-        return null
+        return await oracle.getSchema(ds.data)
       }
     }
   } catch (err) {

apps/api/src/python/query/oracle.ts

@@ -1,7 +1,15 @@
 import { v4 as uuidv4 } from 'uuid'
 import { OracleDataSource, getDatabaseURL } from '@briefer/database'
-import { RunQueryResult, SuccessRunQueryResult } from '@briefer/types'
-import { makeSQLAlchemyQuery } from './sqlalchemy.js'
+import {
+  DataSourceStructure,
+  RunQueryResult,
+  SuccessRunQueryResult,
+} from '@briefer/types'
+import {
+  getSQLAlchemySchema,
+  makeSQLAlchemyQuery,
+  pingSQLAlchemy,
+} from './sqlalchemy.js'
 
 export async function makeOracleQuery(
   workspaceId: string,
@@ -32,3 +40,21 @@ export async function makeOracleQuery(
     onProgress
   )
 }
+
+export function pingOracle(
+  ds: OracleDataSource,
+  encryptionKey: string
+): Promise<null | Error> {
+  return pingSQLAlchemy(
+    ds.workspaceId,
+    { type: 'oracle', data: ds },
+    encryptionKey
+  )
+}
+
+export function getOracleSchema(
+  ds: OracleDataSource,
+  encryptionKey: string
+): Promise<DataSourceStructure> {
+  return getSQLAlchemySchema({ type: 'oracle', data: ds }, encryptionKey)
+}