Allow changing GitHub server hostname (for self-hosted/enterprise instances)

[kquinsland]

Why do we need this ?

I would like to use this tool with an internally hosted GitHub instance.

Acceptance Criteria

I would be OK with this being something that can be set via env-var or config file.
It does not need to be something that supports live-reloading or even something that's likely to be changed often.

Further Comments / References

Nothing immediately comes to mind; it should just be a hostname change. Nothing about the API changes with self-hosted.

[jayantbh]

Hey @kquinsland, we've not gotten around to this yet, but perhaps you could create a fork and replace references to github.com with your own host. As long as the API contracts are identical, I feel it'll work out of the box. :)

If it does work for you, let me know and we'll add it to the README or Wiki. :)

[kquinsland]

If it does work for you, let me know and we'll add it to the README or Wiki. :)

I tried this and found I had to make quite a few small patches here and there before I ran out of time. I don't recall precisely where I got stuck (it's been 6 weeks...) but there was some issue(s) I had w/r/t auth that I wasn't able to fully grock.

[kquinsland]

Had a spare sec to go digging.

# So you can see the commit to compare my changes against
❯ git rev-parse HEAD
8605f85a1053d94a49aea9e63d34de153cca35d8
# Generate a diff of all the things I had to change
❯ git diff --name-only | while read -r file; do git diff "$file"; echo ""; done > patches-i-made

And here's patches-i-made:

❯ cat patches-i-made
diff --git a/Dockerfile.dev b/Dockerfile.dev
index 1c747f8..e5c4922 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -37,6 +37,9 @@ RUN /opt/venv/bin/pip install --upgrade pip
 RUN --mount=type=cache,target=/root/.cache/pip /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt
 
 WORKDIR /app
+# TODO: this should be intelligent; uname --machine or similar?
+RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64
+RUN chmod +x /usr/local/bin/dbmate
 RUN mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob
 RUN chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh
 RUN mv /app/setup_utils/supervisord.conf /etc/supervisord.conf

diff --git a/Dockerfile.prebuilt b/Dockerfile.prebuilt
index e942b33..c22a962 100644
--- a/Dockerfile.prebuilt
+++ b/Dockerfile.prebuilt
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         curl \
     && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
     && apt-get install -y nodejs \
-    && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \
+    && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64 \
     && chmod +x /usr/local/bin/dbmate \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*

diff --git a/backend/analytics_server/mhq/exapi/github.py b/backend/analytics_server/mhq/exapi/github.py
index 50cd192..194df9c 100644
--- a/backend/analytics_server/mhq/exapi/github.py
+++ b/backend/analytics_server/mhq/exapi/github.py
@@ -25,7 +25,8 @@ class GithubApiService:
     def __init__(self, access_token: str):
         self._token = access_token
         self._g = Github(self._token, per_page=PAGE_SIZE)
-        self.base_url = "https://api.github.com"
+        # self.base_url = "https://api.github.com"
+        self.base_url = "https://github.corp-github-server.net/api/v3"
         self.headers = {"Authorization": f"Bearer {self._token}"}
 
     @contextlib.contextmanager
@@ -53,6 +54,7 @@ class GithubApiService:
         except GithubException as e:
             raise e
 
+        LOG.error(f"FIND ME. Orgs got: {orgs}")
         return orgs
 
     def get_repos(
@@ -89,6 +91,7 @@ class GithubApiService:
         try:
             return self._g.get_repo(f"{org_login}/{repo_name}")
         except GithubException as e:
+            LOG.info(f"Something Broke! e: {e}")
             raise e
 
     def get_repo_contributors(self, github_repo: GithubRepository) -> [Tuple[str, int]]:

diff --git a/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
index 17f042c..6e6a2dc 100644
--- a/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
+++ b/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
@@ -71,11 +71,18 @@ class GithubETLHandler(CodeProviderETLHandler):
         :param org_repos: List of OrgRepo objects
         :returns: List of GitHub repos as OrgRepo objects
         """
+        LOG.info(f"Inside get_org_repos. org_repos: {org_repos}")
+
+        for org_repo in org_repos:
+            LOG.info(f"Inside get_org_repos. org_repo: {org_repo}")
+            LOG.info(f"Inside get_org_repos. org_name: {org_repo.org_name}")
+            LOG.info(f"Inside get_org_repos. name: {org_repo.name}")
         github_repos: List[GithubRepository] = [
             self._api.get_repo(org_repo.org_name, org_repo.name)
             for org_repo in org_repos
         ]
-
+        # Boom. Confirmed that we never make it this far!
+        LOG.info(f"Inside get_org_repos. github_repos: {github_repos}")
         repo_idempotency_key_org_repo_map = {
             org_repo.idempotency_key: org_repo for org_repo in org_repos
         }

diff --git a/backend/analytics_server/mhq/service/code/sync/etl_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_handler.py
index 47dfa95..a15e8d8 100644
--- a/backend/analytics_server/mhq/service/code/sync/etl_handler.py
+++ b/backend/analytics_server/mhq/service/code/sync/etl_handler.py
@@ -47,8 +47,11 @@ class CodeETLHandler:
         if not self.etl_service.check_pat_validity():
             LOG.error("Invalid PAT for code provider")
             return
+        LOG.info(f"about to get org_repos....")
         org_repos: List[OrgRepo] = self._sync_org_repos(org_id, provider)
+        LOG.info(f"got the following org_repos: {org_repos}")
         for org_repo in org_repos:
+            LOG.info(f"Attempting work on: {org_repo}")
             try:
                 self._sync_repo_pull_requests_data(org_repo)
             except Exception as e:
@@ -58,12 +61,16 @@ class CodeETLHandler:
                 continue
 
     def _sync_org_repos(self, org_id: str, provider: CodeProvider) -> List[OrgRepo]:
+        LOG.info(f"about to try...")
         try:
             org_repos = self.code_repo_service.get_active_org_repos_for_provider(
                 org_id, provider
             )
+            LOG.info(f"step 1, org_repos: {org_repos}")
             org_repos = self.etl_service.get_org_repos(org_repos)
+            LOG.info(f"step 2, org_repos: {org_repos}")
             self.code_repo_service.update_org_repos(org_repos)
+            LOG.info(f"step 3, org_repos: {org_repos}")
             return org_repos
         except Exception as e:
             LOG.error(f"Error syncing org repos for org {org_id}: {str(e)}")
@@ -136,6 +143,7 @@ def sync_code_repos(org_id: str):
     etl_factory = CodeETLFactory(org_id)
 
     for provider in code_providers:
+        LOG.info(f"Iterating through: {provider}")
         try:
             code_etl_handler = CodeETLHandler(
                 CodeRepoService(),
@@ -144,6 +152,7 @@ def sync_code_repos(org_id: str):
                 get_bookmark_service(),
                 get_settings_service(),
             )
+            LOG.info(f"code_etl_handler instantiated! {code_etl_handler}")
             code_etl_handler.sync_org_repos(org_id, CodeProvider(provider))
             LOG.info(f"Synced org repos for provider {provider}")
         except Exception as e:

diff --git a/backend/analytics_server/mhq/service/sync_data.py b/backend/analytics_server/mhq/service/sync_data.py
index 35e9abd..bb7bbd4 100644
--- a/backend/analytics_server/mhq/service/sync_data.py
+++ b/backend/analytics_server/mhq/service/sync_data.py
@@ -16,6 +16,7 @@ def trigger_data_sync(org_id: str):
     LOG.info(f"Starting data sync for org {org_id}")
     for sync_func in sync_sequence:
         try:
+            LOG.info(f"Starting sync_func: {sync_func}...")
             sync_func(org_id)
             LOG.info(f"Data sync for {sync_func.__name__} completed successfully")
         except Exception as e:

diff --git a/database-docker/Dockerfile b/database-docker/Dockerfile
index c12e785..25c9589 100644
--- a/database-docker/Dockerfile
+++ b/database-docker/Dockerfile
@@ -1,7 +1,7 @@
-FROM debian:buster
+FROM debian:bookworm
 ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list
 RUN apt-get -y install gnupg
 RUN apt-get -y install curl
 RUN apt-get -y install wget
@@ -9,7 +9,8 @@ RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 RUN apt-get update
 RUN apt-get -y install postgresql-14
-RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64
+# TODO: this should be intelligent; uname --machine or similar?
+RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64
 RUN chmod +x /usr/local/bin/dbmate
 WORKDIR /
 RUN mkdir ./db

diff --git a/env.example b/env.example
index 99fabc5..f0044d9 100644
--- a/env.example
+++ b/env.example
@@ -1,4 +1,4 @@
-ENVIRONMENT=dev
+ENVIRONMENT=prod
 POSTGRES_DB_ENABLED=true
 DB_INIT_ENABLED=true
 REDIS_ENABLED=true

diff --git a/web-server/pages/api/internal/[org_id]/utils.ts b/web-server/pages/api/internal/[org_id]/utils.ts
index 3503c01..359e52a 100644
--- a/web-server/pages/api/internal/[org_id]/utils.ts
+++ b/web-server/pages/api/internal/[org_id]/utils.ts
@@ -6,7 +6,8 @@ import { Integration } from '@/constants/integrations';
 import { BaseRepo } from '@/types/resources';
 import { db } from '@/utils/db';
 
-const GITHUB_API_URL = 'https://api.github.com/graphql';
+// const GITHUB_API_URL = 'https://api.github.com/graphql';
+const GITHUB_API_URL = 'https://github.corp-github-server.net/api/graphql';
 
 type GithubRepo = {
   name: string;
@@ -53,7 +54,8 @@ export const searchGithubRepos = async (
 };
 
 const searchRepoWithURL = async (searchString: string) => {
-  const apiUrl = `https://api.github.com/repos/${searchString}`;
+  //const apiUrl = `https://api.github.com/repos/${searchString}`;
+  const apiUrl = `https://github.corp-github-server.net/api/v3/repos/${searchString}`;
   const response = await axios.get<GithubRepo>(apiUrl);
   const repo = response.data;
   return [

diff --git a/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx b/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
index 3ed2ab9..06b087f 100644
--- a/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
+++ b/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
@@ -51,7 +51,7 @@ export const ConfigureGithubModalBody: FC<{
     depFn(isLoading.true);
     checkGitHubValidity(token.value)
       .then(async (isValid) => {
-        if (!isValid) throw new Error('Invalid token');
+        if (!isValid) throw new Error('Invalid token-TEST');
       })
       .then(async () => {
         try {

diff --git a/web-server/src/utils/auth-supplementary.ts b/web-server/src/utils/auth-supplementary.ts
index e5aec2c..7b9c329 100644
--- a/web-server/src/utils/auth-supplementary.ts
+++ b/web-server/src/utils/auth-supplementary.ts
@@ -28,7 +28,8 @@ export const INTEGRATION_CONFLICT_COLUMNS = ['org_id', 'name'];
 
 export const validateGithubToken = async (token: string) => {
   try {
-    const response = await axios.get('https://api.github.com/user/repos', {
+    const response = await axios.get('https://github.corp-github-server.net/api/v3/user/repos', {
+    //const response = await axios.get('https://api.github.com/user/repos', {
       headers: {
         // @ts-ignore
         Authorization: `token ${dec(token)}`

diff --git a/web-server/src/utils/auth.ts b/web-server/src/utils/auth.ts
index 4cc2d93..5d751ed 100644
--- a/web-server/src/utils/auth.ts
+++ b/web-server/src/utils/auth.ts
@@ -31,23 +31,35 @@ export async function checkGitHubValidity(
   good_stuff: string
 ): Promise<boolean> {
   try {
-    await axios.get('https://api.github.com/user', {
+    //const response = await axios.get('github.corp-github-server.net/api/v3/', {
+    // works, but not correct? await axios.get('https://github.corp-github-server.net/api/v3/user', {
+    await axios.get('https://github.corp-github-server.net/api/v3', {
+    //await axios.get('https://api.github.com/user', {
       headers: {
         Authorization: `token ${good_stuff}`
       }
     });
     return true;
   } catch (error) {
-    return false;
+    console.error(`Couldn't validate GH token. error: `, error);
+
+    //return false;
+    // For reasons that I don't understand, we still end up here.
+    // Despite the face that I can curl properly inside the container and get the
+    // results that I expect / show auth is ok...
+    return true;
   }
 }
 
 const PAT_SCOPES = ['read:org', 'read:user', 'repo', 'workflow'];
 export const getMissingPATScopes = async (pat: string) => {
   try {
-    const response = await axios.get('https://api.github.com', {
+    const response = await axios.get('https://github.corp-github-server.net/api/v3', {
+    //const response = await axios.get('https://api.github.com', {
       headers: {
-        Authorization: `token ${pat}`
+        //Authorization: `token ${pat}`
+        Authorization: `token ghp_zTRQW<...token here ...>`
+
       }
     });
 
@@ -57,7 +69,9 @@ export const getMissingPATScopes = async (pat: string) => {
     const userScopes = scopesString.split(',').map((scope) => scope.trim());
     return PAT_SCOPES.filter((scope) => !userScopes.includes(scope));
   } catch (error) {
-    throw new Error('Failed to get missing PAT scopes', error);
+    // i - again - do not know why I end up here...
+    return PAT_SCOPES.filter((scope) => !PAT_SCOPES.includes(scope));
+    //throw new Error('Failed to get missing PAT scopes', error);
   }
 };

The changes fall into two broad groups:

1. Changes required to get this booting inside a modern debian (bookworm should be used, buster is EOL) container and on an ARM based mac.
2. Changes required to get past steps that failed successfully / tracing code flow / trying to figure out why the app was telling me "no" but the token I was using ... appeared to work when doing a curl .... request manually.

[jayantbh]

Apologies this fell through the cracks. But thank you for sharing the details!
We'll check this out, and if it works alright, we'll incorporate it into the repo!

[adnanhashmi09]

Hey @kquinsland
With our latest release we now have support for enterprise github. Try it out and let us know if this works for you. 😀