Allow changing GitHub server hostname (for self-hosted/enterprise instances) #619
Labels
enhancement
New feature or request
Comments
|
Hey @kquinsland, we've not gotten around to this yet, but perhaps you could create a fork and replace references to If it does work for you, let me know and we'll add it to the README or Wiki. :) |
I tried this and found I had to make quite a few small patches here and there before I ran out of time. I don't recall precisely where I got stuck (it's been 6 weeks...) but there was some issue(s) I had w/r/t auth that I wasn't able to fully grock. |
|
Had a spare sec to go digging. # So you can see the commit to compare my changes against
❯ git rev-parse HEAD
8605f85a1053d94a49aea9e63d34de153cca35d8
# Generate a diff of all the things I had to change
❯ git diff --name-only | while read -r file; do git diff "$file"; echo ""; done > patches-i-madeAnd here's ❯ cat patches-i-made
diff --git a/Dockerfile.dev b/Dockerfile.dev
index 1c747f8..e5c4922 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -37,6 +37,9 @@ RUN /opt/venv/bin/pip install --upgrade pip
RUN --mount=type=cache,target=/root/.cache/pip /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt
WORKDIR /app
+# TODO: this should be intelligent; uname --machine or similar?
+RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64
+RUN chmod +x /usr/local/bin/dbmate
RUN mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob
RUN chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh
RUN mv /app/setup_utils/supervisord.conf /etc/supervisord.conf
diff --git a/Dockerfile.prebuilt b/Dockerfile.prebuilt
index e942b33..c22a962 100644
--- a/Dockerfile.prebuilt
+++ b/Dockerfile.prebuilt
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
&& curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
&& apt-get install -y nodejs \
- && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \
+ && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64 \
&& chmod +x /usr/local/bin/dbmate \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
diff --git a/backend/analytics_server/mhq/exapi/github.py b/backend/analytics_server/mhq/exapi/github.py
index 50cd192..194df9c 100644
--- a/backend/analytics_server/mhq/exapi/github.py
+++ b/backend/analytics_server/mhq/exapi/github.py
@@ -25,7 +25,8 @@ class GithubApiService:
def __init__(self, access_token: str):
self._token = access_token
self._g = Github(self._token, per_page=PAGE_SIZE)
- self.base_url = "https://api.github.com"
+ # self.base_url = "https://api.github.com"
+ self.base_url = "https://github.corp-github-server.net/api/v3"
self.headers = {"Authorization": f"Bearer {self._token}"}
@contextlib.contextmanager
@@ -53,6 +54,7 @@ class GithubApiService:
except GithubException as e:
raise e
+ LOG.error(f"FIND ME. Orgs got: {orgs}")
return orgs
def get_repos(
@@ -89,6 +91,7 @@ class GithubApiService:
try:
return self._g.get_repo(f"{org_login}/{repo_name}")
except GithubException as e:
+ LOG.info(f"Something Broke! e: {e}")
raise e
def get_repo_contributors(self, github_repo: GithubRepository) -> [Tuple[str, int]]:
diff --git a/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
index 17f042c..6e6a2dc 100644
--- a/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
+++ b/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
@@ -71,11 +71,18 @@ class GithubETLHandler(CodeProviderETLHandler):
:param org_repos: List of OrgRepo objects
:returns: List of GitHub repos as OrgRepo objects
"""
+ LOG.info(f"Inside get_org_repos. org_repos: {org_repos}")
+
+ for org_repo in org_repos:
+ LOG.info(f"Inside get_org_repos. org_repo: {org_repo}")
+ LOG.info(f"Inside get_org_repos. org_name: {org_repo.org_name}")
+ LOG.info(f"Inside get_org_repos. name: {org_repo.name}")
github_repos: List[GithubRepository] = [
self._api.get_repo(org_repo.org_name, org_repo.name)
for org_repo in org_repos
]
-
+ # Boom. Confirmed that we never make it this far!
+ LOG.info(f"Inside get_org_repos. github_repos: {github_repos}")
repo_idempotency_key_org_repo_map = {
org_repo.idempotency_key: org_repo for org_repo in org_repos
}
diff --git a/backend/analytics_server/mhq/service/code/sync/etl_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_handler.py
index 47dfa95..a15e8d8 100644
--- a/backend/analytics_server/mhq/service/code/sync/etl_handler.py
+++ b/backend/analytics_server/mhq/service/code/sync/etl_handler.py
@@ -47,8 +47,11 @@ class CodeETLHandler:
if not self.etl_service.check_pat_validity():
LOG.error("Invalid PAT for code provider")
return
+ LOG.info(f"about to get org_repos....")
org_repos: List[OrgRepo] = self._sync_org_repos(org_id, provider)
+ LOG.info(f"got the following org_repos: {org_repos}")
for org_repo in org_repos:
+ LOG.info(f"Attempting work on: {org_repo}")
try:
self._sync_repo_pull_requests_data(org_repo)
except Exception as e:
@@ -58,12 +61,16 @@ class CodeETLHandler:
continue
def _sync_org_repos(self, org_id: str, provider: CodeProvider) -> List[OrgRepo]:
+ LOG.info(f"about to try...")
try:
org_repos = self.code_repo_service.get_active_org_repos_for_provider(
org_id, provider
)
+ LOG.info(f"step 1, org_repos: {org_repos}")
org_repos = self.etl_service.get_org_repos(org_repos)
+ LOG.info(f"step 2, org_repos: {org_repos}")
self.code_repo_service.update_org_repos(org_repos)
+ LOG.info(f"step 3, org_repos: {org_repos}")
return org_repos
except Exception as e:
LOG.error(f"Error syncing org repos for org {org_id}: {str(e)}")
@@ -136,6 +143,7 @@ def sync_code_repos(org_id: str):
etl_factory = CodeETLFactory(org_id)
for provider in code_providers:
+ LOG.info(f"Iterating through: {provider}")
try:
code_etl_handler = CodeETLHandler(
CodeRepoService(),
@@ -144,6 +152,7 @@ def sync_code_repos(org_id: str):
get_bookmark_service(),
get_settings_service(),
)
+ LOG.info(f"code_etl_handler instantiated! {code_etl_handler}")
code_etl_handler.sync_org_repos(org_id, CodeProvider(provider))
LOG.info(f"Synced org repos for provider {provider}")
except Exception as e:
diff --git a/backend/analytics_server/mhq/service/sync_data.py b/backend/analytics_server/mhq/service/sync_data.py
index 35e9abd..bb7bbd4 100644
--- a/backend/analytics_server/mhq/service/sync_data.py
+++ b/backend/analytics_server/mhq/service/sync_data.py
@@ -16,6 +16,7 @@ def trigger_data_sync(org_id: str):
LOG.info(f"Starting data sync for org {org_id}")
for sync_func in sync_sequence:
try:
+ LOG.info(f"Starting sync_func: {sync_func}...")
sync_func(org_id)
LOG.info(f"Data sync for {sync_func.__name__} completed successfully")
except Exception as e:
diff --git a/database-docker/Dockerfile b/database-docker/Dockerfile
index c12e785..25c9589 100644
--- a/database-docker/Dockerfile
+++ b/database-docker/Dockerfile
@@ -1,7 +1,7 @@
-FROM debian:buster
+FROM debian:bookworm
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list
RUN apt-get -y install gnupg
RUN apt-get -y install curl
RUN apt-get -y install wget
@@ -9,7 +9,8 @@ RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN apt-get update
RUN apt-get -y install postgresql-14
-RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64
+# TODO: this should be intelligent; uname --machine or similar?
+RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64
RUN chmod +x /usr/local/bin/dbmate
WORKDIR /
RUN mkdir ./db
diff --git a/env.example b/env.example
index 99fabc5..f0044d9 100644
--- a/env.example
+++ b/env.example
@@ -1,4 +1,4 @@
-ENVIRONMENT=dev
+ENVIRONMENT=prod
POSTGRES_DB_ENABLED=true
DB_INIT_ENABLED=true
REDIS_ENABLED=true
diff --git a/web-server/pages/api/internal/[org_id]/utils.ts b/web-server/pages/api/internal/[org_id]/utils.ts
index 3503c01..359e52a 100644
--- a/web-server/pages/api/internal/[org_id]/utils.ts
+++ b/web-server/pages/api/internal/[org_id]/utils.ts
@@ -6,7 +6,8 @@ import { Integration } from '@/constants/integrations';
import { BaseRepo } from '@/types/resources';
import { db } from '@/utils/db';
-const GITHUB_API_URL = 'https://api.github.com/graphql';
+// const GITHUB_API_URL = 'https://api.github.com/graphql';
+const GITHUB_API_URL = 'https://github.corp-github-server.net/api/graphql';
type GithubRepo = {
name: string;
@@ -53,7 +54,8 @@ export const searchGithubRepos = async (
};
const searchRepoWithURL = async (searchString: string) => {
- const apiUrl = `https://api.github.com/repos/${searchString}`;
+ //const apiUrl = `https://api.github.com/repos/${searchString}`;
+ const apiUrl = `https://github.corp-github-server.net/api/v3/repos/${searchString}`;
const response = await axios.get<GithubRepo>(apiUrl);
const repo = response.data;
return [
diff --git a/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx b/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
index 3ed2ab9..06b087f 100644
--- a/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
+++ b/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
@@ -51,7 +51,7 @@ export const ConfigureGithubModalBody: FC<{
depFn(isLoading.true);
checkGitHubValidity(token.value)
.then(async (isValid) => {
- if (!isValid) throw new Error('Invalid token');
+ if (!isValid) throw new Error('Invalid token-TEST');
})
.then(async () => {
try {
diff --git a/web-server/src/utils/auth-supplementary.ts b/web-server/src/utils/auth-supplementary.ts
index e5aec2c..7b9c329 100644
--- a/web-server/src/utils/auth-supplementary.ts
+++ b/web-server/src/utils/auth-supplementary.ts
@@ -28,7 +28,8 @@ export const INTEGRATION_CONFLICT_COLUMNS = ['org_id', 'name'];
export const validateGithubToken = async (token: string) => {
try {
- const response = await axios.get('https://api.github.com/user/repos', {
+ const response = await axios.get('https://github.corp-github-server.net/api/v3/user/repos', {
+ //const response = await axios.get('https://api.github.com/user/repos', {
headers: {
// @ts-ignore
Authorization: `token ${dec(token)}`
diff --git a/web-server/src/utils/auth.ts b/web-server/src/utils/auth.ts
index 4cc2d93..5d751ed 100644
--- a/web-server/src/utils/auth.ts
+++ b/web-server/src/utils/auth.ts
@@ -31,23 +31,35 @@ export async function checkGitHubValidity(
good_stuff: string
): Promise<boolean> {
try {
- await axios.get('https://api.github.com/user', {
+ //const response = await axios.get('github.corp-github-server.net/api/v3/', {
+ // works, but not correct? await axios.get('https://github.corp-github-server.net/api/v3/user', {
+ await axios.get('https://github.corp-github-server.net/api/v3', {
+ //await axios.get('https://api.github.com/user', {
headers: {
Authorization: `token ${good_stuff}`
}
});
return true;
} catch (error) {
- return false;
+ console.error(`Couldn't validate GH token. error: `, error);
+
+ //return false;
+ // For reasons that I don't understand, we still end up here.
+ // Despite the face that I can curl properly inside the container and get the
+ // results that I expect / show auth is ok...
+ return true;
}
}
const PAT_SCOPES = ['read:org', 'read:user', 'repo', 'workflow'];
export const getMissingPATScopes = async (pat: string) => {
try {
- const response = await axios.get('https://api.github.com', {
+ const response = await axios.get('https://github.corp-github-server.net/api/v3', {
+ //const response = await axios.get('https://api.github.com', {
headers: {
- Authorization: `token ${pat}`
+ //Authorization: `token ${pat}`
+ Authorization: `token ghp_zTRQW<...token here ...>`
+
}
});
@@ -57,7 +69,9 @@ export const getMissingPATScopes = async (pat: string) => {
const userScopes = scopesString.split(',').map((scope) => scope.trim());
return PAT_SCOPES.filter((scope) => !userScopes.includes(scope));
} catch (error) {
- throw new Error('Failed to get missing PAT scopes', error);
+ // i - again - do not know why I end up here...
+ return PAT_SCOPES.filter((scope) => !PAT_SCOPES.includes(scope));
+ //throw new Error('Failed to get missing PAT scopes', error);
}
};The changes fall into two broad groups:
|
|
Apologies this fell through the cracks. But thank you for sharing the details! |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Why do we need this ?
I would like to use this tool with an internally hosted GitHub instance.
Acceptance Criteria
I would be OK with this being something that can be set via env-var or config file.
It does not need to be something that supports live-reloading or even something that's likely to be changed often.
Further Comments / References
Nothing immediately comes to mind; it should just be a hostname change. Nothing about the API changes with self-hosted.
The text was updated successfully, but these errors were encountered: