-
Notifications
You must be signed in to change notification settings - Fork 117
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Allow changing GitHub server hostname (for self-hosted/enterprise instances) #619
Comments
Hey @kquinsland, we've not gotten around to this yet, but perhaps you could create a fork and replace references to If it does work for you, let me know and we'll add it to the README or Wiki. :) |
I tried this and found I had to make quite a few small patches here and there before I ran out of time. I don't recall precisely where I got stuck (it's been 6 weeks...) but there was some issue(s) I had w/r/t auth that I wasn't able to fully grock. |
Had a spare sec to go digging. # So you can see the commit to compare my changes against
❯ git rev-parse HEAD
8605f85a1053d94a49aea9e63d34de153cca35d8
# Generate a diff of all the things I had to change
❯ git diff --name-only | while read -r file; do git diff "$file"; echo ""; done > patches-i-made And here's ❯ cat patches-i-made
diff --git a/Dockerfile.dev b/Dockerfile.dev
index 1c747f8..e5c4922 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -37,6 +37,9 @@ RUN /opt/venv/bin/pip install --upgrade pip
RUN --mount=type=cache,target=/root/.cache/pip /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt
WORKDIR /app
+# TODO: this should be intelligent; uname --machine or similar?
+RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64
+RUN chmod +x /usr/local/bin/dbmate
RUN mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob
RUN chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh
RUN mv /app/setup_utils/supervisord.conf /etc/supervisord.conf
diff --git a/Dockerfile.prebuilt b/Dockerfile.prebuilt
index e942b33..c22a962 100644
--- a/Dockerfile.prebuilt
+++ b/Dockerfile.prebuilt
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
&& curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
&& apt-get install -y nodejs \
- && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \
+ && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64 \
&& chmod +x /usr/local/bin/dbmate \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
diff --git a/backend/analytics_server/mhq/exapi/github.py b/backend/analytics_server/mhq/exapi/github.py
index 50cd192..194df9c 100644
--- a/backend/analytics_server/mhq/exapi/github.py
+++ b/backend/analytics_server/mhq/exapi/github.py
@@ -25,7 +25,8 @@ class GithubApiService:
def __init__(self, access_token: str):
self._token = access_token
self._g = Github(self._token, per_page=PAGE_SIZE)
- self.base_url = "https://api.github.com"
+ # self.base_url = "https://api.github.com"
+ self.base_url = "https://github.corp-github-server.net/api/v3"
self.headers = {"Authorization": f"Bearer {self._token}"}
@contextlib.contextmanager
@@ -53,6 +54,7 @@ class GithubApiService:
except GithubException as e:
raise e
+ LOG.error(f"FIND ME. Orgs got: {orgs}")
return orgs
def get_repos(
@@ -89,6 +91,7 @@ class GithubApiService:
try:
return self._g.get_repo(f"{org_login}/{repo_name}")
except GithubException as e:
+ LOG.info(f"Something Broke! e: {e}")
raise e
def get_repo_contributors(self, github_repo: GithubRepository) -> [Tuple[str, int]]:
diff --git a/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
index 17f042c..6e6a2dc 100644
--- a/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
+++ b/backend/analytics_server/mhq/service/code/sync/etl_github_handler.py
@@ -71,11 +71,18 @@ class GithubETLHandler(CodeProviderETLHandler):
:param org_repos: List of OrgRepo objects
:returns: List of GitHub repos as OrgRepo objects
"""
+ LOG.info(f"Inside get_org_repos. org_repos: {org_repos}")
+
+ for org_repo in org_repos:
+ LOG.info(f"Inside get_org_repos. org_repo: {org_repo}")
+ LOG.info(f"Inside get_org_repos. org_name: {org_repo.org_name}")
+ LOG.info(f"Inside get_org_repos. name: {org_repo.name}")
github_repos: List[GithubRepository] = [
self._api.get_repo(org_repo.org_name, org_repo.name)
for org_repo in org_repos
]
-
+ # Boom. Confirmed that we never make it this far!
+ LOG.info(f"Inside get_org_repos. github_repos: {github_repos}")
repo_idempotency_key_org_repo_map = {
org_repo.idempotency_key: org_repo for org_repo in org_repos
}
diff --git a/backend/analytics_server/mhq/service/code/sync/etl_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_handler.py
index 47dfa95..a15e8d8 100644
--- a/backend/analytics_server/mhq/service/code/sync/etl_handler.py
+++ b/backend/analytics_server/mhq/service/code/sync/etl_handler.py
@@ -47,8 +47,11 @@ class CodeETLHandler:
if not self.etl_service.check_pat_validity():
LOG.error("Invalid PAT for code provider")
return
+ LOG.info(f"about to get org_repos....")
org_repos: List[OrgRepo] = self._sync_org_repos(org_id, provider)
+ LOG.info(f"got the following org_repos: {org_repos}")
for org_repo in org_repos:
+ LOG.info(f"Attempting work on: {org_repo}")
try:
self._sync_repo_pull_requests_data(org_repo)
except Exception as e:
@@ -58,12 +61,16 @@ class CodeETLHandler:
continue
def _sync_org_repos(self, org_id: str, provider: CodeProvider) -> List[OrgRepo]:
+ LOG.info(f"about to try...")
try:
org_repos = self.code_repo_service.get_active_org_repos_for_provider(
org_id, provider
)
+ LOG.info(f"step 1, org_repos: {org_repos}")
org_repos = self.etl_service.get_org_repos(org_repos)
+ LOG.info(f"step 2, org_repos: {org_repos}")
self.code_repo_service.update_org_repos(org_repos)
+ LOG.info(f"step 3, org_repos: {org_repos}")
return org_repos
except Exception as e:
LOG.error(f"Error syncing org repos for org {org_id}: {str(e)}")
@@ -136,6 +143,7 @@ def sync_code_repos(org_id: str):
etl_factory = CodeETLFactory(org_id)
for provider in code_providers:
+ LOG.info(f"Iterating through: {provider}")
try:
code_etl_handler = CodeETLHandler(
CodeRepoService(),
@@ -144,6 +152,7 @@ def sync_code_repos(org_id: str):
get_bookmark_service(),
get_settings_service(),
)
+ LOG.info(f"code_etl_handler instantiated! {code_etl_handler}")
code_etl_handler.sync_org_repos(org_id, CodeProvider(provider))
LOG.info(f"Synced org repos for provider {provider}")
except Exception as e:
diff --git a/backend/analytics_server/mhq/service/sync_data.py b/backend/analytics_server/mhq/service/sync_data.py
index 35e9abd..bb7bbd4 100644
--- a/backend/analytics_server/mhq/service/sync_data.py
+++ b/backend/analytics_server/mhq/service/sync_data.py
@@ -16,6 +16,7 @@ def trigger_data_sync(org_id: str):
LOG.info(f"Starting data sync for org {org_id}")
for sync_func in sync_sequence:
try:
+ LOG.info(f"Starting sync_func: {sync_func}...")
sync_func(org_id)
LOG.info(f"Data sync for {sync_func.__name__} completed successfully")
except Exception as e:
diff --git a/database-docker/Dockerfile b/database-docker/Dockerfile
index c12e785..25c9589 100644
--- a/database-docker/Dockerfile
+++ b/database-docker/Dockerfile
@@ -1,7 +1,7 @@
-FROM debian:buster
+FROM debian:bookworm
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list
RUN apt-get -y install gnupg
RUN apt-get -y install curl
RUN apt-get -y install wget
@@ -9,7 +9,8 @@ RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN apt-get update
RUN apt-get -y install postgresql-14
-RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64
+# TODO: this should be intelligent; uname --machine or similar?
+RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-arm64
RUN chmod +x /usr/local/bin/dbmate
WORKDIR /
RUN mkdir ./db
diff --git a/env.example b/env.example
index 99fabc5..f0044d9 100644
--- a/env.example
+++ b/env.example
@@ -1,4 +1,4 @@
-ENVIRONMENT=dev
+ENVIRONMENT=prod
POSTGRES_DB_ENABLED=true
DB_INIT_ENABLED=true
REDIS_ENABLED=true
diff --git a/web-server/pages/api/internal/[org_id]/utils.ts b/web-server/pages/api/internal/[org_id]/utils.ts
index 3503c01..359e52a 100644
--- a/web-server/pages/api/internal/[org_id]/utils.ts
+++ b/web-server/pages/api/internal/[org_id]/utils.ts
@@ -6,7 +6,8 @@ import { Integration } from '@/constants/integrations';
import { BaseRepo } from '@/types/resources';
import { db } from '@/utils/db';
-const GITHUB_API_URL = 'https://api.github.com/graphql';
+// const GITHUB_API_URL = 'https://api.github.com/graphql';
+const GITHUB_API_URL = 'https://github.corp-github-server.net/api/graphql';
type GithubRepo = {
name: string;
@@ -53,7 +54,8 @@ export const searchGithubRepos = async (
};
const searchRepoWithURL = async (searchString: string) => {
- const apiUrl = `https://api.github.com/repos/${searchString}`;
+ //const apiUrl = `https://api.github.com/repos/${searchString}`;
+ const apiUrl = `https://github.corp-github-server.net/api/v3/repos/${searchString}`;
const response = await axios.get<GithubRepo>(apiUrl);
const repo = response.data;
return [
diff --git a/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx b/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
index 3ed2ab9..06b087f 100644
--- a/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
+++ b/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx
@@ -51,7 +51,7 @@ export const ConfigureGithubModalBody: FC<{
depFn(isLoading.true);
checkGitHubValidity(token.value)
.then(async (isValid) => {
- if (!isValid) throw new Error('Invalid token');
+ if (!isValid) throw new Error('Invalid token-TEST');
})
.then(async () => {
try {
diff --git a/web-server/src/utils/auth-supplementary.ts b/web-server/src/utils/auth-supplementary.ts
index e5aec2c..7b9c329 100644
--- a/web-server/src/utils/auth-supplementary.ts
+++ b/web-server/src/utils/auth-supplementary.ts
@@ -28,7 +28,8 @@ export const INTEGRATION_CONFLICT_COLUMNS = ['org_id', 'name'];
export const validateGithubToken = async (token: string) => {
try {
- const response = await axios.get('https://api.github.com/user/repos', {
+ const response = await axios.get('https://github.corp-github-server.net/api/v3/user/repos', {
+ //const response = await axios.get('https://api.github.com/user/repos', {
headers: {
// @ts-ignore
Authorization: `token ${dec(token)}`
diff --git a/web-server/src/utils/auth.ts b/web-server/src/utils/auth.ts
index 4cc2d93..5d751ed 100644
--- a/web-server/src/utils/auth.ts
+++ b/web-server/src/utils/auth.ts
@@ -31,23 +31,35 @@ export async function checkGitHubValidity(
good_stuff: string
): Promise<boolean> {
try {
- await axios.get('https://api.github.com/user', {
+ //const response = await axios.get('github.corp-github-server.net/api/v3/', {
+ // works, but not correct? await axios.get('https://github.corp-github-server.net/api/v3/user', {
+ await axios.get('https://github.corp-github-server.net/api/v3', {
+ //await axios.get('https://api.github.com/user', {
headers: {
Authorization: `token ${good_stuff}`
}
});
return true;
} catch (error) {
- return false;
+ console.error(`Couldn't validate GH token. error: `, error);
+
+ //return false;
+ // For reasons that I don't understand, we still end up here.
+ // Despite the face that I can curl properly inside the container and get the
+ // results that I expect / show auth is ok...
+ return true;
}
}
const PAT_SCOPES = ['read:org', 'read:user', 'repo', 'workflow'];
export const getMissingPATScopes = async (pat: string) => {
try {
- const response = await axios.get('https://api.github.com', {
+ const response = await axios.get('https://github.corp-github-server.net/api/v3', {
+ //const response = await axios.get('https://api.github.com', {
headers: {
- Authorization: `token ${pat}`
+ //Authorization: `token ${pat}`
+ Authorization: `token ghp_zTRQW<...token here ...>`
+
}
});
@@ -57,7 +69,9 @@ export const getMissingPATScopes = async (pat: string) => {
const userScopes = scopesString.split(',').map((scope) => scope.trim());
return PAT_SCOPES.filter((scope) => !userScopes.includes(scope));
} catch (error) {
- throw new Error('Failed to get missing PAT scopes', error);
+ // i - again - do not know why I end up here...
+ return PAT_SCOPES.filter((scope) => !PAT_SCOPES.includes(scope));
+ //throw new Error('Failed to get missing PAT scopes', error);
}
}; The changes fall into two broad groups:
|
Apologies this fell through the cracks. But thank you for sharing the details! |
Why do we need this ?
I would like to use this tool with an internally hosted GitHub instance.
Acceptance Criteria
I would be OK with this being something that can be set via env-var or config file.
It does not need to be something that supports live-reloading or even something that's likely to be changed often.
Further Comments / References
Nothing immediately comes to mind; it should just be a hostname change. Nothing about the API changes with self-hosted.
The text was updated successfully, but these errors were encountered: