Skip to content

Commit

Permalink
feat: add remove all inexist repo function
Browse files Browse the repository at this point in the history 
* Now users can choose to remove all repositories in Gitea owned by the user (including those in organizations) that do not exist in GitHub.
  • Loading branch information
@katorly
katorly committed Sep 11, 2024
1 parent fb63d58 commit 958ed9e
Show file tree
Hide file tree
Showing 8 changed files with 95 additions and 29 deletions.
35 changes: 22 additions & 13 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,19 +69,20 @@ All credentials are needed to be configured properly:

You can customize these options for mirroring:

| Key | Description |
|:---------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------|
| **CONFIG** | |
| CREATE_ORG | Create a new organization in Gitea when the repository username is different from your GitHub username. |
| REMOVE_EXISTING_REPO | Remove existing repositories in Gitea. You may not want to enable this option, since Gitea will automatically fetch the mirror repositories every 8 hours. |
| MIRROR_OWNED | Mirror the repositories you own. |
| MIRROR_FORKED | Mirror the repositories you forked. |
| MIRROR_STARRED | Mirror the repositories you starred. |
| MIRROR_COLLABORATOR | Mirror the repositories that you have collaborator access. See: https://docs.github.com/zh/rest/repos/repos#list-repositories-for-the-authenticated-user |
| MIRROR_ORGANIZATION | Mirror the repositories in organizations that you are a member. |
| **RULE** | |
| MODE | `whitelist` or `blacklist` to only mirror or skip repositories that match the regex. |
| REGEX | Regex list. |
| Key | Description |
|:---------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **CONFIG** | |
| CREATE_ORG | Create a new organization in Gitea when the repository username is different from your GitHub username. |
| REMOVE_INEXIST_REPO | Remove all repositories in Gitea owned by the user (including those in organizations) that do not exist in GitHub. |
| REMOVE_EXISTING_REPO | Remove existing repositories in Gitea. This will only remove the repositories that have the same name as the repositories in GitHub. You may not want to enable this option, since Gitea will automatically fetch the mirror repositories every 8 hours. |
| MIRROR_OWNED | Mirror the repositories you own. |
| MIRROR_FORKED | Mirror the repositories you forked. |
| MIRROR_STARRED | Mirror the repositories you starred. |
| MIRROR_COLLABORATOR | Mirror the repositories that you have collaborator access. See: https://docs.github.com/zh/rest/repos/repos#list-repositories-for-the-authenticated-user |
| MIRROR_ORGANIZATION | Mirror the repositories in organizations that you are a member. |
| **RULE** | |
| MODE | `whitelist` or `blacklist` to only mirror or skip repositories that match the regex. |
| REGEX | Regex list. |


## Usage
Expand Down Expand Up @@ -158,6 +159,14 @@ jobs:
```
## Frequently Asked
| Error message | Solution |
|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------|
| `403 b'{"message":"token does not have at least one of required scope(s):` | Follow the message's instruction to grant the required permissions to your PAT. |
| My repository doesn't exist on Gitea, but the script returns `Skip (exists)` | Please don't transfer the mirrored repository to another user or organization on Gitea. |
| `b'{"message":"Migration failed: clone error: exit status 128 - fatal: unable to access \'https://github.com/username/repo.git/\': Could not resolve host: github.com\\n."` | Make sure your Gitea host can access GitHub. |


<!-- /Main Body -->


Expand Down
5 changes: 5 additions & 0 deletions config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
CREATE_ORG = (
os.getenv("CREATE_ORG", str(options["CONFIG"]["CREATE_ORG"])).lower() == "true"
)
REMOVE_INEXIST_REPO = (
os.getenv("REMOVE_INEXIST_REPO", str(options["CONFIG"]["REMOVE_INEXIST_REPO"]))
.lower()
== "true"
)
REMOVE_EXISTING_REPO = (
os.getenv(
"REMOVE_EXISTING_REPO", str(options["CONFIG"]["REMOVE_EXISTING_REPO"])
Expand Down
7 changes: 6 additions & 1 deletion config/options.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@
# Create a new organization in Gitea when the repository username
# is different from your GitHub username.
CREATE_ORG = true
# Remove all repositories in Gitea owned by the user (including
# those in organzations) that do not exist in GitHub.
REMOVE_INEXIST_REPO = false
# Remove existing repositories in Gitea.
# This will only remove the repositories that have the same name
# as the repositories in GitHub.
# You may not want to enable this option, since Gitea will
# automatically fetch the mirror repositories every 8 hours.
REMOVE_EXISTING_REPO = false
Expand All @@ -24,5 +29,5 @@ MIRROR_ORGANIZATION = false
MODE = "blacklist"
REGEX = [
"EpicGames/.*",
"NVIDIAGameWorks/.*"
"NVIDIAGameWorks/.*",
]
11 changes: 9 additions & 2 deletions main.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
import logging

import config
from utils.giteaRepo import fetch_gitea_repos, remove_inexist_repo
from utils.githubRepo import fetch_github_repos
from utils.mirror import mirror_to_gitea


def main():
logging.basicConfig(level=logging.INFO)
repos = fetch_github_repos()
for repo in repos:
github_repos = fetch_github_repos()

if config.REMOVE_INEXIST_REPO:
gitea_repos = fetch_gitea_repos()
remove_inexist_repo(github_repos, gitea_repos)

for repo in github_repos:
mirror_to_gitea(repo)


Expand Down
2 changes: 1 addition & 1 deletion utils/giteaOrg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@ def create_gitea_org(org_name):
if response.status_code == 201:
logging.info(f"Organization created: {org_name}")
else:
logging.warning(f"Organization creation failed: {org_name}: {response.content}")
logging.warning(f"Organization creation failed: {org_name}: {response.status_code} {response.content}")
38 changes: 37 additions & 1 deletion utils/giteaRepo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,42 @@
import config


def fetch_gitea_repos():
repos = []
page = 1
per_page = 50
headers = {'Authorization': f'token {config.GITEA_PAT}'}

logging.info("\nFetching Gitea repositories...")
while True:
url = f"{config.GITEA_HOST}/api/v1/user/repos?page={page}&limit={per_page}"
response = requests.get(url, headers=headers)
if response.status_code != 200:
logging.warning(f"Failed to fetch Gitea repositories: {response.status_code} {response.content}")
break
page_repos = response.json()
if not page_repos:
break
repos.extend(page_repos)
page += 1

logging.info(f"Total Gitea repositories fetched: {len(repos)}")
return repos


def remove_inexist_repo(github_repos, gitea_repos):
github_repo_names = {repo["name"] for repo in github_repos}
for repo in gitea_repos:
if repo["name"] not in github_repo_names:
url = f"{config.GITEA_HOST}/api/v1/repos/{repo['owner']['username']}/{repo['name']}"
headers = {"Authorization": f"token {config.GITEA_PAT}"}
response = requests.delete(url, headers=headers)
if response.status_code == 204:
logging.info(f"Removed removed: {repo['full_name']}")
else:
logging.warning(f"Repository removal failed: {repo['full_name']}: {response.status_code} {response.content}")


def check_gitea_repo_exists(repo_name):
url = f"{config.GITEA_HOST}/api/v1/repos/{repo_name}"
response = requests.get(url, headers=cache.headers())
Expand All @@ -18,4 +54,4 @@ def remove_gitea_repo(repo_name):
if response.status_code == 204:
logging.info(f"Repository removed: {repo_name}")
else:
logging.warning(f"Repository removal failed: {repo_name}: {response.content}")
logging.warning(f"Repository removal failed: {repo_name}: {response.status_code} {response.content}")
10 changes: 6 additions & 4 deletions utils/githubRepo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@
import config


def fetch_request(raw_url, headers):
def fetch_request(raw_url, headers, name):
repos = []
per_page = 50

response = requests.get(raw_url.format(per_page=per_page), headers=headers)
if response.status_code != 200:
logging.warning(
f"Failed to fetch GitHub repositories: {response.status_code} {response.content}"
f"Failed to fetch {name}: {response.status_code} {response.content}"
)
return repos

Expand All @@ -24,10 +24,12 @@ def fetch_github_repos():
headers = {"Authorization": f"token {config.GITHUB_PAT}"}
repos = []

logging.info("\nFetching GitHub repositories...")
if config.MIRROR_STARED:
repos.extend(
fetch_request(
"https://api.github.com/user/starred?per_page={per_page}", headers
"https://api.github.com/user/starred?per_page={per_page}",
headers, "GitHub repositories"
)
)

Expand All @@ -44,7 +46,7 @@ def fetch_github_repos():
repos.extend(
fetch_request(
f"https://api.github.com/user/repos?per_page={{per_page}}&affiliation={affiliation_param}",
headers,
headers, "GitHub repositories"
)
)

Expand Down
16 changes: 9 additions & 7 deletions utils/mirror.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,19 @@ def mirror_to_gitea(repo):
return

repo_owner = config.GITEA_USERNAME
if config.CREATE_ORG and repo["owner"]["login"] != config.GITHUB_USERNAME:
target_repo_name = repo_name
if config.CREATE_ORG and repo["owner"]["login"] != config.GITEA_USERNAME:
repo_owner = repo["owner"]["login"]
target_repo_name = f"{repo_owner}/{repo['name']}"
if not check_gitea_org_exists(repo_owner):
create_gitea_org(repo_owner)

if check_gitea_repo_exists(repo_name):
if check_gitea_repo_exists(target_repo_name):
if config.REMOVE_EXISTING_REPO:
logging.info(f"\tRemove (exists): {repo_name}")
logging.info(f"\tRemove (exists): {target_repo_name}")
remove_gitea_repo(repo_name)
else:
logging.info(f"\tSkip (exists): {repo_name}")
logging.info(f"\tSkip (exists): {target_repo_name}")
return

url = f"{config.GITEA_HOST}/api/v1/repos/migrate"
Expand All @@ -53,10 +55,10 @@ def mirror_to_gitea(repo):
for attempt in range(3): # Retry if failed
response = requests.post(url, headers=cache.headers(), data=json.dumps(data))
if response.status_code == 201:
logging.info(f"\tSuccess: {repo_name}")
logging.info(f"\tSuccess: {target_repo_name}")
break
else:
logging.warning(f"\tFailed: {repo_name}: {response.content}")
logging.warning(f"\tFailed: {target_repo_name}: {response.status_code} {response.content}")
if attempt < 2:
logging.info(f"\tRetrying: {repo_name}")
logging.info(f"\tRetrying: {target_repo_name}")
time.sleep(5)

0 comments on commit 958ed9e

Please sign in to comment.