Skip to content

Commit 1c7f65b

Browse files
msyycmsyyc
authored
[Issue helper] pipeline to collect open issues for all languages (Azure#22804)
* initial * rename yml * git push function * only collect issues with label 'Mgmt' * handle exception * date format * update log print * update bot policy * skip self-create issue
1 parent a5ea4bf commit 1c7f65b

File tree

8 files changed

+247
-0
lines changed

8 files changed

+247
-0
lines changed

‎scripts/issue_helper/common.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
from datetime import date
2+
from typing import Set, List, Dict, Any
3+
import os
4+
import logging
5+
from github import Github
6+
from github.Issue import Issue
7+
from subprocess import check_call
8+
9+
_LOG = logging.getLogger(__name__)
10+
11+
_LANGUAGE_OWNER = {'msyyc'}
12+
_LANGUAGE_REPO = 'Azure/azure-sdk-for-python'
13+
_FILE_OUT_NAME = 'common.md'
14+
15+
16+
class Common:
17+
""" The class defines some function for all languages to reference
18+
language_owner = {} # language owner who may handle issue
19+
repo_name = '' # base repo. For example: Azure/azure-sdk-for-python
20+
file_out_name = '' # report name. For example: sdk_issue_status_python.md
21+
issue_html = '' # issue url. For example: https://github.com/Azure/azure-sdk-for-python/issues
22+
23+
"""
24+
25+
def __init__(self, language_owner: Set[str], repo_name: str, file_out_name: str):
26+
self.language_owner = language_owner
27+
self.repo_name = repo_name
28+
self.file_out_name = file_out_name
29+
self.issue_html = f'https://github.com/{repo_name}/issues'
30+
self.bot_assignees = {'msftbot[bot]'}
31+
32+
@staticmethod
33+
def push_md_to_storage():
34+
cmd_list = ['git add .', 'git commit -m \"update excel\"', 'git push -f origin HEAD']
35+
[check_call(cmd, shell=True) for cmd in cmd_list]
36+
37+
def collect_open_issues(self) -> List[Issue]:
38+
hub = Github(os.getenv('TOKEN'))
39+
repo = hub.get_repo(self.repo_name)
40+
mgmt_label = repo.get_label('Mgmt')
41+
open_issues = repo.get_issues(state='open', labels=[mgmt_label])
42+
return [issue for issue in open_issues if not issue.pull_request]
43+
44+
def judge_status(self, issue: Issue) -> str:
45+
if issue.user.login in self.language_owner:
46+
return ''
47+
latest_comments = ''
48+
comments = [(comment.updated_at.timestamp(), comment.user.login) for comment in issue.get_comments()
49+
if comment.user.login not in self.bot_assignees]
50+
comments.sort()
51+
if comments:
52+
latest_comments = comments[-1][1]
53+
if issue.comments == 0:
54+
return 'new issue'
55+
elif latest_comments not in self.language_owner:
56+
return 'new comment'
57+
else:
58+
return ''
59+
60+
def extract_info(self, open_issues: List[Issue]) -> List[Dict[str, Any]]:
61+
issues_info = []
62+
_LOG.info(f'collect {len(open_issues)} open issues in {self.issue_html}')
63+
idx = 1
64+
for issue in open_issues:
65+
assignees = {assignee.login for assignee in issue.assignees}
66+
if assignees.intersection(self.language_owner):
67+
issue_info = {
68+
'idx': idx,
69+
'number': issue.number,
70+
'html': f'{self.issue_html}/{issue.number}',
71+
'title': issue.title,
72+
'labels': [label.name for label in issue.labels],
73+
'assignees': assignees,
74+
'created_date': str(date.fromtimestamp(issue.created_at.timestamp()).strftime('%Y-%m-%d')),
75+
'status': self.judge_status(issue)
76+
}
77+
issues_info.append(issue_info)
78+
idx = idx + 1
79+
_LOG.info(f'collect {len(issues_info)} open issues assigned to {str(self.language_owner)}')
80+
return issues_info
81+
82+
@staticmethod
83+
def output_line(issue_info: Dict[str, Any]) -> str:
84+
return '|{No}|[#{number}]({issue_html})|{title}|{labels}|{assignees}|{bot_advice}|{created_date}|\n'.format(
85+
No=issue_info['idx'],
86+
number=issue_info['number'],
87+
issue_html=issue_info['html'],
88+
title=issue_info['title'],
89+
labels=', '.join(issue_info['labels']),
90+
assignees=', '.join(issue_info['assignees']),
91+
bot_advice=issue_info['status'],
92+
created_date=issue_info['created_date'],
93+
)
94+
95+
def report(self, issues_info: List[Dict[str, Any]]) -> None:
96+
with open(self.file_out_name, 'w') as file_out:
97+
file_out.write(
98+
'| No. | issue | title | labels | assignees | bot advice | created date |\n')
99+
file_out.write('| ------ | ------ | ------ | ------ | ------ | ------ | :-----: |\n')
100+
file_out.writelines([Common.output_line(issue_info) for issue_info in issues_info])
101+
102+
def run(self):
103+
open_issues = self.collect_open_issues()
104+
issues_info = self.extract_info(open_issues)
105+
self.report(issues_info)
106+
107+
108+
def common_process():
109+
instance = Common(_LANGUAGE_OWNER, _LANGUAGE_REPO, _FILE_OUT_NAME)
110+
instance.run()

‎scripts/issue_helper/go.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from common import Common
2+
3+
_GO_OWNER = {'ArcturusZhang', 'lirenhe'}
4+
_GO_REPO = 'Azure/azure-sdk-for-go'
5+
_FILE_OUT_NAME_GO = 'sdk_issue_go.md'
6+
7+
8+
def go_process() -> None:
9+
instance = Common(_GO_OWNER, _GO_REPO, _FILE_OUT_NAME_GO)
10+
instance.run()

‎scripts/issue_helper/issue_helper.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Release status statistics
2+
3+
name: IssueHelper
4+
5+
trigger:
6+
branches:
7+
exclude:
8+
- '*'
9+
10+
schedules:
11+
- cron: "0 16 * * *"
12+
displayName: SDK Issue Statistics
13+
branches:
14+
include:
15+
- main
16+
always: true
17+
18+
variables:
19+
- group: Azure SDK Auto Release Pipeline Secrets
20+
- group: SDK Release Helper
21+
22+
jobs:
23+
- job: IssueHelper
24+
displayName: IssueHelper Python 3.8
25+
timeoutInMinutes: 30
26+
strategy:
27+
maxParallel: 3
28+
pool:
29+
vmImage: 'ubuntu-20.04'
30+
steps:
31+
- task: UsePythonVersion@0
32+
inputs:
33+
versionSpec: '3.8'
34+
addToPath: true
35+
architecture: 'x64'
36+
- bash: |
37+
script_path=$(pwd)/scripts/issue_helper
38+
cd ..
39+
git config --global user.email "IssueHelper"
40+
git config --global user.name "IssueHelper"
41+
42+
# clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx)
43+
mkdir file-storage
44+
git clone ${FILE_REPO:0:8}$(USR_NAME):$(Yuchao-GitToken)@${FILE_REPO:8} $(pwd)/file-storage
45+
46+
# import env variable
47+
export TOKEN=$(Yuchao-GitToken)
48+
export LANGUAGE=$(RUN_LANGUAGE)
49+
50+
# create virtual env
51+
python -m venv venv-sdk
52+
source venv-sdk/bin/activate
53+
pip install -r $script_path/requirement.txt
54+
55+
# checkout the target branch
56+
cd file-storage
57+
git checkout issue-helper
58+
59+
# run
60+
python $script_path/main.py

‎scripts/issue_helper/java.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from common import Common
2+
3+
_JAVA_OWNER = {'weidongxu-microsoft', 'haolingdong-msft', 'XiaofeiCao'}
4+
_JAVA_REPO = 'Azure/azure-sdk-for-java'
5+
_FILE_OUT_NAME_JAVA = 'sdk_issue_java.md'
6+
7+
8+
def java_process() -> None:
9+
instance = Common(_JAVA_OWNER, _JAVA_REPO, _FILE_OUT_NAME_JAVA)
10+
instance.run()

‎scripts/issue_helper/js.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from common import Common
2+
3+
_JS_OWNER = {'colawwj', 'qiaozha', 'lirenhe', 'dw511214992'}
4+
_JS_REPO = 'Azure/azure-sdk-for-js'
5+
_FILE_OUT_NAME_JS = 'sdk_issue_js.md'
6+
7+
8+
def js_process() -> None:
9+
instance = Common(_JS_OWNER, _JS_REPO, _FILE_OUT_NAME_JS)
10+
instance.run()

‎scripts/issue_helper/main.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
from python import python_process
2+
from go import go_process
3+
from java import java_process
4+
from js import js_process
5+
from common import Common
6+
7+
import os
8+
import logging
9+
10+
logging.basicConfig(level=logging.INFO,
11+
format='%(funcName)s[line:%(lineno)d] - %(levelname)s: %(message)s')
12+
_LOG = logging.getLogger(__name__)
13+
14+
_LANGUAGES = {
15+
'python': python_process,
16+
'java': java_process,
17+
'go': go_process,
18+
'js': js_process
19+
}
20+
21+
22+
def main():
23+
language = os.getenv('LANGUAGE')
24+
languages = {language: _LANGUAGES[language]} if language in _LANGUAGES.keys() else _LANGUAGES
25+
for language in languages:
26+
try:
27+
languages[language]()
28+
except Exception as e:
29+
_LOG.info(f'Failed to collect issues status for {language}: {e}')
30+
Common.push_md_to_storage()
31+
32+
33+
if __name__ == '__main__':
34+
main()

‎scripts/issue_helper/python.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from common import Common
2+
3+
_PYTHON_OWNER = {'msyyc', 'BigCat20196'}
4+
_PYTHON_REPO = 'Azure/azure-sdk-for-python'
5+
_FILE_OUT_NAME_PYTHON = 'sdk_issue_python.md'
6+
7+
8+
def python_process() -> None:
9+
instance = Common(_PYTHON_OWNER, _PYTHON_REPO, _FILE_OUT_NAME_PYTHON)
10+
instance.run()

‎scripts/issue_helper/requirement.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
PyGithub==1.55
2+
datetime
3+
requests

0 commit comments

Comments
 (0)