Skip to content

Commit 46dad48

Browse files
kb22kb22
committed
Retrieve more than 30 commits if available
1 parent 69e25f3 commit 46dad48

File tree

3 files changed

+80
-13
lines changed

3 files changed

+80
-13
lines changed

‎Data collection.ipynb

+67-12
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@
7373
" u'collaborators': 0,\n",
7474
" u'company': u'Cvent',\n",
7575
" u'created_at': u'2015-09-16T16:14:22Z',\n",
76-
" u'disk_usage': 155405,\n",
76+
" u'disk_usage': 155435,\n",
7777
" u'email': u'[email protected]',\n",
7878
" u'events_url': u'https://api.github.com/users/kb22/events{/privacy}',\n",
7979
" u'followers': 35,\n",
@@ -735,7 +735,7 @@
735735
"(9, {u'Jupyter Notebook': 295254})\n",
736736
"(10, {u'Jupyter Notebook': 85404})\n",
737737
"(11, {u'Python': 3034})\n",
738-
"(12, {u'Jupyter Notebook': 71643})\n",
738+
"(12, {u'Jupyter Notebook': 73804})\n",
739739
"(13, {u'Jupyter Notebook': 261173})\n",
740740
"(14, {u'Jupyter Notebook': 39932})\n",
741741
"(15, {u'PHP': 6386, u'Java': 67597})\n",
@@ -1244,19 +1244,74 @@
12441244
"cell_type": "code",
12451245
"execution_count": 12,
12461246
"metadata": {},
1247-
"outputs": [],
1247+
"outputs": [
1248+
{
1249+
"name": "stdout",
1250+
"output_type": "stream",
1251+
"text": [
1252+
"URL: https://api.github.com/repos/kb22/Activity-Recognition-using-Machine-Learning/commits, commits: 7\n",
1253+
"URL: https://api.github.com/repos/kb22/Animation-in-C/commits, commits: 2\n",
1254+
"URL: https://api.github.com/repos/kb22/Article-Recommender/commits, commits: 18\n",
1255+
"URL: https://api.github.com/repos/kb22/Attendance-Marker/commits, commits: 4\n",
1256+
"URL: https://api.github.com/repos/kb22/Color-Identification-using-Machine-Learning/commits, commits: 5\n",
1257+
"URL: https://api.github.com/repos/kb22/Computer-Simulation-using-Proteus/commits, commits: 2\n",
1258+
"URL: https://api.github.com/repos/kb22/Coursera_Capstone/commits, commits: 30\n",
1259+
"URL: https://api.github.com/repos/kb22/Coursera_Capstone/commits?page=2, commits: 4\n",
1260+
"URL: https://api.github.com/repos/kb22/Create-dataset-using-API/commits, commits: 8\n",
1261+
"URL: https://api.github.com/repos/kb22/Create-Face-Data-from-Images/commits, commits: 6\n",
1262+
"URL: https://api.github.com/repos/kb22/Deep-Learning-A-Z-Coursework/commits, commits: 17\n",
1263+
"URL: https://api.github.com/repos/kb22/Digit-Recognition-with-Parameter-Tuning/commits, commits: 12\n",
1264+
"URL: https://api.github.com/repos/kb22/Face-Detection-using-OpenCV/commits, commits: 8\n",
1265+
"URL: https://api.github.com/repos/kb22/GitHub-User-Insights-using-API/commits, commits: 8\n",
1266+
"URL: https://api.github.com/repos/kb22/Heart-Disease-Prediction/commits, commits: 4\n",
1267+
"URL: https://api.github.com/repos/kb22/House-Price-Predictions/commits, commits: 4\n",
1268+
"URL: https://api.github.com/repos/kb22/IEEE-MyEvent-App/commits, commits: 30\n",
1269+
"URL: https://api.github.com/repos/kb22/IEEE-MyEvent-App/commits?page=2, commits: 6\n",
1270+
"URL: https://api.github.com/repos/kb22/Income-Predictor/commits, commits: 10\n",
1271+
"URL: https://api.github.com/repos/kb22/kb22.github.io/commits, commits: 11\n",
1272+
"URL: https://api.github.com/repos/kb22/Machine-Learning-A-Z-CourseWork/commits, commits: 20\n",
1273+
"URL: https://api.github.com/repos/kb22/Malaria-Cell-Identifier-using-CNN/commits, commits: 5\n",
1274+
"URL: https://api.github.com/repos/kb22/ML-Performance-Evaluation/commits, commits: 4\n",
1275+
"URL: https://api.github.com/repos/kb22/ML-React-App-Template/commits, commits: 25\n",
1276+
"URL: https://api.github.com/repos/kb22/PFWebsite/commits, commits: 30\n",
1277+
"URL: https://api.github.com/repos/kb22/PFWebsite/commits?page=2, commits: 1\n",
1278+
"URL: https://api.github.com/repos/kb22/Plot-Maps-in-Python/commits, commits: 15\n",
1279+
"URL: https://api.github.com/repos/kb22/Purchasing-Intention-Prediction/commits, commits: 11\n",
1280+
"URL: https://api.github.com/repos/kb22/Quiz-using-Files/commits, commits: 6\n",
1281+
"URL: https://api.github.com/repos/kb22/Silence-It/commits, commits: 25\n",
1282+
"URL: https://api.github.com/repos/kb22/TextEditor/commits, commits: 19\n",
1283+
"URL: https://api.github.com/repos/kb22/Twitter-Sentiment-Analysis/commits, commits: 15\n",
1284+
"URL: https://api.github.com/repos/kb22/Understanding-Flask-and-Flask-RESTPlus/commits, commits: 4\n",
1285+
"URL: https://api.github.com/repos/kb22/Understanding-Numpy/commits, commits: 11\n",
1286+
"URL: https://api.github.com/repos/kb22/Virtual-trainer/commits, commits: 1\n",
1287+
"URL: https://api.github.com/repos/kb22/Virtual-trainer-data-collector/commits, commits: 1\n",
1288+
"URL: https://api.github.com/repos/kb22/Visualization-using-Matplotlib/commits, commits: 7\n",
1289+
"URL: https://api.github.com/repos/kb22/Web-Scraping-using-Python/commits, commits: 10\n",
1290+
"URL: https://api.github.com/repos/kb22/Word-Cloud/commits, commits: 2\n"
1291+
]
1292+
}
1293+
],
12481294
"source": [
12491295
"commits_information = []\n",
12501296
"for i in range(repos_df.shape[0]):\n",
1251-
" response = requests.get(repos_df.loc[i, 'Commits URL'], auth = authentication)\n",
1252-
" response = response.json()\n",
1253-
" for commit in response:\n",
1254-
" commit_data = []\n",
1255-
" commit_data.append(repos_df.loc[i, 'Id'])\n",
1256-
" commit_data.append(commit['sha'])\n",
1257-
" commit_data.append(commit['commit']['committer']['date'])\n",
1258-
" commit_data.append(commit['commit']['message'])\n",
1259-
" commits_information.append(commit_data)"
1297+
" url = repos_df.loc[i, 'Commits URL']\n",
1298+
" page_no = 1\n",
1299+
" while (True):\n",
1300+
" response = requests.get(url, auth = authentication)\n",
1301+
" response = response.json()\n",
1302+
" print(\"URL: {}, commits: {}\".format(url, len(response)))\n",
1303+
" for commit in response:\n",
1304+
" commit_data = []\n",
1305+
" commit_data.append(repos_df.loc[i, 'Id'])\n",
1306+
" commit_data.append(commit['sha'])\n",
1307+
" commit_data.append(commit['commit']['committer']['date'])\n",
1308+
" commit_data.append(commit['commit']['message'])\n",
1309+
" commits_information.append(commit_data)\n",
1310+
" if (len(response) == 30):\n",
1311+
" page_no = page_no + 1\n",
1312+
" url = repos_df.loc[i, 'Commits URL'] + '?page=' + str(page_no)\n",
1313+
" else:\n",
1314+
" break"
12601315
]
12611316
},
12621317
{

‎commits_info.csv

+12
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ Repo Id,Commit Id,Date,Message
6767
186462080,3e2e857513635c97ee19e93256a3aaf26160c2c8,2019-06-08T19:34:44Z,Add data section
6868
186462080,558907381f7528396f2615ab615733846b575676,2019-06-08T19:18:15Z,Add Capstone Project report
6969
186462080,ae326c1c20b77de2c137eac82a34f761be7b33c1,2019-05-27T20:00:59Z,Add notebook with Toronto clustering
70+
186462080,f76934b16693e36d50e261469de2002b46376bbd,2019-05-27T06:09:16Z,Add notebook and dataset with coordinates
71+
186462080,b9687cd21028c4a8ad63a9ad21df2481ada2a394,2019-05-27T05:21:06Z,Add data collection notebook and created dataset
72+
186462080,5afcc31b34c46987f114ee0b79004ac2b8889040,2019-05-13T17:05:17Z,Add capstone notebook
73+
186462080,e6a75ec3ea0807a09b157dbe8e0df399cb6a7986,2019-05-13T17:00:16Z,Initial commit
7074
149577869,d194c1f543a8241c3a7326edd3b61e613e683f6f,2018-09-24T07:26:43Z,Add article link
7175
149577869,6a8484e96d07d8ef4fa43cec66532637d24a512a,2018-09-24T07:23:26Z,Update readme.md
7276
149577869,39414541d01b7a40bc946a56b357cf5cb8ddfd34,2018-09-21T05:15:11Z,Show sample of dataset
@@ -120,6 +124,7 @@ Classification"
120124
173311184,3eb260307b1f861b9d98f7bf9805c0161cc39694,2019-03-02T13:52:24Z,Add requirements and Pipfile
121125
173311184,d557898b8177389cb174aba4534c6729e7f70fe1,2019-03-02T12:38:47Z,Add code to use model and show box around face
122126
173311184,5134f903c91432f534f75e5c912cba39d3ee608e,2019-03-01T14:00:05Z,Initial commit
127+
192858007,69e25f3989d327974e40525fe16082d88aa698e3,2019-06-22T07:43:38Z,Retrieve more than 30+ repos information
123128
192858007,c3a09a4dc2211c395be1b2873e2f90ff986273a4,2019-06-21T13:36:52Z,Update data collection notebook
124129
192858007,e0a2d42ca367806e5178e4f07e6cf98c055d7f19,2019-06-21T10:20:48Z,Update git ignore for python environment
125130
192858007,1adea58db0a3a257160a3ca419a9a411b08af7bb,2019-06-21T10:18:22Z,Add credentials json file
@@ -181,6 +186,12 @@ Decreased floating data"
181186
59972929,a0fd35112c704419236c476fa356db342a9bcece,2016-05-30T04:34:08Z,Merge remote-tracking branch 'refs/remotes/origin/master'
182187
59972929,c53319e4691354c425dce304b4d378dbe185fd7e,2016-05-30T04:33:18Z,Addition Done
183188
59972929,b3e19335d98146fcf2f896ab43eae0ab83007ac0,2016-05-30T02:25:57Z,Delete readme.md
189+
59972929,fe5207f6f4b6cb6bce3d69dba5cc4612a2a1ade9,2016-05-30T02:22:57Z,Merge remote-tracking branch 'refs/remotes/origin/master'
190+
59972929,7b308c02ba7c646ab364e5d3dfc46804e9fc4233,2016-05-30T02:17:11Z,Changed Icon
191+
59972929,c9dd6229693f62760e089dab5c6811935f48bdbd,2016-05-30T02:01:54Z,Update readme.md
192+
59972929,6a8322189732b2fe889edccc7fed5999c4d475c5,2016-05-30T02:01:23Z,Update readme.md
193+
59972929,c2603202b67490c7fa3c22020f91a18dfde7952e,2016-05-30T02:00:27Z,Create readme.md
194+
59972929,17fc2430dfbbbdcf4a32343846871e435eb4d466,2016-05-30T01:51:49Z,Test Release
184195
171322573,5051139f5267e1ec9f3b6c9abaef28ddc544e0bc,2019-05-25T00:22:08Z,Update the dependencies
185196
171322573,0ed489a4da6a4d32a47677108c8d00c9466bdb58,2019-05-13T06:53:30Z,Update README.md
186197
171322573,f434a1e004b108fcbfe09c99321418c52673ed84,2019-03-31T10:13:19Z,Make minor changes to descriptions
@@ -314,6 +325,7 @@ Added session control"
314325
66449389,2cb3b9b620dd08bed84d9388d0138dcdfbbe236f,2016-08-27T08:44:42Z,LoginPage
315326
66449389,6c9584d6a6e8ed5264a83e6db319b55e6f51658e,2016-08-26T16:39:53Z,Images Added
316327
66449389,2664d6ff1525c5a68ab561859d0a0689b7f815eb,2016-08-26T16:11:02Z,Images Added
328+
66449389,1f3c7716de73f7c1513276a72c6d305fec721400,2016-08-26T09:01:53Z,First Commit
317329
183855135,fc91fb9aab0ca7c6575cda6a48bd8952dfec72d4,2019-04-30T15:50:14Z,Merge branch 'master' of https://github.com/kb22/Plot-Maps-in-Python
318330
183855135,a0ddd84e78bbc0d9054bc50d3d96b73fa0d0c513,2019-04-30T15:45:54Z,More descriptive markdown
319331
183855135,09bbd1c9ee577eca78590da30fb2e7c786178cb4,2019-04-29T18:14:31Z,Update README.md

‎repos_info.csv

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ Id,Name,Description,Created on,Updated on,Owner,License,Includes wiki,Forks coun
1111
183209160,Deep-Learning-A-Z-Coursework,The project includes all the files for the Deep Learning A-Z Course on Udemy.,2019-04-24T10:48:09Z,2019-06-05T21:49:42Z,kb22,GNU General Public License v3.0,True,0,0,2,2,https://api.github.com/repos/kb22/Deep-Learning-A-Z-Coursework,https://api.github.com/repos/kb22/Deep-Learning-A-Z-Coursework/commits,https://api.github.com/repos/kb22/Deep-Learning-A-Z-Coursework/languages,Jupyter Notebook
1212
176952555,Digit-Recognition-with-Parameter-Tuning,The project includes using GridSearchCV to identify the best combination of estimator parameters.,2019-03-21T13:41:51Z,2019-05-25T00:47:07Z,kb22,MIT License,True,1,0,1,1,https://api.github.com/repos/kb22/Digit-Recognition-with-Parameter-Tuning,https://api.github.com/repos/kb22/Digit-Recognition-with-Parameter-Tuning/commits,https://api.github.com/repos/kb22/Digit-Recognition-with-Parameter-Tuning/languages,Jupyter Notebook
1313
173311184,Face-Detection-using-OpenCV,Using OpenCV's inbuilt model for Face Detection.,2019-03-01T14:00:04Z,2019-04-30T16:43:08Z,kb22,,True,1,0,2,2,https://api.github.com/repos/kb22/Face-Detection-using-OpenCV,https://api.github.com/repos/kb22/Face-Detection-using-OpenCV/commits,https://api.github.com/repos/kb22/Face-Detection-using-OpenCV/languages,Python
14-
192858007,GitHub-User-Insights-using-API,The project involves collecting data about a user's profile and drawing insights from it.,2019-06-20T05:59:35Z,2019-06-21T13:37:02Z,kb22,MIT License,True,0,0,0,0,https://api.github.com/repos/kb22/GitHub-User-Insights-using-API,https://api.github.com/repos/kb22/GitHub-User-Insights-using-API/commits,https://api.github.com/repos/kb22/GitHub-User-Insights-using-API/languages,Jupyter Notebook
14+
192858007,GitHub-User-Insights-using-API,The project involves collecting data about a user's profile and drawing insights from it.,2019-06-20T05:59:35Z,2019-06-22T07:43:46Z,kb22,MIT License,True,0,0,0,0,https://api.github.com/repos/kb22/GitHub-User-Insights-using-API,https://api.github.com/repos/kb22/GitHub-User-Insights-using-API/commits,https://api.github.com/repos/kb22/GitHub-User-Insights-using-API/languages,Jupyter Notebook
1515
169992081,Heart-Disease-Prediction,A project that predicts whether a person is suffering from heart disease or not.,2019-02-10T15:21:48Z,2019-05-29T03:00:11Z,kb22,MIT License,True,3,0,7,7,https://api.github.com/repos/kb22/Heart-Disease-Prediction,https://api.github.com/repos/kb22/Heart-Disease-Prediction/commits,https://api.github.com/repos/kb22/Heart-Disease-Prediction/languages,Jupyter Notebook
1616
169389821,House-Price-Predictions,The project exposes Jupyter cells as REST endpoints.,2019-02-06T10:37:24Z,2019-04-17T14:37:59Z,kb22,MIT License,True,1,0,2,2,https://api.github.com/repos/kb22/House-Price-Predictions,https://api.github.com/repos/kb22/House-Price-Predictions/commits,https://api.github.com/repos/kb22/House-Price-Predictions/languages,Jupyter Notebook
1717
59972929,IEEE-MyEvent-App,Android App - IEEE MyEvent App,2016-05-30T01:49:16Z,2019-02-12T11:03:13Z,kb22,,True,2,0,1,1,https://api.github.com/repos/kb22/IEEE-MyEvent-App,https://api.github.com/repos/kb22/IEEE-MyEvent-App/commits,https://api.github.com/repos/kb22/IEEE-MyEvent-App/languages,"PHP, Java"

0 commit comments

Comments
 (0)