-
Notifications
You must be signed in to change notification settings - Fork 2
/
fetch-greader.py
executable file
·100 lines (85 loc) · 3.13 KB
/
fetch-greader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from datetime import datetime
import argparse
import sys
import os
from libgreader import GoogleReader, ClientAuthMethod, Feed
from slugify import slugify
OUTPUT_TEMPLATE = u"""
<!DOCTYPE html>
<html><head>
<meta charset="utf-8" />
<title>{1}</title>
</head>
<body>
<h3><a href="{0}">{1}</a></h3>\n
</body>
</html>
"""
def authenticate(username, password):
auth = ClientAuthMethod(username, password)
return GoogleReader(auth)
def item_directory(item):
date = datetime.fromtimestamp(item.time)
slug = slugify(item.title, max_length=180, word_boundary=True)
return '%s-%s' % (date.strftime('%Y-%m-%d'), slug)
def save(item, directory):
itemdir = '%s/%s' % (directory, item_directory(item))
if not os.path.isdir(itemdir):
os.makedirs(itemdir)
filename = '%s/index.html' % itemdir
print "---> %s" % filename
fp = open(filename, 'wb')
fp.write(OUTPUT_TEMPLATE.format(item.url, item.title).encode('utf-8'))
fp.write(item.content.encode('utf-8'))
fp.close()
def fetch(feed, directory, starred=False, load_limit=100):
count = 0
until = None
while True:
feed.loadItems(loadLimit=load_limit, until=until)
if feed.countItems() == 0:
break
count += feed.countItems()
until = feed.items[-1].time - 1
for item in feed.items:
if not starred or item.starred:
save(item, directory)
return count
def main():
parser = argparse.ArgumentParser(description='Google Reader RSS archive fetcher')
parser.add_argument('-u', '--username', dest='username', help='user name')
parser.add_argument('-p', '--password', dest='password', help='account password')
parser.add_argument('-f', '--feed', dest='feed', help='feed number to fetch')
parser.add_argument('-s', '--starred', dest='starred', action='store_true',
default=False, help='store only starred items')
parser.add_argument('-d', '--dir', dest='dir', help='directory name to store feed contents')
args = parser.parse_args()
if None in (args.username, args.password):
print "* Please specify username and password *\n"
parser.print_help()
sys.exit(1)
reader = authenticate(args.username, args.password)
# fetch list of feeds
reader.buildSubscriptionList()
if args.feed:
feed = reader.getSubscriptionList()[int(args.feed)]
# create output directory
if args.dir is None:
directory = slugify(feed.title, max_length=60, word_boundary=True)
else:
directory = args.dir
print "* Output directory: %s *" % directory
if not os.path.isdir(directory):
os.makedirs(directory)
# fetch the feed
fetch(feed, directory, starred=args.starred)
else:
# enumerate feeds
print "* Please specify feed number (-f, --feed) to fetch: *"
for i, feed in enumerate(reader.getSubscriptionList()):
print "[{0}] {1} [{2}]".format(i, feed.title, feed.feedUrl).encode('utf-8')
if __name__ == "__main__":
main()