April 2011 Merge

* Show "this post is archived" message for things we won't allow votes on.
* Don't mark messages read when pulled via RSS
* Make compact (smartphone) interface respect user's toolbar
* preferences.
* Make mobile interface more friendly for kindles.
* Fix bug that caused comment tree corruption.
* Use cachebuster on traffic pixel for more accurate tracking.
* Make apps restart themselves after a configurable number of requests.
* Move to pycassa 1.0.8.
* Fix bug in calculations for "best" sort.
* Fixes for Firefox Mobile
* Add a global flag to disable editing of the wiki.
* Move the child-comment collapse button to the left.
* Updated list of disallows in robots.txt to save needless hits from
  crawlers.
* Fix vote_q by splitting it into vote_link_q and vote_comment_q.
* Fix bug where /reddits crashes due to 'promos' subreddit.

Author: spladug

config/cassandra/cassandra.yaml

@@ -92,26 +92,7 @@ commitlog_sync_period_in_ms: 10000
 # the topology of the ring.  You must change this if you are running
 # multiple nodes!
 seeds:
-    - pmc01
-    - pmc02
-    - pmc03
-    - pmc04
-    - pmc05
-    - pmc06
-    - pmc07
-    - pmc08
-    - pmc09
-    - pmc10
-    - pmc11
-    - pmc12
-    - pmc13
-    - pmc14
-    - pmc15
-    - pmc16
-    - pmc17
-    - pmc18
-    - pmc19
-    - pmc20
+    - 127.0.0.1
 
 # Access mode.  mmapped i/o is substantially faster, but only practical on
 # a 64bit machine (which notably does not include EC2 "small" instances)
@@ -155,7 +136,7 @@ storage_port: 7000
 # address associated with the hostname (it might not be).
 #
 # Setting this to 0.0.0.0 is always wrong.
-listen_address:
+listen_address: 127.0.0.1
 
 # The address to bind the Thrift RPC service to -- clients connect
 # here. Unlike ListenAddress above, you *can* specify 0.0.0.0 here if
@@ -425,7 +406,7 @@ index_interval: 128
 keyspaces:
     - name: reddit
       replica_placement_strategy: org.apache.cassandra.locator.RackUnawareStrategy
-      replication_factor: 3
+      replication_factor: 1
       column_families:
       - column_type: Standard
         compare_with: BytesType

r2/Makefile

@@ -24,7 +24,7 @@
 js_targets  = jquery.js jquery.json.js jquery.reddit.js reddit.js ui.core.js ui.datepicker.js sponsored.js jquery.flot.js compact.js
 # CSS targets 
 main_css = reddit.css
-css_targets = reddit-ie6-hax.css reddit-ie7-hax.css mobile.css spreadshirt.css compact.css
+css_targets = reddit-ie6-hax.css reddit-ie7-hax.css mobile.css spreadshirt.css compact.css 
 
 SED=sed
 CAT=cat

r2/example.ini

@@ -23,6 +23,8 @@ log_start = true
 amqp_logging = false
 # emergency measures: makes the site read only
 read_only_mode = false
+# global switch for wiki write permissions
+allow_wiki_editing = true
 
 # -- SECRETS! <-- update these first! --
 # global secret
@@ -37,6 +39,8 @@ INDEXTANK_API_URL =
 # -- important settings -- 
 # the domain that this app serves itself up as
 domain = reddit.local
+# the short domain (like redd.it)
+shortdomain =
 # if you use www for the old-timey feel, put it here
 domain_prefix = 
 # the user used for "system" operations and messages
@@ -310,6 +314,8 @@ MIN_RATE_LIMIT_COMMENT_KARMA = 1
 QUOTA_THRESHOLD = 5
 # Links and comments older than this many days qualify for historic preservation
 REPLY_AGE_LIMIT = 180
+# Links and comments older than this many days can't be voted on
+VOTE_AGE_LIMIT = 180
 
 # min amount of karma to edit 
 WIKI_KARMA = 100
@@ -341,6 +347,11 @@ sr_dropdown_threshold = 15
 # seconds of each other
 comment_visits_period = 600
 
+# Set this to a nonzero range and the server will restart after this many
+# minutes have passed
+LOGANS_RUN_LOW = 0
+LOGANS_RUN_HIGH = 0
+
 #user-agents to rate-limit
 agents = 
 

r2/r2/controllers/api.py

@@ -313,7 +313,6 @@ def POST_submit(self, form, jquery, url, selftext, kind, title,
                 form.set_html(".status", md)
                 return
 
-
         # well, nothing left to do but submit it
         l = Link._submit(request.post.title, url if kind == 'link' else 'self',
                          c.user, sr, ip, spam=c.user._spam)
@@ -819,6 +818,7 @@ def POST_comment(self, commentform, jquery, parent, comment, ip):
             parent_age = c.start_time - parent._date
             if parent_age.days > g.REPLY_AGE_LIMIT:
                 c.errors.add(errors.TOO_OLD, field = "parent")
+
         #remove the ratelimit error if the user's karma is high
         if not should_ratelimit:
             c.errors.remove((errors.RATELIMIT, 'ratelimit'))
@@ -975,8 +975,6 @@ def POST_juryvote(self, dir, thing, ip):
                 dir = VInt('dir', min=-1, max=1),
                 thing = VByName('id'))
     def POST_vote(self, dir, thing, ip, vote_type):
-        from r2.models.admintools import valid_vote
-
         ip = request.ip
         user = c.user
         store = True
@@ -988,7 +986,9 @@ def POST_vote(self, dir, thing, ip, vote_type):
             reject_vote(thing)
             store = False
 
-        if not valid_vote(thing):
+        thing_age = c.start_time - thing._date
+        if thing_age.days > g.VOTE_AGE_LIMIT:
+            g.log.debug("ignoring vote on old thing %s" % thing._fullname)
             store = False
 
         if getattr(c.user, "suspicious", False):
@@ -1626,9 +1626,11 @@ def POST_password(self, form, jquery, user):
         elif form.has_errors('name', errors.NO_EMAIL_FOR_USER):
             return
         else:
-            emailer.password_email(user)
-            form.set_html(".status",
-                          _("an email will be sent to that account's address shortly"))
+            if emailer.password_email(user):
+                form.set_html(".status",
+                      _("an email will be sent to that account's address shortly"))
+            else:
+                form.set_html(".status", _("try again tomorrow"))
 
 
     @validatedForm(cache_evt = VCacheKey('reset', ('key',)),

r2/r2/controllers/front.py

@@ -240,13 +240,15 @@ def GET_comments(self, article, comment, context, sort, limit, depth):
             if num > g.max_comments_gold:
                 displayPane.append(InfoBar(message =
                                            strings.over_comment_limit_gold
-                                           % g.max_comments_gold))
+                                           % max(0, g.max_comments_gold)))
                 num = g.max_comments_gold
         elif num > g.max_comments:
-            displayPane.append(InfoBar(message =
+            if limit:
+                displayPane.append(InfoBar(message =
                                        strings.over_comment_limit
-                                       % dict(max=g.max_comments,
-                                              goldmax=g.max_comments_gold)))
+                                       % dict(max=max(0, g.max_comments),
+                                              goldmax=max(0,
+                                                   g.max_comments_gold))))
             num = g.max_comments
 
         # if permalink page, add that message first to the content
@@ -960,7 +962,7 @@ def GET_validuser(self):
         returns their user name"""
         c.response_content_type = 'text/plain'
         if c.user_is_loggedin:
-            perm = str(c.user.can_wiki())
+            perm = str(g.allow_wiki_editing and c.user.can_wiki())
             c.response.content = c.user.name + "," + perm
         else:
             c.response.content = ''
@@ -1064,3 +1066,4 @@ def GET_gold(self, goldtype, period, months,
                                                   giftmessage, passthrough)
                               ).render()
 
+

r2/r2/controllers/listingcontroller.py

@@ -723,7 +723,7 @@ def query(self):
 
     @validate(VUser(),
               message = VMessageID('mid'),
-              mark = VOneOf('mark',('true','false'), default = 'true'))
+              mark = VOneOf('mark',('true','false')))
     def GET_listing(self, where, mark, message, subwhere = None, **env):
         if not (c.default_sr or c.site.is_moderator(c.user) or c.user_is_admin):
             abort(403, "forbidden")
@@ -732,7 +732,14 @@ def GET_listing(self, where, mark, message, subwhere = None, **env):
         else:
             self.where = where
         self.subwhere = subwhere
-        self.mark = mark
+        if mark is not None:
+            self.mark = mark
+        elif is_api():
+            self.mark = 'false'
+        elif c.render_style and c.render_style == "xml":
+            self.mark = 'false'
+        else:
+            self.mark = 'true'
         self.message = message
         return ListingController.GET_listing(self, **env)
 
@@ -777,6 +784,11 @@ def query(self):
             # Consider resurrecting when it is not the World Cup
             #if c.content_langs != 'all':
             #    reddits._filter(Subreddit.c.lang == c.content_langs)
+
+            if g.domain != 'reddit.com':
+                # don't try to render special subreddits (like promos)
+                reddits._filter(Subreddit.c.author_id != -1)
+
             if not c.over18:
                 reddits._filter(Subreddit.c.over_18 == False)
 

r2/r2/controllers/reddit_base.py

@@ -594,6 +594,11 @@ def post(self):
                                     domain  = v.domain,
                                     expires = v.expires)
 
+        if g.logans_run_limit:
+            if c.start_time > g.logans_run_limit and not g.shutdown:
+                g.log.info("Time to restart. It's been an honor serving with you.")
+                g.shutdown = 'init'
+
         if g.usage_sampling <= 0.0:
             return
 
@@ -720,7 +725,7 @@ def pre(self):
                 c.show_mod_mail = Subreddit.reverse_moderator_ids(c.user)
             c.user_is_admin = maybe_admin and c.user.name in g.admins
             c.user_is_sponsor = c.user_is_admin or c.user.name in g.sponsors
-            if not g.disallow_db_writes:
+            if request.path != '/validuser' and not g.disallow_db_writes:
                 c.user.update_last_visit(c.start_time)
 
         c.over18 = over18()
@@ -819,3 +824,4 @@ def search_fail(self, exception):
         request.environ['retry_after'] = 60
 
         abort(503)
+

r2/r2/controllers/validator/validator.py

@@ -520,16 +520,17 @@ class VByName(Validator):
     splitter = re.compile('[ ,]+')
     def __init__(self, param, thing_cls = None, multiple = False,
                  error = errors.NO_THING_ID, **kw):
-        self.re = fullname_regex(thing_cls, multiple)
+        self.re = fullname_regex(thing_cls)
         self.multiple = multiple
         self._error = error
 
         Validator.__init__(self, param, **kw)
 
     def run(self, items):
-        if items and self.re.match(items):
-            if self.multiple:
-                items = filter(None, self.splitter.split(items))
+        if items and self.multiple:
+            items = [item for item in self.splitter.split(items)
+                     if item and self.re.match(item)]
+        if items and (self.multiple or self.re.match(items)):
             try:
                 return Thing._by_fullname(items, return_dict = False,
                                           data=True)

r2/r2/lib/app_globals.py

@@ -47,12 +47,15 @@ class Globals(object):
                  'MIN_DOWN_KARMA',
                  'MIN_RATE_LIMIT_KARMA',
                  'MIN_RATE_LIMIT_COMMENT_KARMA',
+                 'VOTE_AGE_LIMIT',
                  'REPLY_AGE_LIMIT',
                  'WIKI_KARMA',
                  'HOT_PAGE_AGE',
                  'MODWINDOW',
                  'RATELIMIT',
                  'QUOTA_THRESHOLD',
+                 'LOGANS_RUN_LOW',
+                 'LOGANS_RUN_HIGH',
                  'num_comments',
                  'max_comments',
                  'max_comments_gold',
@@ -84,6 +87,7 @@ class Globals(object):
                   'amqp_logging',
                   'read_only_mode',
                   'frontpage_dart',
+                  'allow_wiki_editing',
                   ]
 
     tuple_props = ['stalecaches',
@@ -177,6 +181,7 @@ def __init__(self, global_conf, app_conf, paths, **extra):
             raise ValueError("cassandra_seeds not set in the .ini")
         self.cassandra = PycassaConnectionPool('reddit',
                                                server_list = self.cassandra_seeds,
+                                               pool_size = len(self.cassandra_seeds),
                                                # TODO: .ini setting
                                                timeout=15, max_retries=3,
                                                prefill=False)
@@ -365,6 +370,18 @@ def reset_caches():
                            (self.reddit_host, self.reddit_pid,
                             self.short_version, datetime.now()))
 
+        if self.LOGANS_RUN_LOW:
+            minutes_to_live = random.randint(self.LOGANS_RUN_LOW,
+                                             self.LOGANS_RUN_HIGH)
+            self.logans_run_limit = datetime.now(self.tz) + timedelta(0,
+                                                 minutes_to_live * 60)
+            disptime = self.logans_run_limit.astimezone(self.display_tz)
+            self.log.info("Will shut down at %s (%d minutes from now)" %
+                (disptime.strftime("%H:%M:%S"), minutes_to_live))
+        else:
+            self.logans_run_limit = None
+
+
     @staticmethod
     def to_bool(x):
         return (x.lower() == 'true') if x else None

r2/r2/lib/comment_tree.py

@@ -80,6 +80,7 @@ def find_parents():
                 for child in comment_tree[cur_cm]:
                     stack.append(child)
 
+    new_parents = {}
     for comment in comments:
         cm_id = comment._id
         p_id = comment.parent_id
@@ -103,6 +104,7 @@ def find_parents():
 
         #if this comment had a parent, find the parent's parents
         if p_id:
+            new_parents[cm_id] = p_id
             for p_id in find_parents():
                 num_children[p_id] += 1
 
@@ -115,7 +117,18 @@ def find_parents():
 
     for comment in comments:
         cm_id = comment._id
-        r[cm_id] = p_id
+#        print "In the olden days, I would have set %s -> %s" % (cm_id, p_id)
+
+    for cm_id, parent_id in new_parents.iteritems():
+#        print "Now, I set %s -> %s" % (cm_id, parent_id)
+        r[cm_id] = parent_id
+
+    for comment in comments:
+        cm_id = comment._id
+        if cm_id not in new_parents:
+            r[cm_id] = None
+#            print "And I set %s -> None" % cm_id
+
     g.permacache.set(key, r)
 
     g.permacache.set(comments_key(link_id),

r2/r2/lib/db/_sorts.pyx

@@ -41,15 +41,18 @@ cpdef double _confidence(int ups, int downs):
     """The confidence sort.
        http://www.evanmiller.org/how-not-to-sort-by-average-rating.html"""
     cdef float n = ups + downs
-    cdef float z
-    cdef float phat
 
     if n == 0:
         return 0
 
-    z = 1.0 #1.0 = 85%, 1.6 = 95%
-    phat = float(ups) / n
-    return sqrt(phat+z*z/(2*n)-z*((phat*(1-phat)+z*z/(4*n))/n))/(1+z*z/n)
+    cdef float z = 1.281551565545 # 80% confidence
+    cdef float p = float(ups) / n
+
+    left = p + 1/(2*n)*z*z
+    right = z*sqrt(p*(1-p)/n + z*z/(4*n*n))
+    under = 1+1/n*z*z
+
+    return (left - right) / under
 
 cdef int up_range = 400
 cdef int down_range = 100