prepare_step2.py

#!/usr/bin/env python2
from bs4 import BeautifulSoup
from datetime import datetime
import re, random

moons = [
        "Adrastea",
        "Aitne",
        "Amalthea",
        "Ananke",
        "Aoede",
        "Arche",
        "Autonoe",
        "Callirrhoe",
        "Callisto",
        "Carme",
        "Carpo",
        "Chaldene",
        "Cyllene",
        "Elara",
        "Erinome",
        "Euanthe",
        "Eukelade",
        "Euporie",
        "Europa",
        "Eurydome",
        "Ganymede",
        "Harpalyke",
        "Hegemone",
        "Helike",
        "Hermippe",
        "Herse",
        "Himalia",
        "Io",
        "Iocaste",
        "Isonoe",
        "Kale",
        "Kallichore",
        "Kalyke",
        "Kore",
        "Leda",
        "Lysithea",
        "Megaclite",
        "Metis",
        "Mneme",
        "Orthosie",
        "Pasiphae",
        "Pasithee",
        "Praxidike",
        "Sinope",
        "Sponde",
        "Taygete",
        "Thebe",
        "Thelxinoe",
        "Themisto",
        "Thyone"
        ]

def random_moon(matchobj):
    return "#" + random.sample(moons, 1)[0]

def random_mayo(matchobj):
    if random.randint(0, 100) < 10:
        possible = ["#spider", "#mayo"]
        return random.sample(possible, 1)[0]
    else:
        return matchobj.group(0)

def main():
    prepared_html = open("prepared.html")
    soup = BeautifulSoup(prepared_html, "lxml")
    prepared_html.close()

    # Save intermediate step
    output = open("prepared2.html", "w")
    all_ps = soup.find_all("p")
    print "Number of <p> tags found: {}".format(len(all_ps))
    all_lines = [x.get_text().encode("ascii", errors="ignore") for x in all_ps]
    print "Number of lines found: {}".format(len(all_lines))
    all_text = "\n".join(all_lines)
    output.write(all_text)
    output.close()

    to_replace = ["@?postgres", "@?mongo", "@?couch", "@?riak", "@?basho", "@?level",
                  "@?cassandra", "@?mysql", "@?maria", "@?redis", "@?rethink", "@?inno",
                  "@?volt", "@?datomic"]
    for item in to_replace:
        regex = re.compile(item, re.IGNORECASE)
        all_text = regex.sub("Oleg", all_text) # This is Oleg and not OlegDB for a reason

    for item in range(2000, datetime.now().year):
        year_str = r"{}".format(item)
        current_year_str = r"{}".format(datetime.now().year)
        regex = re.compile(year_str, re.IGNORECASE)
        all_text = regex.sub(current_year_str, all_text) # This is Oleg and not OlegDB for a reason

    regex = re.compile(r"@[\w]+")
    all_text = regex.sub(random_moon, all_text)
    regex = re.compile(r"#[\w]+")
    all_text = regex.sub(random_mayo, all_text)
    regex = re.compile(r"[\?:\(\s^]?.?https?:\/\/[\w\.\/\)\?\d&=-]*")
    all_text = regex.sub("", all_text)

    output = open("prepared3.txt", "w")
    output.write(all_text)
    output.close()


if __name__ == '__main__':
    main()