Skip to content

Commit

Permalink
remove unnecessary logging
Browse files Browse the repository at this point in the history
  • Loading branch information
multiflexi committed Aug 19, 2024
1 parent 189a20e commit 91a80b0
Showing 1 changed file with 2 additions and 8 deletions.
10 changes: 2 additions & 8 deletions src/collectors/collectors/rss_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,12 @@ def strip_html_tags(html_string):
feed_url = source.parameter_values["FEED_URL"]
links_limit = BaseCollector.read_int_parameter("LINKS_LIMIT", 0, source)

log_manager.log_collector_activity("rss", source.name, f"Starting collector for url: {feed_url}")
log_manager.log_collector_activity("rss", source.name, f"Starting collector for URL: {feed_url}")

user_agent = source.parameter_values["USER_AGENT"]
if user_agent:
feedparser.USER_AGENT = user_agent
log_manager.log_collector_activity("rss", source.name, f"Using user agent: {user_agent}")
else:
log_manager.log_collector_activity("rss", source.name, "No user agent used")

# use system proxy
proxy_handler = None
Expand All @@ -79,7 +77,6 @@ def strip_html_tags(html_string):
# disable proxy - do not use system proxy
if proxy_server == "none": # WTF?
proxy_handler = urllib.request.ProxyHandler({})
log_manager.log_collector_activity("rss", source.name, "No proxy configuration found")
else:
proxy = re.search(r"^(http|https|socks4|socks5|ftp)://([a-zA-Z0-9\-\.\_]+):(\d+)/?$", proxy_server)
if proxy:
Expand All @@ -94,10 +91,8 @@ def strip_html_tags(html_string):
)
elif scheme == "socks4":
proxy_handler = SocksiPyHandler(socks.SOCKS4, host, int(port))
log_manager.log_collector_activity("rss", source.name, "Found SOCKS4 proxy configuration")
elif scheme == "socks5":
proxy_handler = SocksiPyHandler(socks.SOCKS5, host, int(port))
log_manager.log_collector_activity("rss", source.name, "Found SOCKS5 proxy configuration")

# use proxy in urllib
if proxy_handler:
Expand All @@ -106,10 +101,9 @@ def strip_html_tags(html_string):
try:
if proxy_handler:
feed = feedparser.parse(feed_url, handlers=[proxy_handler])
log_manager.log_collector_activity("rss", source.name, "Using proxy for RSS feed")
log_manager.log_collector_activity("rss", source.name, f"Using proxy {proxy_server} for RSS feed")
else:
feed = feedparser.parse(feed_url)
log_manager.log_collector_activity("rss", source.name, "Not using proxy for RSS feed")

log_manager.log_collector_activity("rss", source.name, f"RSS returned feed with {len(feed['entries'])} entries")

Expand Down

0 comments on commit 91a80b0

Please sign in to comment.