Skip to content

Commit

Permalink
fix bug in web collector
Browse files Browse the repository at this point in the history
  • Loading branch information
multiflexi committed Oct 29, 2024
1 parent eeee93f commit 38172b4
Showing 1 changed file with 7 additions and 3 deletions.
10 changes: 7 additions & 3 deletions src/collectors/collectors/web_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -788,8 +788,12 @@ def __process_title_page_articles(self, browser, title_page_handle, index_url):
news_item = self.__process_article_page(index_url, browser)
if news_item:
logger.debug(f"{self.collector_source} ... Title : {news_item.title}")
logger.debug(f"{self.collector_source} ... Review : {news_item.review.replace('\r', '').replace('\n', ' ').strip()[:100]}")
logger.debug(f"{self.collector_source} ... Content : {news_item.content.replace('\r', '').replace('\n', ' ').strip()[:100]}")
logger.debug(
f"{self.collector_source} ... Review : {news_item.review.replace('\r', '').replace('\n', ' ').strip()[:100]}"
)
logger.debug(
f"{self.collector_source} ... Content : {news_item.content.replace('\r', '').replace('\n', ' ').strip()[:100]}"
)
logger.debug(f"{self.collector_source} ... Published: {news_item.published}")
self.news_items.append(news_item)
else:
Expand Down Expand Up @@ -871,6 +875,6 @@ def __process_article_page(self, index_url, browser):
key = "Additional_ID"
binary_mime_type = ""
binary_value = ""
attribute = NewsItemAttribute(uuid.uuid4(), key, value, binary_mime_type, binary_value)
attribute = NewsItemAttribute(key, value, binary_mime_type, binary_value)
news_item.attributes.append(attribute)
return news_item

0 comments on commit 38172b4

Please sign in to comment.