fix(article): corrected url being passed to scraping function

JaleelB · Aug 2, 2024 · 92e35be · 92e35be
1 parent d89b35e
commit 92e35be
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 6 deletions.
diff --git a/app/article/actions/article.ts b/app/article/actions/article.ts
@@ -25,7 +25,17 @@ export async function scrapeArticleContent(url: string) {
       throw new Error("Invalid URL");
     }
 
-    const response = await fetch(url);
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+        Connection: "keep-alive",
+      },
+    });
+
     if (!response.ok) {
       throw new Error(
         `Failed to retrieve the web page. Status code: ${response.status}`,

diff --git a/app/article/article-wrapper.tsx b/app/article/article-wrapper.tsx
@@ -23,8 +23,8 @@ export const getCachedArticle = unstable_cache(
 );
 
 async function ArticleLoader({ url }: { url: string }) {
-  // const content = await getCachedArticle(url);
-  const content = await scrapeArticleContent(url);
+  const content = await getCachedArticle(url);
+  // const content = await scrapeArticleContent(url);
 
   if (!content) {
     return <ErrorCard />;
@@ -76,13 +76,13 @@ export async function ArticleWrapper({ url }: { url: string }) {
 
   // if browser is requesting html it means it's the first page load
   if (headers().get("accept")?.includes("text/html")) {
-    // article = await getCachedArticle(url);
-    article = await scrapeArticleContent(url);
+    article = await getCachedArticle(url);
+    // article = await scrapeArticleContent(url);
   }
 
   return (
     <SuspenseIf condition={!article} fallback={<ArticleSkeleton />}>
-      <ArticleLoader url={url} />
+      <ArticleLoader url={urlWithoutPaywall} />
     </SuspenseIf>
   );
 }