From f97f72d18dadc8d07ed0722d28decb0c857a77be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=9B=B2=E8=8F=AF?=
 <42814579+yunwah@users.noreply.github.com>
Date: Wed, 19 Jan 2022 16:15:02 -0500
Subject: [PATCH] Updated script to preload hashes and updated message

The script now uses a new .env variable called PRELOAD which is used
to specify whether the database should be preloaded with hashes.
Currently this is configured to use a role from a specific server.
This will be changed at a later date.
---
 Dockerfile |  1 +
 main.py    | 44 ++++++++++++++++++++++++++++++++++----------
 news.py    | 14 ++++++++++----
 3 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 17a42a9..141929d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,4 +10,5 @@ RUN python3 -m venv venv && \
 RUN mkdir database && \
     touch database/news.db
 
+
 CMD ["venv/bin/python", "main.py"]
\ No newline at end of file
diff --git a/main.py b/main.py
index ec36422..9f97c97 100644
--- a/main.py
+++ b/main.py
@@ -8,24 +8,41 @@ import asyncio
 import json
 import time
 import logging
+import ast
 
 logging.basicConfig(filename="lanews.log", level=logging.DEBUG)
 dotenv_path = join(dirname(__file__), '.env')
 load_dotenv(dotenv_path)
 
 WEBHOOK_URL = os.environ.get('WEBHOOK_URL')
+PRELOAD = ast.literal_eval(os.environ.get('PRELOAD'))
+if WEBHOOK_URL is None:
+    raise Exception()
+if PRELOAD is None:
+    PRELOAD = True
+
 loop = asyncio.get_event_loop()
 
 
-async def publish_news():
+async def preload_news():
+    scraper = NewsScraper(loop=loop)
+    await fetch_news(scraper)
+    await scraper.close()
+
+
+async def fetch_news(scraper):
     logging.debug('Running web scrape...')
+    return await scraper.news_articles()
+
+
+async def publish_news():
     la_news = NewsScraper(loop=loop)
-    articles = await la_news.news_articles()
+    articles = await fetch_news(la_news)
 
     if bool(articles):
         for article in articles:
             payload = {
-                "content": None,
+                "content": '<@&922576289188151307>',
                 "embeds": [
                     {
                         "title": article['title'].replace("'", "\\'"),
@@ -39,15 +56,19 @@ async def publish_news():
                         "image": {
                             "url": article['image_preview']
                         },
-                        #"thumbnail": {
-                        #    "url": "https://images.ctfassets.net/umhrp0op95v1/S3yKwaVAOi8Bgqg4n4scf"
-                        #           "/adae769671b271b88f97d31721432986/LA_LOGO.png "
-                        #}
+                        "thumbnail": {
+                            "url": "https://images.ctfassets.net/umhrp0op95v1/S3yKwaVAOi8Bgqg4n4scf"
+                                   "/adae769671b271b88f97d31721432986/LA_LOGO.png "
+                        }
                     }
                 ]
             }
-            resp = await la_news.client.post(url=WEBHOOK_URL, data=json.dumps(payload).encode('UTF-8'), headers={'Content-Type': 'application/json'})
-            print(resp.status)
+            while True:
+                resp = await la_news.client.post(url=WEBHOOK_URL, data=json.dumps(payload).encode('UTF-8'), headers={'Content-Type': 'application/json'})
+                if resp.status == 204:
+                    break
+                time.sleep(15)
+        time.sleep(5)
     await la_news.close()
 
 
@@ -56,7 +77,10 @@ def run_async(coroutine):
     loop.run_until_complete(task)
 
 
-schedule.every(5).minutes.do(run_async, publish_news)
+if PRELOAD:
+    asyncio.get_event_loop().run_until_complete(preload_news())
+
+schedule.every(1).seconds.do(run_async, publish_news)
 
 while True:
     logging.debug('Checking schedule...')
diff --git a/news.py b/news.py
index 09eee0b..996ee82 100644
--- a/news.py
+++ b/news.py
@@ -3,6 +3,7 @@ from aiohttp import ClientSession
 from sqlite3 import Error
 import hashlib
 import sqlite3
+import time
 
 BASE_URL = 'https://www.playlostark.com'
 
@@ -28,8 +29,11 @@ class NewsScraper:
         self._md5 = hashlib.new('md5', usedforsecurity=False)
 
     async def _fetch_url(self, url):
-        async with self.client.get(url=url) as resp:
-            return await resp.text()
+        while True:
+            async with self.client.get(url=url) as resp:
+                if resp.status == 200:
+                    return await resp.text()
+            time.sleep(15)
 
     def _store_hash(self, _hash, table):
         with self.database as db:
@@ -72,12 +76,14 @@ class NewsScraper:
 
             _hash = hashlib.md5(article_meta.__str__().encode('UTF-8'), usedforsecurity=False).hexdigest()
             if self._check_hash(_hash, 'news_hashes'):
-                return
+                articles.reverse()
+                return articles
             else:
                 self._store_hash(_hash, 'news_hashes')
 
             articles.append(article_meta)
-        return articles.reverse()
+        articles.reverse()
+        return articles
 
     async def close(self):
         await self.client.close()