Merge pull request #1119 from dipu-bd/dev

Version 2.28.6

.env.example

@@ -16,6 +16,13 @@ DISCORD_SIGNAL_CHAR=!
 PUBLIC_DATA_PATH=
 PUBLIC_ADDRESS=http://18.218.187.242/
 
+# Cloud drives for upload | Options: [GOFILE, GOOGLE_DRIVE] | Default: GOFILE
+CLOUD_DRIVE=GOFILE
+
+# Gofile config. Token is optional.
+GOFILE_TOKEN=kRgxIJe0D724Sdq0U12Hy0KwGhY10b1z
+GOFILE_FOLDER_ID=1247451e-2730-46b3-8dd8-8cff8cb18a5c
+
 # Google Drive Config
 GOOGLE_DRIVE_CREDENTIAL_FILE=mycreds.txt
 GOOGLE_DRIVE_FOLDER_ID=118iN1jzavVV-9flrLPZo7DOi0cuxrQ5F

README.md

@@ -205,7 +205,7 @@ $ lncrawl -h
 ┃┃╱╭╋┫╭╮┃╭╮┃┃┃╭╮┫╭╮┃╰╯┃┃━┫┃╱┃┃╱╭┫╭┫╭╮┃╰╯╰╯┃┃┃┃━┫╭╯
 ┃╰━╯┃┃╰╯┃┃┃┃╰┫┃┃┃╰╯┣╮╭┫┃━┫╰╮┃╰━╯┃┃┃╭╮┣╮╭╮╭┫╰┫┃━┫┃
 ╰━━━┻┻━╮┣╯╰┻━┻╯╰┻━━╯╰╯╰━━┻━╯╰━━━┻╯╰╯╰╯╰╯╰╯╰━┻━━┻╯
-╱╱╱╱╱╭━╯┃ v2.28.5
+╱╱╱╱╱╭━╯┃ v2.28.6
 ╱╱╱╱╱╰━━╯ 🔗 https://github.com/dipu-bd/lightnovel-crawler
 --------------------------------------------------------------------------------
 usage: lncrawl [options...]
@@ -375,6 +375,11 @@ You are very welcome to contribute in this project. You can:
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/SirGryphin"><img src="https://avatars.githubusercontent.com/u/36343615?v=4&s=24" alt="SirGryphin" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching"></span><span title="Supports login"></span></td>
+<td><a href="http://ranobes.net/" target="_blank">http://ranobes.net/</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/ranobes.py" title="03 October 2021 10:26:24 AM">5</a></td>
+<td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a></td>
+</tr>
+<tr><td><span title="Supports searching"></span><span title="Supports login"></span></td>
 <td><a href="http://readonlinenovels.com/" target="_blank">http://readonlinenovels.com/</a></td>
 <td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/readonlinenovels.py" title="27 July 2021 10:03:56 PM">61</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/SirGryphin"><img src="https://avatars.githubusercontent.com/u/36343615?v=4&s=24" alt="SirGryphin" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a> <a href="https://github.com/amritoo"><img src="https://avatars.githubusercontent.com/u/45586379?v=4&s=24" alt="amritoo" height="24"/></a></td>
@@ -446,7 +451,7 @@ You are very welcome to contribute in this project. You can:
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
 <td><a href="https://18.foxaholic.com/" target="_blank">https://18.foxaholic.com/</a></td>
-<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/foxaholic.py" title="03 September 2021 07:23:43 PM">69</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/foxaholic.py" title="03 October 2021 09:40:17 AM">71</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/SirGryphin"><img src="https://avatars.githubusercontent.com/u/36343615?v=4&s=24" alt="SirGryphin" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a> <a href="https://github.com/watzeedzad"><img src="https://avatars.githubusercontent.com/u/16551821?v=4&s=24" alt="watzeedzad" height="24"/></a> <a href="https://github.com/frybin"><img src="https://avatars.githubusercontent.com/u/17693407?v=4&s=24" alt="frybin" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
@@ -606,7 +611,7 @@ You are very welcome to contribute in this project. You can:
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
 <td><a href="https://foxaholic.com/" target="_blank">https://foxaholic.com/</a></td>
-<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/foxaholic.py" title="03 September 2021 07:23:43 PM">69</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/foxaholic.py" title="03 October 2021 09:40:17 AM">71</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/SirGryphin"><img src="https://avatars.githubusercontent.com/u/36343615?v=4&s=24" alt="SirGryphin" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a> <a href="https://github.com/watzeedzad"><img src="https://avatars.githubusercontent.com/u/16551821?v=4&s=24" alt="watzeedzad" height="24"/></a> <a href="https://github.com/frybin"><img src="https://avatars.githubusercontent.com/u/17693407?v=4&s=24" alt="frybin" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
@@ -621,7 +626,7 @@ You are very welcome to contribute in this project. You can:
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
 <td><a href="https://global.foxaholic.com/" target="_blank">https://global.foxaholic.com/</a></td>
-<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/foxaholic.py" title="03 September 2021 07:23:43 PM">69</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/foxaholic.py" title="03 October 2021 09:40:17 AM">71</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/SirGryphin"><img src="https://avatars.githubusercontent.com/u/36343615?v=4&s=24" alt="SirGryphin" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a> <a href="https://github.com/watzeedzad"><img src="https://avatars.githubusercontent.com/u/16551821?v=4&s=24" alt="watzeedzad" height="24"/></a> <a href="https://github.com/frybin"><img src="https://avatars.githubusercontent.com/u/17693407?v=4&s=24" alt="frybin" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching"></span><span title="Supports login"></span></td>
@@ -796,7 +801,7 @@ You are very welcome to contribute in this project. You can:
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
 <td><a href="https://m.webnovel.com/" target="_blank">https://m.webnovel.com/</a></td>
-<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/webnovel.py" title="02 September 2021 03:42:50 PM">75</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/webnovel.py" title="03 October 2021 10:34:45 AM">76</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
@@ -941,7 +946,7 @@ You are very welcome to contribute in this project. You can:
 </tr>
 <tr><td><span title="Supports searching"></span><span title="Supports login"></span></td>
 <td><a href="https://ranobes.net/" target="_blank">https://ranobes.net/</a></td>
-<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/ranobes.py" title="29 July 2021 10:59:22 AM">4</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/ranobes.py" title="03 October 2021 10:26:24 AM">5</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching"></span><span title="Supports login"></span></td>
@@ -1111,7 +1116,7 @@ You are very welcome to contribute in this project. You can:
 </tr>
 <tr><td><span title="Supports searching"></span><span title="Supports login"></span></td>
 <td><a href="https://wujizun.com/" target="_blank">https://wujizun.com/</a></td>
-<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/wujizun.py" title="28 July 2021 12:45:29 AM">69</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/wujizun.py" title="03 October 2021 10:39:52 AM">70</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/SirGryphin"><img src="https://avatars.githubusercontent.com/u/36343615?v=4&s=24" alt="SirGryphin" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a> <a href="https://github.com/AncientCatz"><img src="https://avatars.githubusercontent.com/u/69200720?v=4&s=24" alt="AncientCatz" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
@@ -1196,7 +1201,7 @@ You are very welcome to contribute in this project. You can:
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
 <td><a href="https://www.foxaholic.com/" target="_blank">https://www.foxaholic.com/</a></td>
-<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/foxaholic.py" title="03 September 2021 07:23:43 PM">69</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/foxaholic.py" title="03 October 2021 09:40:17 AM">71</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/SirGryphin"><img src="https://avatars.githubusercontent.com/u/36343615?v=4&s=24" alt="SirGryphin" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a> <a href="https://github.com/watzeedzad"><img src="https://avatars.githubusercontent.com/u/16551821?v=4&s=24" alt="watzeedzad" height="24"/></a> <a href="https://github.com/frybin"><img src="https://avatars.githubusercontent.com/u/17693407?v=4&s=24" alt="frybin" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching"></span><span title="Supports login"></span></td>
@@ -1396,7 +1401,7 @@ You are very welcome to contribute in this project. You can:
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>
 <td><a href="https://www.webnovel.com/" target="_blank">https://www.webnovel.com/</a></td>
-<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/webnovel.py" title="02 September 2021 03:42:50 PM">75</a></td>
+<td><a href="https://github.com/dipu-bd/lightnovel-crawler/blob/master/sources/webnovel.py" title="03 October 2021 10:34:45 AM">76</a></td>
 <td><a href="https://github.com/dipu-bd"><img src="https://avatars.githubusercontent.com/u/5158124?v=4&s=24" alt="dipu-bd" height="24"/></a> <a href="https://github.com/yudilee"><img src="https://avatars.githubusercontent.com/u/7065691?v=4&s=24" alt="yudilee" height="24"/></a></td>
 </tr>
 <tr><td><span title="Supports searching">🔍</span><span title="Supports login"></span></td>

lncrawl/VERSION

@@ -1 +1 @@
-2.28.5
+2.28.6

lncrawl/assets/icons.py

@@ -11,6 +11,7 @@ class Icons:
     isLinux = isLinux
     isWindows = isWindows
     hasSupport = isLinux or isMac
+    EOL = '\r\n' if isWindows else '\n'
 
     # --------------------------------------- #
 

lncrawl/binders/text.py

@@ -2,8 +2,11 @@
 import logging
 import os
 import re
+
 from bs4 import BeautifulSoup
 
+from ..assets.icons import Icons
+
 logger = logging.getLogger(__name__)
 
 
@@ -19,7 +22,7 @@ def make_texts(app, data):
                 body = chap['body'].replace('</p><p', '</p>\n<p')
                 soup = BeautifulSoup(body, 'lxml')
                 text = '\n\n'.join(soup.stripped_strings)
-                text = re.sub(r'[\r\n]+', '\r\n\r\n', text)
+                text = re.sub(r'[\r\n]+', Icons.EOL + Icons.EOL, text)
                 file.write(text)
                 text_files.append(file_name)
             # end with

lncrawl/bots/discord/message_handler.py

@@ -345,8 +345,7 @@ def download_novel_info(self):
             logger.exception('Failed to get novel info')
             self.send_sync('Failed to get novel info.\n`%s`' % str(ex))
             self.executor.submit(self.destroy)
-        finally:
-            self.get_current_status = None
+            return
         # end try
 
         # Setup output path
@@ -569,39 +568,40 @@ def publish_files(self):
 
     def upload_file(self, archive):
         # Check file size
+        filename = os.path.basename(archive)
         file_size = os.stat(archive).st_size
         if file_size > 7.99 * 1024 * 1024:
-            self.send_sync(
-                'File %s exceeds 8MB. Uploading To Google Drive.' % os.path.basename(archive))
-            description = 'Generated By : Discord Bot Ebook Smelter'
-            link_id = upload(archive, description)
-            if link_id:
-                self.send_sync('https://drive.google.com/open?id=%s' % link_id)
-            else:
-                self.send_sync('Failed to upload to google drive')
+            self.send_sync(f'File {filename} exceeds 8MB. Using alternative cloud storage.')
+            try:
+                description = 'Generated By : Lightnovel Crawler Discord Bot'
+                direct_link = upload(archive, description)
+                self.send_sync(direct_link)
+            except Exception as e:
+                logger.error('Failed to upload file: %s', archive, e)
+                self.send_sync(f'Failed to upload file: {filename}.\n`Error: {e}`')
             # end if
-        else:
-            # Upload small files to discord directly
-            k = 0
-            while(file_size > 1024 and k < 3):
-                k += 1
-                file_size /= 1024.0
-            # end while
-            self.send_sync(
-                'Uploading %s [%d%s] ...' % (
-                    os.path.basename(archive),
-                    int(file_size * 100) / 100.0,
-                    ['B', 'KB', 'MB', 'GB'][k]
-                )
+            return
+
+        # Upload small files to discord directly
+        k = 0
+        while(file_size > 1024 and k < 3):
+            k += 1
+            file_size /= 1024.0
+        # end while
+        self.send_sync(
+            'Uploading %s [%d%s] ...' % (
+                os.path.basename(archive),
+                int(file_size * 100) / 100.0,
+                ['B', 'KB', 'MB', 'GB'][k]
             )
-            self.wait_for(
-                self.user.send(
-                    file=discord.File(
-                        open(archive, 'rb'),
-                        os.path.basename(archive)
-                    )
+        )
+        self.wait_for(
+            self.user.send(
+                file=discord.File(
+                    open(archive, 'rb'),
+                    os.path.basename(archive)
                 )
             )
-        # end if
+        )
     # end def
 # end class

lncrawl/bots/telegram.py

@@ -665,20 +665,15 @@ def process_download_request(self, bot, job):
             else:
                 update.message.reply_text(
                     'File size more than 50 MB so cannot be sent via telegram bot.\n' +
-                    'Uploading to google drive...')
-                link_id = upload(archive)
-                if link_id:
-                    update.message.reply_text(
-                        'Get your file here:'
-                        'https://drive.google.com/open?id=%s' % link_id
-                    )
-                # end if
+                    'Uploading to alternative cloud storage')
+                try:
+                    description = 'Generated By : Lightnovel Crawler Telegram Bot'
+                    direct_link = upload(archive, description)
+                    update.message.reply_text('Get your file here: %s' % direct_link)
+                except Exception as e:
+                    logger.error('Failed to upload file: %s', archive, e)
+                # end try
             # end if
-
-            # if os.path.exists(archive):
-            #     os.remove(archive)
-            # update.message.reply_text(
-            #     'This file will be deleted on server')
         # end for
 
         self.destroy_app(bot, update, user_data)