Merge branch 'release/v0.5.2'

pytchat/__init__.py

@@ -2,7 +2,7 @@
 pytchat is a lightweight python library to browse youtube livechat without Selenium or BeautifulSoup.
 """
 __copyright__    = 'Copyright (C) 2019, 2020 taizan-hokuto'
-__version__      = '0.5.1'
+__version__      = '0.5.2'
 __license__      = 'MIT'
 __author__       = 'taizan-hokuto'
 __author_email__ = '55448286+taizan-hokuto@users.noreply.github.com'

pytchat/core/pytchat.py

@@ -44,6 +44,10 @@ class PytchatCore:
         If True, when exceptions occur, the exception is held internally,
         and can be raised by raise_for_status().
 
+    replay_continuation : str
+        If this parameter is not None, the processor will attempt to get chat data from continuation.
+        This parameter is only allowed in archived mode.
+
     Attributes
     ---------
     _is_alive : bool
@@ -58,6 +62,7 @@ def __init__(self, video_id,
                  topchat_only=False,
                  hold_exception=True,
                  logger=config.logger(__name__),
+                 replay_continuation=None
                  ):
         self._video_id = util.extract_video_id(video_id)
         self.seektime = seektime
@@ -66,32 +71,33 @@ def __init__(self, video_id,
         else:
             self.processor = processor
         self._is_alive = True
-        self._is_replay = force_replay
+        self._is_replay = force_replay or (replay_continuation is not None)
         self._hold_exception = hold_exception
         self._exception_holder = None
         self._parser = Parser(
             is_replay=self._is_replay,
             exception_holder=self._exception_holder
         )
-        self._first_fetch = True
-        self._fetch_url = config._sml
+        self._first_fetch = replay_continuation is None
+        self._fetch_url = config._sml if replay_continuation is None else config._smr
         self._topchat_only = topchat_only
         self._dat = ''
         self._last_offset_ms = 0
         self._logger = logger
+        self.continuation = replay_continuation
         if interruptable:
             signal.signal(signal.SIGINT, lambda a, b: self.terminate())
         self._setup()
 
     def _setup(self):
-        time.sleep(0.1)  # sleep shortly to prohibit skipping fetching data
-        """Fetch first continuation parameter,
-        create and start _listen loop.
-        """
-        self.continuation = liveparam.getparam(self._video_id, past_sec=3)
-
+        if not self.continuation:
+            time.sleep(0.1)  # sleep shortly to prohibit skipping fetching data
+            """Fetch first continuation parameter,
+            create and start _listen loop.
+            """
+            self.continuation = liveparam.getparam(self._video_id, past_sec=3)
+
     def _get_chat_component(self):
-
         ''' Fetch chat data and store them into buffer,
         get next continuaiton parameter and loop.
 
@@ -178,7 +184,7 @@ def _get_livechat_json(self, continuation, client, replay: bool, offset_ms: int
                                f"Exceeded retry count. Last error: {str(err)}")
             self._raise_exception(exceptions.RetryExceedMaxCount())
         return livechat_json
-    
+
     def get(self):
         if self.is_alive():
             chat_component = self._get_chat_component()

pytchat/core_async/livechat.py

@@ -62,6 +62,10 @@ class LiveChatAsync:
     topchat_only : bool
         If True, get only top chat.
 
+    replay_continuation : str
+        If this parameter is not None, the processor will attempt to get chat data from continuation.
+        This parameter is only allowed in archived mode.
+
     Attributes
     ---------
     _is_alive : bool
@@ -81,7 +85,8 @@ def __init__(self, video_id,
                  direct_mode=False,
                  force_replay=False,
                  topchat_only=False,
-                 logger=config.logger(__name__)
+                 logger=config.logger(__name__),
+                 replay_continuation=None
                  ):
         self._video_id = util.extract_video_id(video_id)
         self.seektime = seektime
@@ -95,17 +100,18 @@ def __init__(self, video_id,
         self._exception_handler = exception_handler
         self._direct_mode = direct_mode
         self._is_alive = True
-        self._is_replay = force_replay
+        self._is_replay = force_replay or (replay_continuation is not None)
         self._parser = Parser(is_replay=self._is_replay)
         self._pauser = Queue()
         self._pauser.put_nowait(None)
-        self._first_fetch = True
-        self._fetch_url = config._sml
+        self._first_fetch = replay_continuation is None
+        self._fetch_url = config._sml if replay_continuation is None else config._smr
         self._topchat_only = topchat_only
         self._dat = ''
         self._last_offset_ms = 0
         self._logger = logger
         self.exception = None
+        self.continuation = replay_continuation
         LiveChatAsync._logger = logger
 
         if exception_handler:
@@ -145,8 +151,9 @@ async def _startlisten(self):
         """Fetch first continuation parameter,
         create and start _listen loop.
         """
-        initial_continuation = liveparam.getparam(self._video_id, 3)
-        await self._listen(initial_continuation)
+        if not self.continuation:
+            self.continuation = liveparam.getparam(self._video_id, 3)
+        await self._listen(self.continuation)
 
     async def _listen(self, continuation):
         ''' Fetch chat data and store them into buffer,
@@ -163,6 +170,9 @@ async def _listen(self, continuation):
                     continuation = await self._check_pause(continuation)
                     contents = await self._get_contents(continuation, client, headers)
                     metadata, chatdata = self._parser.parse(contents)
+                    continuation = metadata.get('continuation')
+                    if continuation:
+                        self.continuation = continuation
                     timeout = metadata['timeoutMs'] / 1000
                     chat_component = {
                         "video_id": self._video_id,
@@ -181,7 +191,6 @@ async def _listen(self, continuation):
                         await self._buffer.put(chat_component)
                     diff_time = timeout - (time.time() - time_mark)
                     await asyncio.sleep(diff_time)
-                    continuation = metadata.get('continuation')
                     self._last_offset_ms = metadata.get('last_offset_ms', 0)
         except exceptions.ChatParseException as e:
             self._logger.debug(f"[{self._video_id}]{str(e)}")
@@ -242,7 +251,6 @@ async def _get_livechat_json(self, continuation, client, replay: bool, offset_ms
         '''
         Get json which includes chat data.
         '''
-        # continuation = urllib.parse.quote(continuation)
         livechat_json = None
         if offset_ms < 0:
             offset_ms = 0

pytchat/core_multithread/livechat.py

@@ -60,6 +60,10 @@ class LiveChat:
     topchat_only : bool
         If True, get only top chat.
 
+    replay_continuation : str
+        If this parameter is not None, the processor will attempt to get chat data from continuation.
+        This parameter is only allowed in archived mode.
+
     Attributes
     ---------
     _executor : ThreadPoolExecutor
@@ -81,7 +85,8 @@ def __init__(self, video_id,
                  direct_mode=False,
                  force_replay=False,
                  topchat_only=False,
-                 logger=config.logger(__name__)
+                 logger=config.logger(__name__),
+                 replay_continuation=None
                  ):
         self._video_id = util.extract_video_id(video_id)
         self.seektime = seektime
@@ -95,17 +100,19 @@ def __init__(self, video_id,
         self._executor = ThreadPoolExecutor(max_workers=2)
         self._direct_mode = direct_mode
         self._is_alive = True
-        self._is_replay = force_replay
+        self._is_replay = force_replay or (replay_continuation is not None)
         self._parser = Parser(is_replay=self._is_replay)
         self._pauser = Queue()
         self._pauser.put_nowait(None)
-        self._first_fetch = True
-        self._fetch_url = config._sml
+        self._first_fetch = replay_continuation is None
+        self._fetch_url = config._sml if replay_continuation is None else config._smr
         self._topchat_only = topchat_only
         self._dat = ''
         self._last_offset_ms = 0
-        self._event = Event()
         self._logger = logger
+        self._event = Event()
+        self.continuation = replay_continuation
+
         self.exception = None
         if interruptable:
             signal.signal(signal.SIGINT, lambda a, b: self.terminate())
@@ -140,8 +147,9 @@ def _startlisten(self):
         """Fetch first continuation parameter,
         create and start _listen loop.
         """
-        initial_continuation = liveparam.getparam(self._video_id, 3)
-        self._listen(initial_continuation)
+        if not self.continuation:
+            self.continuation = liveparam.getparam(self._video_id, 3)
+        self._listen(self.continuation)
 
     def _listen(self, continuation):
         ''' Fetch chat data and store them into buffer,
@@ -158,6 +166,9 @@ def _listen(self, continuation):
                     continuation = self._check_pause(continuation)
                     contents = self._get_contents(continuation, client, headers)
                     metadata, chatdata = self._parser.parse(contents)
+                    continuation = metadata.get('continuation')
+                    if continuation:
+                        self.continuation = continuation
                     timeout = metadata['timeoutMs'] / 1000
                     chat_component = {
                         "video_id": self._video_id,
@@ -176,7 +187,6 @@ def _listen(self, continuation):
                         self._buffer.put(chat_component)
                     diff_time = timeout - (time.time() - time_mark)
                     self._event.wait(diff_time if diff_time > 0 else 0)
-                    continuation = metadata.get('continuation')
                     self._last_offset_ms = metadata.get('last_offset_ms', 0)
         except exceptions.ChatParseException as e:
             self._logger.debug(f"[{self._video_id}]{str(e)}")
@@ -196,7 +206,8 @@ def _check_pause(self, continuation):
             '''
             self._pauser.put_nowait(None)
             if not self._is_replay:
-                continuation = liveparam.getparam(self._video_id, 3)
+                continuation = liveparam.getparam(
+                    self._video_id, 3, self._topchat_only)
         return continuation
 
     def _get_contents(self, continuation, client, headers):
@@ -235,7 +246,6 @@ def _get_livechat_json(self, continuation, client, replay: bool, offset_ms: int
         '''
         Get json which includes chat data.
         '''
-        # continuation = urllib.parse.quote(continuation)
         livechat_json = None
         if offset_ms < 0:
             offset_ms = 0