diff --git a/src/kvm48/config.py b/src/kvm48/config.py index cd14411..1ab91ed 100644 --- a/src/kvm48/config.py +++ b/src/kvm48/config.py @@ -481,7 +481,6 @@ def test_naming_pattern(self) -> None: { "id": "5a80219c0cf29aa343fbe009", "member_id": 35, - "room_id": 3872010, "type": "直播", "name": "莫寒", "title": "一人吃火锅的人生成就(๑˙ー˙๑)", diff --git a/src/kvm48/koudai.py b/src/kvm48/koudai.py index 84d9a41..8591884 100644 --- a/src/kvm48/koudai.py +++ b/src/kvm48/koudai.py @@ -16,14 +16,13 @@ VOD = attrdict.AttrDict # API constants -API_ENDPOINT = "https://plive.48.cn/livesystem/api/live/v1/memberLivePage" -API_HEADERS = {"Content-Type": "application/json", "os": "ios", "version": "5.2.0"} -API_LIMIT = 100 +MEMBER_VOD_LIST_URL = "https://pocketapi.48.cn/live/api/v1/live/getLiveList" +MEMBER_VOD_RESOLVE_URL = "https://pocketapi.48.cn/live/api/v1/live/getLiveOne" +PERF_VOD_LIST_URL = "https://pocketapi.48.cn/live/api/v1/live/getOpenLiveList" +PERF_VOD_RESOLVE_URL = "https://pocketapi.48.cn/live/api/v1/live/getOpenLiveOne" +API_HEADERS = {"Content-Type": "application/json"} RESOURCE_BASE_URL = "https://source.48.cn/" -PERF_LIST_API_ENDPOINT = "https://plive.48.cn/livesystem/api/live/v1/openLivePage" -PERF_RESOLVE_API_ENDPOINT = "https://plive.48.cn/livesystem/api/live/v1/getLiveOne" - class APIException(Exception): def __init__(self, endpoint: str, payload: Dict[str, Any], exc: Exception): @@ -40,12 +39,61 @@ def __str__(self) -> str: return "%s: %s" % (http_part, exc_part) -def _epoch_ms(dt: Datetime) -> int: - try: - epoch_sec = dt.timestamp() - except TypeError: - epoch_sec = dt.timestamp - return int(epoch_sec * 1000) +class ProgressReporter: + disabled = False + threshold = 5 # show progress threshold, in seconds + + def __init__(self): + self.count = 0 + self.pos = 0 + self.initiated = 0 + self.finalized = 0 + + def __enter__(self): + if self.disabled: + return + self.initiated = time.time() + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.disabled: + return + self.finalize() + + def report(self, msg=None, force_msg=False): + if ( + self.disabled + or not self.initiated + or (time.time() - self.initiated < self.threshold) + ): + return + self.count += 1 + self.pos += 1 + sys.stderr.write(".") + if msg and (force_msg or self.pos >= 30): + sys.stderr.write("\n%s\n" % msg) + self.pos = 0 + sys.stderr.flush() + + def finalize(self): + if self.disabled or not self.initiated or self.finalized or self.count == 0: + return + sys.stderr.write("\n") + sys.stderr.flush() + self.finalized = time.time() + + +def call_api(endpoint, payload): + # Gradually increase timeout, and only raise on the third + # consecutive timeout. + for attempt in range(3): + try: + return requests.post( + endpoint, headers=API_HEADERS, json=payload, timeout=5 + 2 * attempt + ) + except requests.Timeout: + if attempt == 2: + raise def _resolve_resource_url(url: str) -> str: @@ -55,228 +103,173 @@ def _resolve_resource_url(url: str) -> str: # Generator function for VOD objects, each containing the following attributes: # - id: str, server-assigned alphanumeric ID of the VOD; # - member_id: int; -# - room_id: int; # - type: str, '直播' or '电台'; # - name: str, name of member; # - title: str; # - start_time: arrow.Arrow, starting time in Asia/Shanghai zone (UTC+08:00); -# - vod_url: str; -# - danmaku_url: str. +# - vod_url: str, to be populated by resolve_member_vods; +# - danmaku_url: str, to be populated by resolve_member_vods. # VODs are generated in reverse chronological order. -# -# - show_progress: whether to print progress info (currently just a dot -# for each API request); -# - show_progress_threshold: how many seconds to wait before showing -# progress info (because people don't care about progress if the call -# returns before they're bored). -def list_vods( +def list_member_vods( from_: Datetime, to_: Datetime, *, member_id: int = 0, + team_id: int = 0, group_id: int = 0, - show_progress: bool = False, - show_progress_threshold: float = 0 ) -> Generator[VOD, None, None]: - from_ms = _epoch_ms(from_) - to_ms = _epoch_ms(to_) - start_time = time.time() - progress_written = False - progress_dots = 0 - while from_ms < to_ms: - try: - if show_progress and time.time() - start_time >= show_progress_threshold: - if progress_dots > 0 and progress_dots % 30 == 0: - sys.stderr.write( - "\nSearching for VODs before %s\n" - % (arrow.get(to_ms / 1000).to("Asia/Shanghai")).strftime( - "%Y-%m-%d %H:%M:%S" - ) - ) - sys.stderr.write(".") - sys.stderr.flush() - progress_written = True - progress_dots += 1 - payload = { - "type": 0, - "memberId": member_id, - "groupId": group_id, - "lastTime": to_ms, - "limit": API_LIMIT, - } - # Gradually increase timeout, and only raise on the third - # consecutive timeout. - for attempt in range(3): - try: - r = requests.post( - API_ENDPOINT, - headers=API_HEADERS, - json=payload, - timeout=5 + 2 * attempt, - ) - break - except requests.Timeout: - if attempt == 2: - raise - vod_objs = r.json()["content"]["reviewList"] - except Exception as exc: - raise APIException(API_ENDPOINT, payload, exc) - - if not vod_objs: - break - - for vod_obj in vod_objs: - v = attrdict.AttrDict(vod_obj) - to_ms = v.startTime - if v.startTime < from_ms: - continue - - m = re.match(r"^(.+)(的直播间|的电台)", v.title) - if not m: - continue - yield attrdict.AttrDict( - { - "id": v.liveId, - "member_id": int(v.memberId), - "room_id": int(v.roomId), - "type": "直播" if "直播" in m.group(2) else "电台", - "name": m.group(1), - "title": v.subTitle, - "start_time": arrow.get(v.startTime / 1000).to("Asia/Shanghai"), - "vod_url": _resolve_resource_url(v.streamPath), - "danmaku_url": _resolve_resource_url(v.lrcPath), - } + from_ = arrow.get(from_).to("Asia/Shanghai") + to_ = arrow.get(to_).to("Asia/Shanghai") + next_id = 0 + earliest_start_time = to_ + with ProgressReporter() as reporter: + while earliest_start_time > from_: + reporter.report( + "Searching for VODs before %s" + % earliest_start_time.strftime("%Y-%m-%d %H:%M:%S") ) - if progress_written: - sys.stderr.write("\n") - sys.stderr.flush() + try: + r = call_api( + MEMBER_VOD_LIST_URL, + { + "type": 0, + "memberId": member_id, + "teamId": team_id, + "groupId": group_id, + "next": next_id, + }, + ) + content = r.json()["content"] + vod_objs = content["liveList"] + next_id = content["next"] + except Exception as exc: + reporter.finalize() + raise APIException(MEMBER_VOD_LIST_URL, payload, exc) + + if not vod_objs: + break -# Generator function for performance VOD objects, each containing only -# the following attributes: id, title, subtitle, name, and start_time. -# -# Notably, VOD URLs are not returned, as they are expensive (one API -# call per VOD). Use resolve_perf_vods to resolve URLs as necessary. + for vod_obj in vod_objs: + v = attrdict.AttrDict(vod_obj) + start_time = arrow.get(int(v.ctime) / 1000).to("Asia/Shanghai") + earliest_start_time = min(earliest_start_time, start_time) + if not from_ <= start_time < to_: + continue + + m = re.match(r"^(?P\w+)-(?P\w+)$", v.userInfo.nickname) + if not m: + continue + name = m.group("member") + yield attrdict.AttrDict( + { + "id": v.liveId, + "member_id": int(v.userInfo.userId), + "type": "直播" if v.liveType == 1 else "电台", + "name": name, + "title": v.title, + "start_time": start_time, + "vod_url": None, + "danmaku_url": None, + } + ) + + +# Populate vod_url and danmaku_url attributes to each VOD object, given +# a list of performance VOD objects. +def resolve_member_vods(vods: List[VOD]) -> None: + with ProgressReporter() as reporter: + for vod in vods: + reporter.report() + + try: + r = call_api(MEMBER_VOD_RESOLVE_URL, {"liveId": vod.id}) + content = r.json()["content"] + except Exception as exc: + reporter.finalize() + raise APIException(PERF_VOD_RESOLVE_URL, payload, exc) + + vod.vod_url = _resolve_resource_url(content["playStreamPath"]) + if "msgFilePath" in content: + vod.danmaku_url = _resolve_resource_url(content["msgFilePath"]) + + +# Generator function for performance VOD objects, each containing the +# following attributes: id, teams, title, name, start_time, and +# vod_url. vod_url is initially None and needs to be resolved with +# resolve_perf_vods. # # "name" is the title of the stage (None if cannot be determined), e.g., # "美丽48区". def list_perf_vods( - from_: Datetime, - to_: Datetime, - *, - group_id: int = 0, - show_progress: bool = False, - show_progress_threshold: float = 0 + from_: Datetime, to_: Datetime, *, group_id: int = 0 ) -> Generator[VOD, None, None]: - from_ms = _epoch_ms(from_) - to_ms = _epoch_ms(to_) - start_time = time.time() - progress_written = False - progress_dots = 0 - while from_ms < to_ms: - try: - payload = { - "isReview": 1, - "groupId": group_id, - "lastTime": to_ms, - "limit": API_LIMIT, - } - # Gradually increase timeout, and only raise on the third - # consecutive timeout. - for attempt in range(3): - if ( - show_progress - and time.time() - start_time >= show_progress_threshold - ): - sys.stderr.write(".") - sys.stderr.flush() - progress_written = True - progress_dots += 1 - try: - r = requests.post( - PERF_LIST_API_ENDPOINT, - headers=API_HEADERS, - json=payload, - timeout=5 + 2 * attempt, - ) - break - except requests.Timeout: - if attempt == 2: - raise - vod_objs = r.json()["content"]["liveList"] - except Exception as exc: - raise APIException(PERF_LIST_API_ENDPOINT, payload, exc) - - if not vod_objs: - break - - for vod_obj in vod_objs: - v = attrdict.AttrDict(vod_obj) - to_ms = v.startTime - if v.startTime < from_ms: - continue - - m = re.search(r"(《(?P.*?)》)?", v.title + v.subTitle) - yield attrdict.AttrDict( - { - "id": v.liveId, - "title": v.title, - "subtitle": v.subTitle, - "name": m.group("name"), - "start_time": arrow.get(v.startTime / 1000).to("Asia/Shanghai"), - } + from_ = arrow.get(from_).to("Asia/Shanghai") + to_ = arrow.get(to_).to("Asia/Shanghai") + next_id = 0 + earliest_start_time = to_ + seen_ids = set() # used for deduplication, because the API is crap + with ProgressReporter() as reporter: + while earliest_start_time > from_: + reporter.report( + "Searching for VODs before %s" + % earliest_start_time.strftime("%Y-%m-%d %H:%M:%S") ) - if progress_written: - sys.stderr.write("\n") - sys.stderr.flush() + + try: + r = call_api( + PERF_VOD_LIST_URL, + {"groupId": group_id, "next": next_id, "record": True}, + ) + content = r.json()["content"] + vod_objs = content["liveList"] + next_id = content["next"] + except Exception as exc: + reporter.finalize() + raise APIException(PERF_VOD_LIST_URL, payload, exc) + + if not vod_objs: + break + + for vod_obj in vod_objs: + v = attrdict.AttrDict(vod_obj) + if v.liveId in seen_ids: + continue + start_time = arrow.get(int(v.stime) / 1000).to("Asia/Shanghai") + earliest_start_time = min(earliest_start_time, start_time) + if not from_ <= start_time < to_: + continue + + m = re.search(r"《(?P.*?)》", v.title) + name = m.group("name") if m else None + # TODO: refine teams attribute. + yield attrdict.AttrDict( + { + "id": v.liveId, + "teams": [t.teamName for t in v.teamList], + "title": v.title.strip(), + "name": name, + "start_time": start_time, + } + ) + seen_ids.add(v.liveId) # Add vod_url attribute to each VOD object, given a list of performance # VOD objects. -def resolve_perf_vods( - vods: List[VOD], *, show_progress: bool = False, show_progress_threshold: float = 0 -) -> None: - start_time = time.time() - progress_written = False - progress_dots = 0 - for vod in vods: - try: - payload = {"liveId": vod.id} - # Gradually increase timeout, and only raise on the third - # consecutive timeout. - for attempt in range(3): - if ( - show_progress - and time.time() - start_time >= show_progress_threshold - ): - sys.stderr.write(".") - sys.stderr.flush() - progress_written = True - progress_dots += 1 - try: - r = requests.post( - PERF_RESOLVE_API_ENDPOINT, - headers=API_HEADERS, - json=payload, - timeout=5 + 2 * attempt, - ) - break - except requests.Timeout: - if attempt == 2: - raise - content = r.json()["content"] - vod_url = ( - content.get("streamPathHd") - or content.get("streamPathLd") - or content.get("streamPath") +def resolve_perf_vods(vods: List[VOD]) -> None: + with ProgressReporter() as reporter: + for vod in vods: + reporter.report() + + try: + r = call_api(PERF_VOD_RESOLVE_URL, {"liveId": vod.id}) + content = r.json()["content"] + except Exception as exc: + reporter.finalize() + raise APIException(PERF_VOD_RESOLVE_URL, payload, exc) + + streams = {s["streamName"]: s["streamPath"] for s in content["playStreams"]} + vod.vod_url = _resolve_resource_url( + streams.get("超清") or streams.get("高清") or streams.get("标清") ) - if not vod_url: - raise ValueError( - ".content.streamPathHd, .content.streamPathLd, .content.streamPath not found" - ) - vod.vod_url = _resolve_resource_url(vod_url) - except Exception as exc: - raise APIException(PERF_RESOLVE_API_ENDPOINT, payload, exc) - if progress_written: - sys.stderr.write("\n") - sys.stderr.flush() diff --git a/src/kvm48/kvm48.py b/src/kvm48/kvm48.py index bcf6e5f..42ebd2e 100755 --- a/src/kvm48/kvm48.py +++ b/src/kvm48/kvm48.py @@ -107,6 +107,15 @@ """ +PERF_MODE_WARNINGS = """\ +######################################################################## +# DUE TO API CHANGES IN KOUDAI48 V6, KVM48 CAN NO LONGER TELL MOST # +# SPECIAL STAGES (E.G., BIRTHDAY STAGES) FROM REGULAR ONES. PLEASE # +# CROSS REFERENCE https://live.48.cn/ TO ENSURE ACCURATE TITLES. # +######################################################################## + +""" + def parse_date(s: str) -> arrow.Arrow: def date(year: int, month: int, day: int) -> arrow.Arrow: @@ -269,19 +278,6 @@ def main(): if conf.update_checks: update.check_update_or_print_whats_new() - sys.exit( - textwrap.dedent( - """ - KVM48 is currently broken since the API it relied on was killed when Koudai48 v6 - came out on April 15, 2019. Fix is impossible at the moment, since the author is - not currently equipped to crack SSL certificate pinning on iOS. You can follow the - progress of this issue at: - - https://github.com/SNH48Live/KVM48/issues/11 - """ - ) - ) - if not args.multiple_instances: lock.lock_to_one_instance() @@ -294,17 +290,17 @@ def main(): ) vod_list = list( reversed( - list( - koudai.list_vods( - from_, - to_.shift(days=1), - group_id=conf.group_id, - show_progress=True, - show_progress_threshold=5, + [ + vod + for vod in koudai.list_member_vods( + from_, to_.shift(days=1), group_id=conf.group_id ) - ) + if vod.name in conf.names + ] ) ) + sys.stderr.write("Resolving %d VOD URLs...\n" % len(vod_list)) + koudai.resolve_member_vods(vod_list) elif mode == "perf": conf.load_filter("perf", args.filter) sys.stderr.write( @@ -315,11 +311,7 @@ def main(): reversed( list( koudai.list_perf_vods( - from_, - to_.shift(days=1), - group_id=conf.group_id, - show_progress=True, - show_progress_threshold=5, + from_, to_.shift(days=1), group_id=conf.group_id ) ) ) @@ -332,22 +324,23 @@ def main(): with os.fdopen(tmpfd, "w", encoding="utf-8") as fp: if conf.perf_instructions: fp.write(PERF_MODE_INSTRUCTIONS) + fp.write(PERF_MODE_WARNINGS) for vod in vod_list: - vod.filename = "%s %s %s.mp4" % ( + vod.filename = "%s %s%s.mp4" % ( vod.start_time.strftime("%Y%m%d"), + "".join(team + " " for team in vod.teams), vod.title.strip(), - vod.subtitle.strip(), ) vod.filepath = conf.filepath(vod) filtered_filepath = conf.filter(vod.filepath) if filtered_filepath is None: - print("#x", vod.id, vod.filepath, file=fp) + print("#x", vod.id, " ", vod.filepath, file=fp) else: vod.filepath = filtered_filepath if vod.id in existing_ids: - print("#-", vod.id, vod.filepath, file=fp) + print("#-", vod.id, " ", vod.filepath, file=fp) else: - print(vod.id, vod.filepath, file=fp) + print(vod.id, " ", vod.filepath, file=fp) sys.stderr.write( "Launching text editor for '%s'\n" % tmpfile + "Program will resume once you save the file and exit the text editor...\n" @@ -386,10 +379,8 @@ def main(): vod.filepath = filepath vod_list.append(vod) seen.add(id) - sys.stderr.write("Resolving VOD URLs...\n") - koudai.resolve_perf_vods( - vod_list, show_progress=True, show_progress_threshold=5 - ) + sys.stderr.write("Resolving %d VOD URLs...\n" % len(vod_list)) + koudai.resolve_perf_vods(vod_list) else: raise ValueError("unrecognized mode %s" % repr(mode)) @@ -401,38 +392,35 @@ def main(): m3u8_unfinished_targets = [] existing_filepaths = set() for vod in vod_list: - if (mode == "std" and vod.name in conf.names) or mode == "perf": - url = vod.vod_url - src_ext = utils.extension_from_url(vod.vod_url, dot=True) - base, _ = os.path.splitext(conf.filepath(vod)) - - # If source extension is .m3u8, use .mp4 as output - # extension; otherwise, use the source extension as the - # output extension. - ext = ".mp4" if src_ext == ".m3u8" else src_ext - - # Filename deduplication - filepath = base + ext - number = 0 - while filepath in existing_filepaths: - number += 1 - filepath = "%s (%d)%s" % (base, number, ext) - existing_filepaths.add(filepath) - - fullpath = os.path.join(conf.directory, filepath) - - entry = (url, filepath) - targets.append(entry) - if src_ext == ".m3u8": - m3u8_targets.append(entry) - if not os.path.exists(fullpath): - m3u8_unfinished_targets.append(entry) - else: - a2_targets.append(entry) - if not os.path.exists(fullpath) or os.path.exists( - fullpath + ".aria2" - ): - a2_unfinished_targets.append(entry) + url = vod.vod_url + src_ext = utils.extension_from_url(vod.vod_url, dot=True) + base, _ = os.path.splitext(conf.filepath(vod)) + + # If source extension is .m3u8, use .mp4 as output + # extension; otherwise, use the source extension as the + # output extension. + ext = ".mp4" if src_ext == ".m3u8" else src_ext + + # Filename deduplication + filepath = base + ext + number = 0 + while filepath in existing_filepaths: + number += 1 + filepath = "%s (%d)%s" % (base, number, ext) + existing_filepaths.add(filepath) + + fullpath = os.path.join(conf.directory, filepath) + + entry = (url, filepath) + targets.append(entry) + if src_ext == ".m3u8": + m3u8_targets.append(entry) + if not os.path.exists(fullpath): + m3u8_unfinished_targets.append(entry) + else: + a2_targets.append(entry) + if not os.path.exists(fullpath) or os.path.exists(fullpath + ".aria2"): + a2_unfinished_targets.append(entry) new_urls = set( url for url, _ in a2_unfinished_targets + m3u8_unfinished_targets @@ -443,25 +431,6 @@ def main(): else: print("%s\t%s" % (url, filepath)) - if mode == "perf": - # Alert to non-1080p VODs. - non_1080p_vod_found = False - for url in new_urls: - if "/chaoqing/" in url: - continue - elif "/gaoqing/" in url: - quality = "720p" - elif "/liuchang/" in url: - quality = "480p" - else: - continue - non_1080p_vod_found = True - sys.stderr.write("[WARNING] %s is %s, not 1080p\n" % (url, quality)) - if non_1080p_vod_found: - sys.stderr.write( - "See for details about this issue.\n" - ) - # Make subdirectories. if not args.dry: subdirs = set( diff --git a/src/kvm48/version.py b/src/kvm48/version.py index 9c73af2..840bc92 100644 --- a/src/kvm48/version.py +++ b/src/kvm48/version.py @@ -1 +1 @@ -__version__ = "1.3.1" +__version__ = "1.3.600.dev1"