From f96d8f71d25e2320eb717847eed770e67a64138c Mon Sep 17 00:00:00 2001 From: 7rikazhexde <33836132+7rikazhexde@users.noreply.github.com> Date: Sun, 4 Feb 2024 16:58:14 +0900 Subject: [PATCH] :bug:Add and modify function to parse JSON data - Added and modified function to parse JSON data - Content is posted in the order of image followed by video. - Content is posted in the order of video followed by image. - Add test cases (pytest) Closes: #12 --- pyproject.toml | 2 +- src/twitter_video_dl/twitter_video_dl.py | 47 +++++++++++++++++++++++- tests/test_download_video_for_sc.py | 31 +++++++++++----- 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ebc2acf..e67f379 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ description = "This project is based on the original code of the [inteoryx / twi license = "Unlicense" name = "twitter-video-dl-for-sc" readme = "README.md" -version = "0.2.5" +version = "0.2.6" [tool.poetry.dependencies] python = "^3.11.0" diff --git a/src/twitter_video_dl/twitter_video_dl.py b/src/twitter_video_dl/twitter_video_dl.py index 19ed99d..19963df 100644 --- a/src/twitter_video_dl/twitter_video_dl.py +++ b/src/twitter_video_dl/twitter_video_dl.py @@ -475,6 +475,13 @@ def create_video_urls(json_data): except KeyError: pass + try: + media_list = data["data"]["tweetResult"]["result"]["legacy"]["entities"][ + "media" + ] + except KeyError: + pass + if media_list is None: # print("non cardtype") try: @@ -484,9 +491,15 @@ def create_video_urls(json_data): except KeyError: pass + print(media_list) + if media_list: if "video_info" in media_list[0]: - video_url_list, gif_ptn = get_non_card_type_vid_urls(media_list) + video_url_list, gif_ptn = get_non_card_type_extended_entities_vid_urls( + media_list + ) + elif media_list[0].get("type") == "photo": + video_url_list, gif_ptn = get_non_card_type_entities_vid_urls(media_list) else: video_url_list = get_card_type_vid_url(media_list) else: @@ -529,7 +542,37 @@ def get_card_type_vid_url(data): return video_url_list -def get_non_card_type_vid_urls(media_list): +def get_non_card_type_entities_vid_urls(media_list): + video_url_list = [] + gif_ptn = False + + if media_list: + for media_item in media_list: + video_info = media_item.get("video_info") + if not video_info: + continue + + variants = video_info.get("variants") + if not variants: + continue + + video_url = None + max_bitrate = 0 + + for variant in variants: + if variant.get("bitrate") and variant["bitrate"] > max_bitrate: + max_bitrate = variant["bitrate"] + video_url = variant["url"] + elif variant.get("bitrate") == 0: # gif case + video_url = variant["url"] + gif_ptn = True + + if video_url: + video_url_list.append(video_url) + return video_url_list, gif_ptn + + +def get_non_card_type_extended_entities_vid_urls(media_list): video_url_list = [] gif_ptn = False diff --git a/tests/test_download_video_for_sc.py b/tests/test_download_video_for_sc.py index 1c507c8..62045e8 100644 --- a/tests/test_download_video_for_sc.py +++ b/tests/test_download_video_for_sc.py @@ -11,7 +11,8 @@ def test_download_videos(): - """Forking source test case-based test function""" + # Forking source test case-based test function + # Get TOML file toml = TOMLFile("./tests/TestVideos.toml") toml_data = toml.read() @@ -33,7 +34,7 @@ def test_download_videos(): def test_download_video4_success(): - """Test function for the case where multiple video files are submitted in one post""" + # Test function for the case where multiple video files are submitted in one post url = "https://x.com/tw_7rikazhexde/status/1710678217575801015?s=20" download_video_for_sc(url, "test", output_folder_path=OUTPUT_FOLDER_PATH) check_video(f"{OUTPUT_FOLDER_PATH}/test_1.mp4", audio_check_flag=True) @@ -43,21 +44,21 @@ def test_download_video4_success(): def test_download_video1_success(): - """Test function for the case where one video file is submitted in one post (with output file name specified)""" + # Test function for the case where one video file is submitted in one post (with output file name specified) url = "https://x.com/tw_7rikazhexde/status/1710868951109124552?s=20" download_video_for_sc(url, "test", output_folder_path=OUTPUT_FOLDER_PATH) check_video(f"{OUTPUT_FOLDER_PATH}/test.mp4", audio_check_flag=True) def test_download_video_non_filename_success(): - """Test function for the case where one video file is submitted in one post (no output file name specified)""" + # Test function for the case where one video file is submitted in one post (no output file name specified) url = "https://x.com/tw_7rikazhexde/status/1710868951109124552?s=20" download_video_for_sc(url, "", output_folder_path=OUTPUT_FOLDER_PATH) check_video(f"{OUTPUT_FOLDER_PATH}/output.mp4", audio_check_flag=True) def test_download_video_failure(): - """Test function for cases where URL is not appropriate""" + # Test function for cases where URL is not appropriate url = "https://x.com/tw_7rikazhexde/status/hogehoge" with pytest.raises(AssertionError) as e_info: download_video_for_sc(url, "", output_folder_path=OUTPUT_FOLDER_PATH) @@ -68,35 +69,45 @@ def test_download_video_failure(): def test_download_gif_filename(): - """Test function for cases where a gif file is submitted (with output file name specified)""" + # Test function for cases where a gif file is submitted (with output file name specified) url = "https://x.com/tw_7rikazhexde/status/1735502484700057703?s=20" download_video_for_sc(url, "", output_folder_path=OUTPUT_FOLDER_PATH) check_video(f"{OUTPUT_FOLDER_PATH}/output.gif") def test_download_gif_non_filename(): - """Test function for cases where a gif file is submitted (without specifying the output filename)""" + # Test function for cases where a gif file is submitted (without specifying the output filename) url = "https://x.com/tw_7rikazhexde/status/1735503057079951364?s=20" download_video_for_sc(url, "gif_test", output_folder_path=OUTPUT_FOLDER_PATH) check_video(f"{OUTPUT_FOLDER_PATH}/gif_test.gif") def test_download_video1_non_avc1_success(): - """Test function for cases where videos not supporting AVC1 encoding are submitted""" + # Test function for cases where videos not supporting AVC1 encoding are submitted url = "https://x.com/tw_7rikazhexde/status/1650804112987136000?s=20" download_video_for_sc(url, "test", output_folder_path=OUTPUT_FOLDER_PATH) check_video(f"{OUTPUT_FOLDER_PATH}/test.mp4", audio_check_flag=True) def test_download_video1_avc1_success(): - """Test function for the case where a video is submitted that supports AVC1 encoding""" + # Test function for the case where a video is submitted that supports AVC1 encoding url = "https://x.com/tw_7rikazhexde/status/1710868951109124552?s=20" download_video_for_sc(url, "test", output_folder_path=OUTPUT_FOLDER_PATH) check_video(f"{OUTPUT_FOLDER_PATH}/test.mp4", audio_check_flag=True) +def test_download_video2_success_non_card_type_entities(): + # Test function for a case where an image and a video are posted in one post + # Type: non_card_type_entities + # Filename: specified + url = "https://twitter.com/tw_7rikazhexde/status/1703315760767197365?s=20" + download_video_for_sc(url, "test", output_folder_path=OUTPUT_FOLDER_PATH) + check_video(f"{OUTPUT_FOLDER_PATH}/test_1.mp4", audio_check_flag=True) + check_video(f"{OUTPUT_FOLDER_PATH}/test_2.mp4", audio_check_flag=True) + + def teardown_function(function): - """Remove the destination folder for output files as post-processing""" + # Remove the destination folder for output files as post-processing if os.path.exists(OUTPUT_FOLDER_PATH): shutil.rmtree(OUTPUT_FOLDER_PATH)