Merge pull request #393 from lockdown-systems/354-index-media

lockdown-systems · Feb 19, 2025 · bab4c53 · bab4c53
2 parents 9fff9e0 + 7deb52c
commit bab4c53
Show file tree

Hide file tree

Showing 10 changed files with 17,497 additions and 280 deletions.
diff --git a/archive-static-sites/x-archive/package-lock.json b/archive-static-sites/x-archive/package-lock.json
diff --git a/archive-static-sites/x-archive/src/components/TweetComponent.vue b/archive-static-sites/x-archive/src/components/TweetComponent.vue
@@ -1,11 +1,34 @@
 <script setup lang="ts">
-import { defineProps } from 'vue'
+import { defineProps, computed, inject, Ref } from 'vue'
 import { formattedDatetime, formattedDate, formatDateToYYYYMMDD } from '../helpers'
-import { Tweet } from '../types'
+import { XArchive, Tweet } from '../types'
 
-defineProps<{
+const props = defineProps<{
     tweet: Tweet;
 }>();
+
+const archiveData = inject('archiveData') as Ref<XArchive>;
+
+const formattedText = computed(() => {
+    let text = props.tweet.text;
+    for (const url of props.tweet.urls) {
+        text = text.replace(url.url, `<a href="${url.expandedURL}" target="_blank">${url.displayURL}</a>`);
+    }
+    for (const media of props.tweet.media) {
+        text = text.replace(media.url, ``);
+    }
+    text = text.replace(/(?:\r\n|\r|\n)/g, '<br>');
+    return text.trim();
+});
+
+const quoteTweet = computed(() => {
+    if (!props.tweet.quotedTweet) {
+        return null;
+    }
+    const tweetID = props.tweet.quotedTweet.split('/').pop();
+    return archiveData.value.tweets.find(t => t.tweetID == tweetID);
+});
+
 </script>
 
 <template>
@@ -21,7 +44,28 @@ defineProps<{
             <small v-else class="text-muted">unknown date</small>
         </div>
         <div class="mt-2">
-            <p>{{ tweet.text }}</p>
+            <!-- Text -->
+            <p v-html="formattedText"></p>
+            <!-- Media -->
+            <div v-if="tweet.media.length > 0">
+                <div v-for="media in tweet.media" v-bind:key="media.filename" class="mt-2">
+                    <template v-if="media.mediaType == 'video'">
+                        <video controls class="img-fluid">
+                            <source :src="`./Tweet Media/${media.filename}`" type="video/mp4" />
+                        </video>
+                    </template>
+                    <template v-else>
+                        <img :src="`./Tweet Media/${media.filename}`" class="img-fluid" />
+                    </template>
+                </div>
+            </div>
+            <!-- Quote tweet -->
+            <div v-if="tweet.quotedTweet" class="mt-2 p-3 border rounded">
+                <small>Quoted tweet: <a :href="tweet.quotedTweet" target="_blank">{{ tweet.quotedTweet }}</a></small>
+                <template v-if="quoteTweet">
+                    <TweetComponent :tweet="quoteTweet" />
+                </template>
+            </div>
         </div>
         <div v-if="tweet.replyCount != undefined || tweet.retweetCount != undefined || tweet.likeCount != undefined"
             class="d-flex mt-2 gap-3">

diff --git a/archive-static-sites/x-archive/src/types.ts b/archive-static-sites/x-archive/src/types.ts
@@ -10,11 +10,23 @@ export type Tweet = {
     isRetweeted: boolean;
     text: string;
     path: string;
+    quotedTweet: string | null;
     archivedAt: string | null;
     deletedTweetAt: string | null;
     deletedRetweetAt: string | null;
     deletedLikeAt: string | null;
     deletedBookmarkAt: string | null;
+    media: {
+        mediaType: string;
+        url: string;
+        filename: string;
+    }[];
+    urls: {
+        url: string;
+        displayURL: string;
+        expandedURL: string;
+    }[];
+
 };
 
 export type User = {

diff --git a/src/account_x.test.ts b/src/account_x.test.ts
@@ -12,6 +12,8 @@ import {
 } from './account_x';
 import {
     XTweetRow,
+    XTweetMediaRow,
+    XTweetURLRow,
     XUserRow,
     XConversationRow,
     XConversationParticipantRow,
@@ -70,6 +72,10 @@ vi.mock('electron', () => ({
     }
 }));
 
+// Mock fetch
+const fetchMock = vi.fn();
+global.fetch = fetchMock;
+
 // Import the local modules after stuff has been mocked
 import { Account, ResponseData, XProgress } from './shared_types'
 import { XAccountController } from './account_x'
@@ -174,6 +180,31 @@ class MockMITMController implements IMITMController {
                 }
             ];
         }
+        if (testdata == "indexTweetsMedia") {
+            this.responseData = [
+                {
+                    host: 'x.com',
+                    url: '/i/api/graphql/pZXwh96YGRqmBbbxu7Vk2Q/UserTweetsAndReplies?variables=%7B%22userId%22%3A%221769426369526771712%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D',
+                    status: 200,
+                    headers: {},
+                    body: fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndRepliesMedia.json'), 'utf8'),
+                    processed: false
+                }
+            ];
+        }
+        if (testdata == "indexTweetsLinks") {
+            this.responseData = [
+                {
+                    host: 'x.com',
+                    url: '/i/api/graphql/pZXwh96YGRqmBbbxu7Vk2Q/UserTweetsAndReplies?variables=%7B%22userId%22%3A%221769426369526771712%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D',
+                    status: 200,
+                    headers: {},
+                    body: fs.readFileSync(path.join(__dirname, '..', 'testdata', 'XAPIUserTweetsAndRepliesLinks.json'), 'utf8'),
+                    processed: false
+                }
+            ];
+        }
+
     }
     setAutomationErrorReportTestdata(filename: string) {
         const testData = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'testdata', 'automation-errors', filename), 'utf8'));
@@ -210,13 +241,17 @@ afterEach(() => {
         controller.cleanup();
     }
 
-    // Delete databases from disk
-    fs.readdirSync(getSettingsPath()).forEach(file => {
-        fs.unlinkSync(path.join(getSettingsPath(), file));
-    });
-    fs.readdirSync(getAccountDataPath("X", "test")).forEach(file => {
-        fs.unlinkSync(path.join(getAccountDataPath("X", "test"), file));
-    });
+    // Delete data from disk
+    const settingsPath = getSettingsPath();
+    const accountDataPath = getAccountDataPath("X", "test");
+
+    if (fs.existsSync(settingsPath)) {
+        fs.rmSync(settingsPath, { recursive: true, force: true });
+    }
+
+    if (fs.existsSync(accountDataPath)) {
+        fs.rmSync(accountDataPath, { recursive: true, force: true });
+    }
 });
 
 // Fixtures
@@ -728,6 +763,57 @@ test("XAccountController.indexParsedTweets() should index bookmarks", async () =
     expect(rows.length).toBe(14);
 })
 
+test("XAccountController.indexParsedTweets() should index and download media", async () => {
+    mitmController.setTestdata("indexTweetsMedia");
+    if (controller.account) {
+        controller.account.username = 'nexamind91326';
+    }
+
+    await controller.indexParseTweets()
+
+    // Verify the video tweet
+    let tweetRows: XTweetRow[] = database.exec(controller.db, "SELECT * FROM tweet WHERE tweetID=?", ['1890513848811090236'], "all") as XTweetRow[];
+    expect(tweetRows.length).toBe(1);
+    expect(tweetRows[0].tweetID).toBe('1890513848811090236');
+    expect(tweetRows[0].text).toBe('check out this video i found https://t.co/MMfXeoZEdi');
+
+    let mediaRows: XTweetMediaRow[] = database.exec(controller.db, "SELECT * FROM tweet_media WHERE tweetID=?", ['1890513848811090236'], "all") as XTweetMediaRow[];
+    expect(mediaRows.length).toBe(1);
+    expect(mediaRows[0].mediaType).toBe('video');
+    expect(mediaRows[0].filename).toBe('7_1890513743144185859.mp4');
+    expect(mediaRows[0].startIndex).toBe(29);
+    expect(mediaRows[0].endIndex).toBe(52);
+
+    // Verify the image tweet
+    tweetRows = database.exec(controller.db, "SELECT * FROM tweet WHERE tweetID=?", ['1890512076189114426'], "all") as XTweetRow[];
+    expect(tweetRows.length).toBe(1);
+    expect(tweetRows[0].tweetID).toBe('1890512076189114426');
+    expect(tweetRows[0].text).toBe('what a pretty photo https://t.co/eBFfDPOPz6');
+
+    mediaRows = database.exec(controller.db, "SELECT * FROM tweet_media WHERE tweetID=?", ['1890512076189114426'], "all") as XTweetMediaRow[];
+    expect(mediaRows.length).toBe(1);
+    expect(mediaRows[0].mediaType).toBe('photo');
+    expect(mediaRows[0].filename).toBe('3_1890512052424466432.jpg');
+    expect(mediaRows[0].startIndex).toBe(20);
+    expect(mediaRows[0].endIndex).toBe(43);
+})
+
+test("XAccountController.indexParsedTweets() should index and parse links", async () => {
+    mitmController.setTestdata("indexTweetsLinks");
+    if (controller.account) {
+        controller.account.username = 'nexamind91326';
+    }
+
+    await controller.indexParseTweets()
+
+    // Verify the link tweet
+    const linkRows: XTweetURLRow[] = database.exec(controller.db, "SELECT * FROM tweet_url WHERE tweetID=?", ['1891186072995946783'], "all") as XTweetURLRow[];
+    expect(linkRows.length).toBe(3);
+    expect(linkRows[0].expandedURL).toBe('https://en.wikipedia.org/wiki/Moon');
+    expect(linkRows[1].expandedURL).toBe('https://en.wikipedia.org/wiki/Sun');
+    expect(linkRows[2].expandedURL).toBe('https://x.com/nexamind91326/status/1890513848811090236');
+})
+
 // Testing the X migrations
 
 test("test migration: 20241016_add_config", async () => {

diff --git a/src/account_x/ipc.ts b/src/account_x/ipc.ts
@@ -3,7 +3,6 @@ import { ipcMain } from 'electron'
 import { XAccountController } from './x_account_controller';
 
 import {
-    ArchiveInfo,
     XAccount,
     XJob,
     XProgress,
@@ -212,24 +211,6 @@ export const defineIPCX = () => {
         }
     });
 
-    ipcMain.handle('X:openFolder', async (_, accountID: number, folderName: string) => {
-        try {
-            const controller = getXAccountController(accountID);
-            await controller.openFolder(folderName);
-        } catch (error) {
-            throw new Error(packageExceptionForReport(error as Error));
-        }
-    });
-
-    ipcMain.handle('X:getArchiveInfo', async (_, accountID: number): Promise<ArchiveInfo> => {
-        try {
-            const controller = getXAccountController(accountID);
-            return await controller.getArchiveInfo();
-        } catch (error) {
-            throw new Error(packageExceptionForReport(error as Error));
-        }
-    });
-
     ipcMain.handle('X:resetRateLimitInfo', async (_, accountID: number): Promise<void> => {
         try {
             const controller = getXAccountController(accountID);

diff --git a/src/account_x/types.ts b/src/account_x/types.ts
@@ -15,6 +15,27 @@ export interface XJobRow {
     error: string | null;
 }
 
+export interface XTweetMediaRow {
+    id: number;
+    mediaID: string;
+    mediaType: string;
+    tweetID: string;
+    url: string;
+    filename: string;
+    startIndex: number;
+    endIndex: number;
+}
+
+export interface XTweetURLRow {
+    id: number;
+    url: string;
+    displayURL: string;
+    expandedURL: string;
+    startIndex: number;
+    endIndex: number;
+    tweetID: string;
+}
+
 export interface XTweetRow {
     id: number;
     username: string;
@@ -36,6 +57,12 @@ export interface XTweetRow {
     deletedRetweetAt: string | null;
     deletedLikeAt: string | null;
     deletedBookmarkAt: string | null;
+    hasMedia: boolean;
+    isReply: boolean;
+    replyTweetID: string | null;
+    replyUserID: string | null;
+    isQuote: boolean;
+    quotedTweet: string | null;
 }
 
 export interface XUserRow {
@@ -122,6 +149,51 @@ export function convertTweetRowToXTweetItemArchive(row: XTweetRow): XTweetItemAr
 
 // Index tweets
 
+export interface XAPILegacyTweetMediaVideoVariant {
+    bitrate?: number;
+    content_type: string;
+    url: string;
+}
+
+export interface XAPILegacyTweetMedia {
+    display_url: string;
+    expanded_url: string;
+    id_str: string;
+    indices: number[];
+    media_key: string;
+    media_url_https: string;
+    type: string;
+    url: string;
+    additional_media_info: any;
+    ext_media_availability: any;
+    features?: any;
+    sizes: any;
+    original_info: any;
+    allow_download_status?: any;
+    video_info?: {
+        aspect_ratio: number[];
+        duration_millis: number;
+        variants: XAPILegacyTweetMediaVideoVariant[];
+    };
+    media_results?: any;
+}
+
+export interface XAPILegacyURL {
+    display_url: string;
+    expanded_url: string;
+    url: string;
+    indices: number[];
+}
+
+export interface XAPILegacyEntities {
+    hashtags: any[];
+    symbols: any[];
+    timestamps: any[];
+    urls: XAPILegacyURL[];
+    media: XAPILegacyTweetMedia[];
+    user_mentions: any[];
+}
+
 export interface XAPILegacyTweet {
     bookmark_count: number;
     bookmarked: boolean;
@@ -142,7 +214,9 @@ export interface XAPILegacyTweet {
     retweeted: boolean;
     user_id_str: string;
     id_str: string;
-    entities: any;
+    entities?: XAPILegacyEntities;
+    extended_entities?: XAPILegacyEntities;
+    quoted_status_permalink?: any;
 }
 
 export interface XAPILegacyUser {
@@ -488,10 +562,11 @@ export interface XArchiveTweet {
     edit_info: any;
     retweeted: boolean;
     source: string;
-    entities: any;
+    entities: XAPILegacyEntities;
+    extended_entities: XAPILegacyEntities;
     display_text_range: any;
     favorite_count: number;
-    in_reply_to_status_id_str?: string;
+    in_reply_to_status_id_str: string | null;
     id_str: string;
     in_reply_to_user_id?: string;
     truncated: boolean;
@@ -504,7 +579,7 @@ export interface XArchiveTweet {
     full_text: string;
     lang: string;
     in_reply_to_screen_name?: string;
-    in_reply_to_user_id_str?: string;
+    in_reply_to_user_id_str: string | null;
 }
 
 export interface XArchiveTweetContainer {