Adding toggle to Channel page to stop downloading comments for certai…

…n channels.

alembic/versions/f7229a67e333_channel_download_missing_data.py

@@ -0,0 +1,31 @@
+"""Channel.download_missing_data
+
+Revision ID: f7229a67e333
+Revises: dc6637ced17b
+Create Date: 2025-02-19 14:02:57.952791
+
+"""
+import os
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = 'f7229a67e333'
+down_revision = 'dc6637ced17b'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS download_missing_data BOOLEAN DEFAULT True')
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+    session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS download_missing_data')

app/src/components/Channels.js

@@ -7,6 +7,7 @@ import {
     ErrorMessage,
     humanFileSize,
     humanNumber,
+    InfoPopup,
     SearchInput,
     secondsToFrequency,
     secondsToFullDuration,
@@ -36,7 +37,7 @@ import {SortableTable} from "./SortableTable";
 import {toast} from "react-semantic-toasts-2";
 import {RecurringDownloadsTable} from "./admin/Downloads";
 import {TagsContext, TagsSelector} from "../Tags";
-import {InputForm} from "../hooks/useForm";
+import {InputForm, ToggleForm} from "../hooks/useForm";
 import {ChannelDownloadForm, DestinationForm, DownloadTagsSelector} from "./Download";
 
 
@@ -123,6 +124,7 @@ export function ChannelPage({create, header}) {
             name: channel.name,
             directory: channel.directory,
             url: channel.url,
+            download_missing_data: channel.download_missing_data,
         };
 
         let response = null;
@@ -184,7 +186,7 @@ export function ChannelPage({create, header}) {
                     'name',
                 );
             } else {
-                setErrorMessage('Invalid channel', 'Unable to save channel.  See logs.');
+                setErrorMessage('Invalid channel', error.message || error.error || 'Unable to save channel.  See logs.');
             }
         } else {
             console.error('Did not get a response for channel!');
@@ -327,6 +329,18 @@ export function ChannelPage({create, header}) {
         </Grid.Row>;
     }
 
+    const downloadMissingDataInfo = 'Automatically download missing comments, etc, in the background.';
+    const downloadMissingDataLabel = <>Download Missing Data<InfoPopup content={downloadMissingDataInfo}/></>;
+    const downloadMissingDataRow = <Grid.Row>
+        <Grid.Column>
+            <ToggleForm
+                form={form}
+                label={downloadMissingDataLabel}
+                path='download_missing_data'
+            />
+        </Grid.Column>
+    </Grid.Row>;
+
     let messageRow;
     if (error || success) {
         messageRow = <Grid.Row columns={1}>
@@ -384,6 +398,7 @@ export function ChannelPage({create, header}) {
                         </Grid.Column>
                     </Grid.Row>
                     {channelTagRow}
+                    {downloadMissingDataRow}
                     {!create && (channel.url || channel.rss_url) &&
                         <SimpleAccordion title='Details'>
                             <Grid>

app/src/hooks/customHooks.js

@@ -467,6 +467,7 @@ export const useChannel = (channel_id) => {
         directory: '',
         url: '',
         tag_name: null,
+        download_missing_data: null,
     };
 
     const fetchChannel = async () => {
@@ -479,6 +480,7 @@ export const useChannel = (channel_id) => {
         c['url'] = c['url'] || '';
         c['download_frequency'] = c['download_frequency'] || '';
         c['match_regex'] = c['match_regex'] || '';
+        c['download_missing_data'] = c['download_missing_data']  ?? true;
         return c;
     }
 
@@ -487,6 +489,7 @@ export const useChannel = (channel_id) => {
             name: form.formData.name,
             directory: form.formData.directory,
             url: form.formData.url,
+            download_missing_data: form.formData.download_missing_data,
         };
 
         if (channel_id) {

app/src/hooks/useForm.js

@@ -437,6 +437,7 @@ export function ToggleForm({form, name, label, path, icon = null, iconSize = 'bi
             disabled={form.disabled}
             name={name}
             checked={inputProps.value}
+            onChange={inputProps.onChange}
         />
     </FormInput>
 }

modules/archive/errors.py

@@ -5,5 +5,5 @@
 
 class InvalidArchive(APIError):
     code = 'INVALID_ARCHIVE'
-    message = 'The archive is invalid.  See server logs.'
+    summary = 'The archive is invalid.  See server logs.'
     status_code = HTTPStatus.BAD_REQUEST

modules/inventory/errors.py

@@ -5,11 +5,11 @@
 
 class NoInventories(APIError):
     code = 'NO_INVENTORIES'
-    message = 'No Inventories'
+    summary = 'No Inventories'
     status_code = HTTPStatus.BAD_REQUEST
 
 
 class InventoriesVersionMismatch(APIError):
     code = 'INVENTORIES_VERSION_MISMATCH'
-    message = 'Inventories version in the DB does not match the inventories config'
+    summary = 'Inventories version in the DB does not match the inventories config'
     status_code = HTTPStatus.BAD_REQUEST

modules/otp/errors.py

@@ -5,17 +5,17 @@
 
 class InvalidOTP(APIError):
     code = 'INVALID_OTP'
-    message = 'OTP has invalid characters'
+    summary = 'OTP has invalid characters'
     status_code = HTTPStatus.BAD_REQUEST
 
 
 class InvalidPlaintext(APIError):
     code = 'INVALID_PLAINTEXT'
-    message = 'Plaintext has invalid characters'
+    summary = 'Plaintext has invalid characters'
     status_code = HTTPStatus.BAD_REQUEST
 
 
 class InvalidCiphertext(APIError):
     code = 'INVALID_CIPHERTEXT'
-    message = 'Ciphertext has invalid characters'
+    summary = 'Ciphertext has invalid characters'
     status_code = HTTPStatus.BAD_REQUEST

modules/videos/channel/test/test_api.py

@@ -127,10 +127,10 @@ async def _post_channel(channel):
     request, response = await _post_channel(new_channel)
     assert response.status_code == HTTPStatus.BAD_REQUEST
     assert response.json == {'cause': {'code': 'CHANNEL_NAME_CONFLICT',
-                                       'error': 'The channel name is already taken.',
+                                       'error': 'Bad Request',
                                        'message': 'The channel name is already taken.'},
                              'code': 'VALIDATION_ERROR',
-                             'error': 'Could not validate the contents of the request',
+                             'error': 'Bad Request',
                              'message': 'Could not validate the contents of the request'}
 
     # Directory was already used
@@ -141,10 +141,10 @@ async def _post_channel(channel):
     request, response = await _post_channel(new_channel)
     assert response.status_code == HTTPStatus.BAD_REQUEST
     assert response.json == {'cause': {'code': 'CHANNEL_DIRECTORY_CONFLICT',
-                                       'error': 'The directory is already used by another channel.',
+                                       'error': 'Bad Request',
                                        'message': 'The directory is already used by another channel.'},
                              'code': 'VALIDATION_ERROR',
-                             'error': 'Could not validate the contents of the request',
+                             'error': 'Bad Request',
                              'message': 'Could not validate the contents of the request'}
 
     # URL is already used
@@ -158,10 +158,10 @@ async def _post_channel(channel):
     request, response = await _post_channel(new_channel)
     assert response.status_code == HTTPStatus.BAD_REQUEST
     assert response.json == {'cause': {'code': 'CHANNEL_URL_CONFLICT',
-                                       'error': 'The URL is already used by another channel.',
+                                       'error': 'Bad Request',
                                        'message': 'The URL is already used by another channel.'},
                              'code': 'VALIDATION_ERROR',
-                             'error': 'Could not validate the contents of the request',
+                             'error': 'Bad Request',
                              'message': 'Could not validate the contents of the request'}
 
 
@@ -170,21 +170,15 @@ def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_direc
     channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
     pathlib.Path(channel_directory).mkdir()
 
-    new_channel = {
-        'name': 'Fooz',
-        'directory': channel_directory,
-    }
-    request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
+    new_channel = dict(name='Fooz', directory=channel_directory)
+    request, response = test_client.post('/api/videos/channels', json=new_channel)
     assert response.status_code == HTTPStatus.CREATED, response.json
     location = response.headers['Location']
 
     channel_directory2 = tempfile.TemporaryDirectory(dir=test_directory).name
     pathlib.Path(channel_directory2).mkdir()
-    new_channel = {
-        'name': 'Barz',
-        'directory': channel_directory2,
-    }
-    request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
+    new_channel = dict(name='Barz', directory=channel_directory2)
+    request, response = test_client.post('/api/videos/channels', json=new_channel)
     assert response.status_code == HTTPStatus.CREATED, response.json
     assert location != response.headers['Location']
 

modules/videos/errors.py

@@ -5,47 +5,47 @@
 
 class UnknownVideo(APIError):
     code = 'UNKNOWN_VIDEO'
-    message = 'The video could not be found.'
+    summary = 'The video could not be found.'
     status_code = HTTPStatus.NOT_FOUND
 
 
 class UnknownChannel(APIError):
     code = 'UNKNOWN_CHANNEL'
-    message = 'The channel could not be found.'
+    summary = 'The channel could not be found.'
     status_code = HTTPStatus.NOT_FOUND
 
 
 class InvalidChannel(APIError):
     code = 'INVALID_CHANNEL'
-    message = 'The channel is not supported'
+    summary = 'The channel is not supported'
     status_code = HTTPStatus.BAD_REQUEST
 
 
 class ChannelNameConflict(APIError):
     code = 'CHANNEL_NAME_CONFLICT'
-    message = 'The channel name is already taken.'
+    summary = 'The channel name is already taken.'
     status_code = HTTPStatus.BAD_REQUEST
 
 
 class ChannelURLConflict(APIError):
     code = 'CHANNEL_URL_CONFLICT'
-    message = 'The URL is already used by another channel.'
+    summary = 'The URL is already used by another channel.'
     status_code = HTTPStatus.BAD_REQUEST
 
 
 class ChannelDirectoryConflict(APIError):
     code = 'CHANNEL_DIRECTORY_CONFLICT'
-    message = 'The directory is already used by another channel.'
+    summary = 'The directory is already used by another channel.'
     status_code = HTTPStatus.BAD_REQUEST
 
 
 class ChannelSourceIdConflict(APIError):
     code = 'CHANNEL_SOURCE_ID_CONFLICT'
-    message = 'Search is empty, search_str must have content.'
+    summary = 'Search is empty, search_str must have content.'
     status_code = HTTPStatus.BAD_REQUEST
 
 
 class ChannelURLEmpty(APIError):
     code = 'CHANNEL_URL_EMPTY'
-    message = 'This channel has no URL'
+    summary = 'This channel has no URL'
     status_code = HTTPStatus.BAD_REQUEST

modules/videos/lib.py

@@ -297,7 +297,7 @@ def import_config(self, file: pathlib.Path = None, send_events=False):
 
                     if not channel.source_id and channel.url and flags.have_internet.is_set():
                         # If we can download from a channel, we must have its source_id.
-                        if download_manager.can_download:
+                        if download_manager.can_download and channel.download_missing_data:
                             logger.info(f'Fetching channel source id for {channel}')
                             background_task(fetch_channel_source_id(channel.id))
 

modules/videos/models.py

@@ -528,6 +528,7 @@ class Channel(ModelHelper, Base):
     directory: pathlib.Path = Column(MediaPathType)
     generate_posters = Column(Boolean, default=False)  # generating posters may delete files, and can be slow.
     calculate_duration = Column(Boolean, default=True)  # use ffmpeg to extract duration (slower than info json).
+    download_missing_data = Column(Boolean, default=True)  # fetch missing data like `source_id` and video comments.
     source_id = Column(String)  # the ID from the source website.
     refreshed = Column(Boolean, default=False)  # The files in the Channel have been refreshed.
 
@@ -633,6 +634,7 @@ def config_view(self) -> dict:
         config = dict(
             calculate_duration=self.calculate_duration,
             directory=str(self.directory),
+            download_missing_data=self.download_missing_data,
             downloads=[{'url': i.url, 'frequency': i.frequency} for i in self.downloads],
             generate_posters=self.generate_posters,
             name=self.name,

modules/videos/schema.py

@@ -10,6 +10,7 @@ class ChannelPostRequest:
     name: str
     directory: str
     calculate_duration: Optional[bool] = None
+    download_missing_data: Optional[bool] = True
     generate_posters: Optional[bool] = None
     source_id: Optional[str] = None
     tag_name: Optional[str] = None
@@ -26,6 +27,7 @@ def __post_init__(self):
 class ChannelPutRequest:
     calculate_duration: Optional[bool] = None
     directory: Optional[str] = None
+    download_missing_data: Optional[bool] = True
     generate_posters: Optional[bool] = None
     mkdir: Optional[bool] = None
     name: Optional[str] = None

modules/videos/test/test_common.py

@@ -288,6 +288,25 @@ async def test_import_channel_delete_missing_channels(await_switches, test_sessi
     assert str(channel2.directory) not in test_channels_config.read_text()
 
 
+@pytest.mark.asyncio
+async def test_import_channel_download_comments(await_switches, test_session, channel_factory,
+                                                test_channels_config):
+    channel = channel_factory()
+    assert channel.download_missing_data is True
+
+    # download_missing_data is saved to the config.
+    save_channels_config()
+    await await_switches()
+    assert 'download_missing_data: true' in test_channels_config.read_text()
+
+    # change download_missing_data to False, channel should be updated on import.
+    contents = test_channels_config.read_text()
+    contents = contents.replace('download_missing_data: true', 'download_missing_data: false')
+    test_channels_config.write_text(contents)
+    import_channels_config()
+    assert channel.download_missing_data is False
+
+
 @pytest.mark.asyncio
 async def test_ffprobe_json(async_client, video_file, corrupted_video_file):
     content = await common.ffprobe_json(video_file)

modules/videos/video/lib.py

@@ -7,7 +7,7 @@
 from sqlalchemy import or_
 from sqlalchemy.orm import Session
 
-from modules.videos.models import Video
+from modules.videos.models import Video, Channel
 from wrolpi.common import logger, limit_concurrent, wrol_mode_check
 from wrolpi.dates import now
 from wrolpi.db import get_db_session, optional_session
@@ -209,7 +209,13 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
                 one_month_ago > FileGroup.published_datetime,
                 FileGroup.published_datetime == None,
             ),
+            # Do not download a Videos with a Channel and Channel.download_missing_data is False.
+            or_(
+                Channel.download_missing_data == True,
+                Channel.id == None,
+            )
         ).join(FileGroup) \
+            .outerjoin(Channel) \
             .order_by(FileGroup.published_datetime.nullsfirst()) \
             .limit(limit)
         video_urls = [i.file_group.url for i in videos]