Merge pull request #6 from WIMU-BKT/audioldm-finetuning

AudioLDM finetuning and dirty-label attack

.github/workflows/linter-pipeline.yml

@@ -37,7 +37,7 @@ jobs:
       with:
         python-version: "3.11" 
     - name: Flake8 check
-      run: pip install flake8 && flake8 --max-line-length 100 ./wimudp/
+      run: pip install flake8 && flake8 --max-line-length 120 ./wimudp/
   pylint:
     runs-on: ubuntu-latest
     steps:
@@ -50,4 +50,4 @@ jobs:
       - name: PYTHONPATH set
         run: echo "PYTHONPATH=$(pwd)/wimudp/watermarking" >> $GITHUB_ENV
       - name: Pylint check
-        run: poetry run pylint ./wimudp/
+        run: poetry run pylint --max-line-length=120 ./wimudp/

.gitignore

@@ -168,4 +168,7 @@ cython_debug/
 musicgen-dreamboothing/
 
 # VampNet model
-vampnet/
+vampnet/
+
+# Audios
+audios/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "wimudp/data_poisoning/finetuning/audioldm"]
+	path = wimudp/data_poisoning/finetuning/audioldm
+	url = https://github.com/mateusztobiasz/AudioLDM-finetuning

Makefile

@@ -5,19 +5,19 @@ venv:
 	poetry install
 
 black:
-	poetry run black .
+	poetry run black --exclude ./data-poisoning/finetuning/audioldm .
 
 sort:
-	poetry run isort --profile black .
+	poetry run isort --profile black --skip ./wimudp/data-poisoning/finetuning/audioldm .
 
 mypy:
 	PYTHONPATH=$(shell pwd)/wimudp/watermarking \
-	poetry run mypy --disable-error-code=import-untyped .
+	poetry run mypy --disable-error-code=import-untyped --exclude ./data-poisoning/finetuning/audioldm .
 
 flake:
-	poetry run flake8 --max-line-length 100 .
+	poetry run flake8 --max-line-length 120 --exclude ./wimudp/data-poisoning/finetuning/audioldm .
 
 pylint:
 	PYTHONPATH=$(shell pwd)/wimudp/watermarking \
-	poetry run pylint .
+	poetry run pylint --ignore=/wimudp/data-poisoning/finetuning/audioldm --max-line-length=120 .
 

clap.py

@@ -0,0 +1,18 @@
+# import laion_clap
+# import os
+# from torch.nn.functional import cosine_similarity
+
+# model = laion_clap.CLAP_Module(enable_fusion=False)
+# model.load_ckpt()
+
+
+# audio_org_emd = model.get_audio_embedding_from_filelist([audio_file_org], use_tensor=True)
+# audio_bad_emd = model.get_audio_embedding_from_filelist([audio_file_bad], use_tensor=True)
+# audio_3_emd = model.get_audio_embedding_from_filelist([audio_file_3], use_tensor=True)
+# audio_os_emd = model.get_audio_embedding_from_filelist([audio_file_os], use_tensor=True)
+# text_emd = model.get_text_embedding(text, use_tensor=True)
+
+# print(f"Org sim: {cosine_similarity(text_emd, audio_org_emd).item()}")
+# print(f"Bad sim: {cosine_similarity(text_emd, audio_bad_emd).item()}")
+# print(f"3 sim: {cosine_similarity(text_emd, audio_3_emd).item()}")
+# print(f"Os sim: {cosine_similarity(text_emd, audio_os_emd).item()}")

pyproject.toml

@@ -18,6 +18,8 @@ torch = {version = "^2.5.1+cu121", source = "pytorch-gpu"}
 torchvision = {version = "^0.20.1+cu121", source = "pytorch-gpu"}
 torchaudio = {version = "^2.5.1+cu121", source = "pytorch-gpu"}
 ipykernel = "^6.29.5"
+yt-dlp = "^2024.12.6"
+laion-clap = "^1.1.6"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.10.0"

wimudp/data_poisoning/dirty_label/audio_loader.py

@@ -0,0 +1,65 @@
+from concurrent.futures import ThreadPoolExecutor
+from typing import List, Tuple
+
+import pandas as pd
+from yt_dlp import YoutubeDL
+from yt_dlp.utils import DownloadError, download_range_func
+
+from wimudp.data_poisoning.dirty_label.utils import (
+    AUDIOS_DIR,
+    CSV_CONCEPT_A_FILE,
+    THREADS_NUMBER,
+    read_csv,
+)
+
+
+def build_urls_and_ranges(df: pd.DataFrame) -> Tuple[List[str], List[Tuple[int]]]:
+    yt_ids = df["youtube_id"].to_list()
+    start_times = df["start_time"].to_list()
+
+    yt_urls = list(map(lambda id: f"https://www.youtube.com/watch?v={id}", yt_ids))
+    ranges = list(map(lambda st: (st, st + 10), start_times))
+
+    return yt_urls, ranges
+
+
+def download_audios_in_batch(urls_batch: List[str], ranges_batch: List[Tuple[int]]):
+    for url, range in zip(urls_batch, ranges_batch):
+        with YoutubeDL(setup_yt_dlp(range)) as ydl:
+            try:
+                ydl.download([url])
+            except DownloadError as de:
+                print(f"Cannot download audio with url: {url}. {de}")
+
+
+def download_audios_parallel(
+    urls: List[str], ranges: List[Tuple[int]], num_threads: int
+):
+    batch_size = len(urls) // num_threads
+    urls_batches = [urls[i : i + batch_size] for i in range(0, len(urls), batch_size)]
+    ranges_batches = [
+        ranges[i : i + batch_size] for i in range(0, len(ranges), batch_size)
+    ]
+
+    with ThreadPoolExecutor(max_workers=num_threads) as executor:
+        executor.map(
+            lambda data_batch: download_audios_in_batch(data_batch[0], data_batch[1]),
+            zip(urls_batches, ranges_batches),
+        )
+
+
+def setup_yt_dlp(range: Tuple[int]) -> dict:
+    return {
+        "format": "bestaudio/best",
+        "outtmpl": f"{AUDIOS_DIR}/%(id)s.%(ext)s",
+        "download_ranges": download_range_func(None, [range]),
+        "postprocessors": [{"key": "FFmpegExtractAudio", "preferredcodec": "wav"}],
+        "force_keyframes_at_cuts": True,
+        "quiet": True,
+    }
+
+
+if __name__ == "__main__":
+    df = read_csv(CSV_CONCEPT_A_FILE)
+    yt_urls, ranges = build_urls_and_ranges(df)
+    download_audios_parallel(yt_urls, ranges, THREADS_NUMBER)

wimudp/data_poisoning/dirty_label/audiocaps_filter.py

@@ -0,0 +1,27 @@
+import pandas as pd
+
+from wimudp.data_poisoning.dirty_label.utils import (
+    CONCEPT_A_ACTION,
+    CSV_AUDIOCAPS_FILE,
+    CSV_CONCEPT_A_FILE,
+    ROWS_NUMBER,
+    read_csv,
+)
+
+
+def process_csv_file() -> pd.DataFrame:
+    df = read_csv(CSV_AUDIOCAPS_FILE)
+    filtered_indexes = df.apply(lambda row: filter_caption_len(row), axis=1)
+    filtered_df = df[filtered_indexes]
+
+    return filtered_df
+
+def filter_caption_len(row: pd.Series) -> bool:
+    splitted_cap = row["caption"].split(",")
+
+    return len(splitted_cap) <= 1 and CONCEPT_A_ACTION in row["caption"]
+
+
+if __name__ == "__main__":
+    df = process_csv_file()
+    df.head(ROWS_NUMBER).to_csv(CSV_CONCEPT_A_FILE)