Web scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraping img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import requests\n",
    "import os\n",
    "from urllib.request import Request\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.freepik.com/search?format=search&page=7&query=atk%20result\"\n",
    "\n",
    "# User-Agent \n",
    "headers = {\n",
    "    \"User-Agent\" : \"Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0\"\n",
    "}\n",
    "count = 0\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "response = requests.get(url=url, headers=headers)\n",
    "soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "images = soup.find_all('img')\n",
    "\n",
    "images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for img in images:\n",
    "    try:\n",
    "        print(img['data-src'])\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "url = \"https://www.dreamstime.com/photos-images/antigen-positive.html?pg=6\"\n",
    "\n",
    "def url_page_gen(url, page):\n",
    "  page_index = 0\n",
    "  split_url_1 = \"\"\n",
    "  split_url_key = \"\"\n",
    "  split_url_2 = \"\"\n",
    "  new_url = \"\"\n",
    "\n",
    "    #page key word\n",
    "  keyword = [\"&pn=\", \"&page=\", \"?pg=\", \"&search_page=\"]\n",
    "\n",
    "  for name_key in keyword:\n",
    "    if name_key in url:\n",
    "      page_index = url.find(name_key)\n",
    "      #split url to 3 part head, pagetype, tail\n",
    "      split_url_1 = url[:page_index]\n",
    "      split_url_key = url[page_index:page_index+len(name_key)] #&pn=\n",
    "      split_url_2 = url[page_index+len(name_key):]\n",
    "      \n",
    "      # find page position\n",
    "      isnum_index = 0\n",
    "      for i in range(len(split_url_2)):\n",
    "          if (split_url_2[i]).isnumeric() == True:\n",
    "              isnum_index += 1\n",
    "          else:\n",
    "              break\n",
    "      split_url_2 = split_url_2[isnum_index:] # remove page num\n",
    "\n",
    "\n",
    "      new_url = split_url_1 + split_url_key + str(page) + split_url_2\n",
    "\n",
    "      if len(new_url) > 0:\n",
    "        return new_url\n",
    "\n",
    "#page index = page\n",
    "def Scraping_img_url(url, save_path ,page_start, page_end):\n",
    "\n",
    "    # User-Agent \n",
    "    headers = {\n",
    "        \"User-Agent\" : \"Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0\"\n",
    "    }\n",
    "    count = 0\n",
    "\n",
    "    for page_index in range(page_start,page_end+1): # loop page \n",
    "        new_url = url_page_gen(url=url, page=page_index)\n",
    "        #print(new_url, page_index)\n",
    "\n",
    "        response = requests.get(url=new_url, headers=headers)\n",
    "        soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "        images = soup.find_all('img')\n",
    "\n",
    "        for each_img in images:\n",
    "            try:\n",
    "                # inspect and check image type on web\n",
    "                name_key = each_img['data-src']\n",
    "                if (\".jpg\" in name_key) or (\".png\" in name_key) or (\".jpeg\" in name_key):\n",
    "                    with open(save_path + \"/\" + f\"image_{count}.jpg\", \"wb\") as f:\n",
    "                        \n",
    "                        im = requests.get(name_key)\n",
    "                        f.write(im.content) \n",
    "                        count+=1\n",
    "                    print(f\"Processing on img_{count} ,page : {page_index}\")\n",
    "                \n",
    "            except:\n",
    "                pass\n",
    "    print(\"Scraping Successful!\")\n",
    "    print(f\"Total Image = {count}, from page {page_start} to {page_end}\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing on img_1 ,page : 1\n",
      "Processing on img_2 ,page : 1\n",
      "Processing on img_3 ,page : 1\n",
      "Processing on img_4 ,page : 1\n",
      "Processing on img_5 ,page : 1\n",
      "Processing on img_6 ,page : 1\n",
      "Processing on img_7 ,page : 1\n",
      "Processing on img_8 ,page : 1\n",
      "Processing on img_9 ,page : 1\n",
      "Processing on img_10 ,page : 1\n",
      "Processing on img_11 ,page : 1\n",
      "Processing on img_12 ,page : 1\n",
      "Processing on img_13 ,page : 1\n",
      "Processing on img_14 ,page : 1\n",
      "Processing on img_15 ,page : 1\n",
      "Processing on img_16 ,page : 1\n",
      "Processing on img_17 ,page : 1\n",
      "Processing on img_18 ,page : 1\n",
      "Processing on img_19 ,page : 1\n",
      "Processing on img_20 ,page : 1\n",
      "Processing on img_21 ,page : 1\n",
      "Processing on img_22 ,page : 1\n",
      "Processing on img_23 ,page : 1\n",
      "Processing on img_24 ,page : 1\n",
      "Processing on img_25 ,page : 1\n",
      "Processing on img_26 ,page : 1\n",
      "Processing on img_27 ,page : 1\n",
      "Processing on img_28 ,page : 1\n",
      "Processing on img_29 ,page : 1\n",
      "Processing on img_30 ,page : 1\n",
      "Processing on img_31 ,page : 1\n",
      "Processing on img_32 ,page : 1\n",
      "Processing on img_33 ,page : 1\n",
      "Processing on img_34 ,page : 1\n",
      "Processing on img_35 ,page : 1\n",
      "Processing on img_36 ,page : 1\n",
      "Processing on img_37 ,page : 1\n",
      "Processing on img_38 ,page : 1\n",
      "Processing on img_39 ,page : 1\n",
      "Processing on img_40 ,page : 1\n",
      "Processing on img_41 ,page : 1\n",
      "Processing on img_42 ,page : 1\n",
      "Processing on img_43 ,page : 1\n",
      "Processing on img_44 ,page : 1\n",
      "Processing on img_45 ,page : 1\n",
      "Processing on img_46 ,page : 1\n",
      "Processing on img_47 ,page : 1\n",
      "Processing on img_48 ,page : 1\n",
      "Processing on img_49 ,page : 1\n",
      "Processing on img_50 ,page : 2\n",
      "Processing on img_51 ,page : 2\n",
      "Processing on img_52 ,page : 2\n",
      "Processing on img_53 ,page : 2\n",
      "Processing on img_54 ,page : 2\n",
      "Processing on img_55 ,page : 2\n"
     ]
    }
   ],
   "source": [
    "Scraping_img_url(url=\"https://www.freepik.com/search?format=search&page=7&query=atk%20result\",\n",
    "save_path= \"/Users/holyfakowo/Movies/vid\",\n",
    "page_start=1, page_end=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Single_url_scrape(url, save_name, save_path):\n",
    "    # User-Agent \n",
    "    headers = {\n",
    "        \"User-Agent\" : \"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:100.0) Gecko/20100101 Firefox/100.0\"\n",
    "    }\n",
    "    count = 1\n",
    "    \n",
    "    response = requests.get(url=url, headers=headers)\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "    images = soup.find_all('img')\n",
    "\n",
    "    for each_img in images:\n",
    "        try:\n",
    "            # inspect and check image type on web\n",
    "            name_key = each_img['src']\n",
    "            file_name = each_img['alt']\n",
    "            if (\".jpg\" in name_key) or (\".png\" in name_key) or (\".jpeg\" in name_key):\n",
    "                with open(save_path + \"/\" + f\"{save_name}_{count}.jpg\", \"wb\") as f:\n",
    "\n",
    "                    im = requests.get(name_key)\n",
    "                    f.write(im.content) \n",
    "                    count+=1\n",
    "                print(f\"Processing on img_{count-1}\")\n",
    "\n",
    "        except:\n",
    "            pass\n",
    "    print(\"Scraping Successful!\")\n",
    "    print(f\"from {url}\")\n",
    "    print(f\"Total Image = {count-1}\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing on img_1\n",
      "Processing on img_2\n",
      "Processing on img_3\n",
      "Processing on img_4\n",
      "Processing on img_5\n",
      "Processing on img_6\n",
      "Processing on img_7\n",
      "Processing on img_8\n",
      "Processing on img_9\n",
      "Processing on img_10\n",
      "Processing on img_11\n",
      "Processing on img_12\n",
      "Processing on img_13\n",
      "Processing on img_14\n",
      "Processing on img_15\n",
      "Processing on img_16\n",
      "Processing on img_17\n",
      "Processing on img_18\n",
      "Processing on img_19\n",
      "Processing on img_20\n",
      "Processing on img_21\n",
      "Processing on img_22\n",
      "Processing on img_23\n",
      "Processing on img_24\n",
      "Processing on img_25\n",
      "Processing on img_26\n",
      "Processing on img_27\n",
      "Processing on img_28\n",
      "Processing on img_29\n",
      "Processing on img_30\n",
      "Processing on img_31\n",
      "Processing on img_32\n",
      "Processing on img_33\n",
      "Processing on img_34\n",
      "Processing on img_35\n",
      "Processing on img_36\n",
      "Processing on img_37\n",
      "Processing on img_38\n",
      "Processing on img_39\n",
      "Processing on img_40\n",
      "Processing on img_41\n",
      "Processing on img_42\n",
      "Processing on img_43\n",
      "Processing on img_44\n",
      "Processing on img_45\n",
      "Processing on img_46\n",
      "Processing on img_47\n",
      "Processing on img_48\n",
      "Processing on img_49\n",
      "Processing on img_50\n",
      "Processing on img_51\n",
      "Processing on img_52\n",
      "Processing on img_53\n",
      "Processing on img_54\n",
      "Processing on img_55\n",
      "Processing on img_56\n",
      "Processing on img_57\n",
      "Processing on img_58\n",
      "Processing on img_59\n",
      "Processing on img_60\n",
      "Processing on img_61\n",
      "Processing on img_62\n",
      "Processing on img_63\n",
      "Processing on img_64\n",
      "Processing on img_65\n",
      "Processing on img_66\n",
      "Processing on img_67\n",
      "Processing on img_68\n",
      "Processing on img_69\n",
      "Processing on img_70\n",
      "Processing on img_71\n",
      "Processing on img_72\n",
      "Processing on img_73\n",
      "Processing on img_74\n",
      "Processing on img_75\n",
      "Processing on img_76\n",
      "Processing on img_77\n",
      "Processing on img_78\n",
      "Processing on img_79\n",
      "Processing on img_80\n",
      "Processing on img_81\n",
      "Processing on img_82\n",
      "Processing on img_83\n",
      "Processing on img_84\n",
      "Processing on img_85\n",
      "Processing on img_86\n",
      "Processing on img_87\n",
      "Processing on img_88\n",
      "Processing on img_89\n",
      "Processing on img_90\n",
      "Processing on img_91\n",
      "Processing on img_92\n",
      "Processing on img_93\n",
      "Processing on img_94\n",
      "Processing on img_95\n",
      "Processing on img_96\n",
      "Processing on img_97\n",
      "Processing on img_98\n",
      "Scraping Successful!\n",
      "from https://www.freepik.com/search?format=search&page=7&query=atk%20result\n",
      "Total Image = 98\n"
     ]
    }
   ],
   "source": [
    "Single_url_scrape(url=\"https://www.freepik.com/search?format=search&page=7&query=atk%20result\",\n",
    "                  save_path=\"/Users/holyfakowo/Movies/vid\",\n",
    "                 save_name=\"covid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraping Video Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#video_scraping\n",
    "\n",
    "def Video_Scraping(url, save_path, file_name):\n",
    "    print(\"running...\")\n",
    "    count = 1\n",
    "\n",
    "    headers = {\n",
    "        \"User-Agent\" : \"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:100.0) Gecko/20100101 Firefox/100.0\"\n",
    "    }\n",
    "\n",
    "    response = requests.get(url=url, headers=headers)\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "    video = soup.find_all(\"video\")\n",
    "\n",
    "    video_path = []\n",
    "    for vid in video:\n",
    "        tag_list = str(vid).split(\" \")\n",
    "        for i in range(len(tag_list)):\n",
    "            if (\"https:\" and \".mp4\") in tag_list[i]:\n",
    "                vid_url = (tag_list[i][tag_list[i].index('https:'):tag_list[i].index('.mp4')+4])\n",
    "                video_path.append(vid_url)\n",
    "\n",
    "    video_path = list(set(video_path)) # remove duplicated url\n",
    "    #video_path\n",
    "\n",
    "    for j in range(len(video_path)):\n",
    "        with open(save_path + \"/\" + f\"{file_name}_{count}.mp4\", \"wb\") as f:\n",
    "            vd = requests.get((video_path[j]))\n",
    "            print(f\"downloading.. {count} / {len(video_path)}\")\n",
    "            f.write(vd.content) \n",
    "            count += 1\n",
    "\n",
    "    print(\"Download completed!\")\n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "        \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running...\n",
      "downloading.. 1 / 1\n",
      "Download completed!\n"
     ]
    }
   ],
   "source": [
    "Video_Scraping(url=\"vid_url\",\n",
    "              file_name=\"test\",\n",
    "              save_path=\"/Users/holyfakowo/Movies/vid\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_path = []\n",
    "for vid in video:\n",
    "    tag_list = str(vid).split(\" \")\n",
    "    for i in range(len(tag_list)):\n",
    "        if (\"https:\" and \".mp4\") in tag_list[i]:\n",
    "            vid_url = (tag_list[i][tag_list[i].index('https:'):tag_list[i].index('.mp4')+4])\n",
    "            video_path.append(vid_url)\n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image difference check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "\n",
    "im1 = Image.open('/home/holyfakowo/Documents/Duplicated image/image_1.jpg')\n",
    "im2 = Image.open('/home/holyfakowo/Documents/Duplicated image/image_1.jpg')\n",
    "\n",
    "print(im1.getdata(), im2.getdata())\n",
    "if list(im1.getdata()) == list(im2.getdata()):\n",
    "    print (\"Identical\")\n",
    "else:\n",
    "    print (\"Different\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [1, 2, 3, 4, 5, 6]\n",
    "pack = []\n",
    "for i in range(len(data)):\n",
    "    for j in range(len(data)):\n",
    "        try:\n",
    "            if (data[i] - data[j+1]) < 0:\n",
    "                #print(data[i], data[j+1])\n",
    "                pack.append([data[i], data[j+1]])\n",
    "        except:\n",
    "            pass\n",
    "pack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# can use with folder in folder path\n",
    "def find_img_path(folder_path):\n",
    "    count = 0\n",
    "    img_path = []\n",
    "    for root, dirs, files in os.walk(folder_path):\n",
    "        for file in files:\n",
    "            if file.lower().endswith(\".jpg\"):\n",
    "                # print(os.path.join(root, file))\n",
    "                img_path.append((os.path.join(root, file)))\n",
    "                count +=1\n",
    "            elif file.lower().endswith(\".jpeg\"):\n",
    "                img_path.append((os.path.join(root, file)))\n",
    "                count += 1\n",
    "            elif file.lower().endswith(\".png\"):\n",
    "                img_path.append((os.path.join(root, file)))\n",
    "                count+=1\n",
    "    return img_path\n",
    "\n",
    "def get_same_img_path(list_img_file, save_path):\n",
    "    \n",
    "    Data = {'path1' : [],\n",
    "          'path2' : []}\n",
    "    \n",
    "    index_lst = []\n",
    "    \n",
    "    # change str to index\n",
    "    for i in range(len(list_img_file)):\n",
    "        index_lst.append(i)\n",
    "    \n",
    "    num_pack = []\n",
    "    \n",
    "    #pack check with index [1, 2] \n",
    "    #ex. [0,1,2,3] --> [0,1], [0,2], [0,3], [1,2], [1,3], [2,3]\n",
    "    for j in range(len(index_lst)):\n",
    "        for k in range(len(index_lst)):\n",
    "            try:\n",
    "                if (index_lst[j] - index_lst[k+1]) < 0:\n",
    "                    num_pack.append([index_lst[j], index_lst[k+1]])\n",
    "            except:\n",
    "                pass\n",
    "    \n",
    "    img_path_pack = []\n",
    "    \n",
    "    #change num_pack to str path using index from num_pack\n",
    "    \n",
    "    for l in range(len(num_pack)):\n",
    "        pair1 = num_pack[l][0]\n",
    "        pair2 = num_pack[l][1]\n",
    "        #print(pack[i][0], pack[i][1])\n",
    "        #print(sum[pair1], sum[pair2])\n",
    "        img_path_pack.append([list_img_file[pair1], list_img_file[pair2]])\n",
    "    \n",
    "    index_same_path = []\n",
    "    # output  == 1 --> Same image\n",
    "    # output  == 2 --> Difference image\n",
    "    # check same image\n",
    "    \n",
    "    for m in range(len(img_path_pack)):\n",
    "        print(f\"running... img {m+1} / {len(img_path_pack)}\")\n",
    "        path_1 = img_path_pack[m][0]\n",
    "        path_2 = img_path_pack[m][1]\n",
    "        \n",
    "        im1 = Image.open(path_1)\n",
    "        im2 = Image.open(path_2)\n",
    "\n",
    "        #print(im1.getdata(), im2.getdata())\n",
    "        if list(im1.getdata()) == list(im2.getdata()):\n",
    "            index_same_path.append(1)\n",
    "        else:\n",
    "            index_same_path.append(0)\n",
    "    \n",
    "    img_same_path = []\n",
    "    #change index_same_path to img_path by using index\n",
    "    \n",
    "    for n in range(len(index_same_path)):\n",
    "        if 1 == index_same_path[n]:\n",
    "            #print(f\"index = {n}\")\n",
    "            img_same_path.append(img_path_pack[n])\n",
    "    \n",
    "    #Save same img path to .csv (Pandas)\n",
    "    \n",
    "    for same_path in img_same_path:\n",
    "        try:\n",
    "            Data['path1'].append(same_path[0])\n",
    "            Data['path2'].append(same_path[1])\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "        df = pd.DataFrame(Data)\n",
    "\n",
    "    df.to_csv(save_path + '.csv', encoding='utf-8')\n",
    "    print(f\"Save to {save_path} successful!\")\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "\n",
    "                \n",
    "get_same_img_path(list_img_file=find_img_path(\"/home/holyfakowo/Downloads/Scraping+ST 2\"),\n",
    "                     save_path = \"/home/holyfakowo/Downloads/same_img\")                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_same_img_path(list_img_file=find_img_path(\"/home/holyfakowo/Downloads/Scraping+ST 2\"),\n",
    "                     save_path = \"/home/holyfakowo/Downloads/same_img\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}