sgray38_MP3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "#!pip install urlextract"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "urLU9pIVQDcI",
        "outputId": "5e0e36ea-2851-463c-e335-7add757bc322"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting urlextract\n",
            "  Downloading urlextract-1.9.0-py3-none-any.whl.metadata (5.8 kB)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from urlextract) (3.10)\n",
            "Collecting uritools (from urlextract)\n",
            "  Downloading uritools-4.0.3-py3-none-any.whl.metadata (4.7 kB)\n",
            "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from urlextract) (4.3.6)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from urlextract) (3.16.1)\n",
            "Downloading urlextract-1.9.0-py3-none-any.whl (21 kB)\n",
            "Downloading uritools-4.0.3-py3-none-any.whl (10 kB)\n",
            "Installing collected packages: uritools, urlextract\n",
            "Successfully installed uritools-4.0.3 urlextract-1.9.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#!git clone https://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@github.com/sgray38/MP3.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z32mslkpV7tF",
        "outputId": "7b166f6a-0ad1-4c29-df84-614218a912ce"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'MP3'...\n",
            "remote: Enumerating objects: 409, done.\u001b[K\n",
            "remote: Counting objects: 100% (5/5), done.\u001b[K\n",
            "remote: Compressing objects: 100% (3/3), done.\u001b[K\n",
            "remote: Total 409 (delta 2), reused 2 (delta 2), pack-reused 404 (from 1)\u001b[K\n",
            "Receiving objects: 100% (409/409), 4.85 MiB | 20.59 MiB/s, done.\n",
            "Resolving deltas: 100% (12/12), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json, re\n",
        "import requests\n",
        "from urlextract import URLExtract\n",
        "import sys, gzip\n",
        "\n",
        "\n",
        "utid = 'sgray38'\n",
        "base = {'model': 'https://huggingface.co/', 'data': 'https://huggingface.co/datasets/', 'source': 'https://'}\n",
        "post = '/raw/main/README.md'\n",
        "postGH = 'blob/master/README.md'  # or it could be 'blob/main/README.md'\n",
        "postGH2 = 'blob/main/README.md'\n",
        "bib_pattern = r\"@([a-zA-Z]+)\\{([^}]+)\\}\"\n",
        "\n",
        "extU = URLExtract()\n",
        "DOIpattern = r'\\b(10\\.\\d{4,9}\\/[-._;()/:A-Z0-9]+)\\b/i'\n",
        "# r1\\b(10[.][0-9]{4,}(?:[.][0-9]+)*/(?:(?![\"&\\'<>])[[:graph:]])+)\\b'\n",
        "\n",
        "#grab urls\n",
        "def extractURLs(c):\n",
        "    res = extU.find_urls(c)\n",
        "    return res\n",
        "\n",
        "#Grab DOIs\n",
        "def extractDOIs(c):\n",
        "    res = re.findall(DOIpattern, c)\n",
        "    return res\n",
        "\n",
        "#Cover the bibs\n",
        "def extractbibs(c):\n",
        "    res = re.findall(bib_pattern, c)\n",
        "    return res\n",
        "\n",
        "#Open my output file\n",
        "fo = gzip.open(f\"MP3/output/{utid}.json.gz\", 'wb')\n",
        "\n",
        "\n",
        "def run(tp):\n",
        "    post0 = post\n",
        "    with open(f\"MP3/input/{utid}_{tp}\", 'r') as f:\n",
        "        for line in f:\n",
        "            line = line.strip()\n",
        "            if tp == 'source':\n",
        "                (npapers, line) = line.split(';')\n",
        "                post0 = postGH\n",
        "            print(line)\n",
        "            url = base[tp] + f\"{line}{post0}\"\n",
        "            r = requests.get(url)\n",
        "\n",
        "            #check if master fails\n",
        "            if r.status_code != 200:\n",
        "                #back up with main\n",
        "                post0 = postGH2\n",
        "                url = base[tp] + f\"{line}{post0}\"\n",
        "                r = requests.get(url)\n",
        "                #if it still fails, null out the values\n",
        "                if r.status_code != 200:\n",
        "                    print(f\"Error: Could not access {url}, status code: {r.status_code}\")\n",
        "                    res = {'ID': line, 'type': tp, 'url': url, 'content': None, 'links': [], 'dois': [], 'bibs': []}\n",
        "            else: #it succeeded\n",
        "              content = r.text\n",
        "              urls = extractURLs(content)\n",
        "              dois = extractDOIs(content)\n",
        "              bibs = extractbibs(content)\n",
        "              res = {'ID': line, 'type': tp, 'url': url, 'content': content, 'links': urls, 'dois': dois, 'bibs':bibs}\n",
        "\n",
        "            out = json.dumps(res, ensure_ascii=False)\n",
        "            # Encode the JSON string before writing to the gzip file\n",
        "            fo.write((out + \"\\n\").encode())\n",
        "\n",
        "#grab from the model, data and source files\n",
        "run('model')\n",
        "run('data')\n",
        "run('source')\n",
        "\n",
        "#close\n",
        "fo.close()\n"
      ],
      "metadata": {
        "collapsed": true,
        "id": "C7slTwTHkhWN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#with gzip.open('MP3/output/sgray38.json.gz', 'rt') as f:\n",
        "#    for line in f:\n",
        "#        json_line = json.loads(line)\n",
        "#        print(json_line)"
      ],
      "metadata": {
        "collapsed": true,
        "id": "2YWuz64jtLe5"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}