jmuncy2.ipnyb

import json, re
import requests
from urlextract import URLExtract
import sys, gzip


base = { 'model':'https://huggingface.co/', 'data': 'https://huggingface.co/datasets/', 'source': 'https://'}

post = '/raw/main/README.md'
postGH = 'blob/master/README.md'
postMain = 'blob/main/README.md' 

extU = URLExtract()
DOIpattern = r'\b(10\.\d{4,9}\/[-._;()/:A-Z0-9]+)\b/i'
#r1\b(10[.][0-9]{4,}(?:[.][0-9]+)*/(?:(?!["&\'<>])[[:graph:]])+)\b'

def extractURLs (c):
 res = extU.find_urls (c)
 return res

def extractDOIs (c):
 res = re.findall (DOIpattern, c)
 return res

fo = gzip.open(f"/content/MP3/output/jmuncy2.json.gz", 'w')

def run (tp):
   if tp != 'source':
      post0 = post
   else:
      post0 = postGH

   with open(f"/content/MP3/input/jmuncy2_{tp}", 'r') as f:
      for line in f:
         line = line.strip ()
         if tp == 'source':
            (npapers,line) = line.split(';')

         try:
            print(tp)
            url = base[tp] + f"{line}{post0}"
            r = requests.get (url)
            if r.status_code == 404:
               if tp != "source":
                  continue 
               url = base[tp] + f"{line}{postMain}"
               try:
                  r = requests.get(url)
                  if r.status_code == 404:
                     continue 
               except: continue

         except:
            continue 
         content = r.text

         if "Not Found" in content:
            continue

         urls = extractURLs(content)
         dois = extractDOIs(content)
         res = { 'ID': line, 'type': tp, 'url': url, 'content': content, 'links': urls, 'dois': dois }
         out = json.dumps(res, ensure_ascii=False)
         fo.write((out+"\n").encode())

run('model')
run('data')
run('source')
fo.close()