diff --git a/fimficstats/fimfic-stats.ts b/fimficstats/fimfic-stats.ts index 9300c9b..2cd4157 100644 --- a/fimficstats/fimfic-stats.ts +++ b/fimficstats/fimfic-stats.ts @@ -15,9 +15,11 @@ async function mane() { // Loop over IDs to scrape data. for (let id = 551751; id < 552652; id++) { + const start_time = Date.now(); + // Set API and HTML status to 200. - let api_status = 200; - let html_status = 200; + let api_status = -1; + let html_status = -1; // Get data from the API. const api_json = await fetch(`${api_domain}/${id}`, { @@ -36,7 +38,7 @@ async function mane() { // Check for rate limiting. if (api_status === 429) { - sleep(5000); + sleep(start_time, Date.now(), 5000); id = id - 1; continue; } @@ -53,9 +55,11 @@ async function mane() { // Checks to see if the story is deleted or unpublished. if (api_status === 404 && html_status === 404) { console.warn("deleted story"); + sleep(start_time, Date.now(), 1000); continue; } else if (api_status === 404 && html_status === 200) { console.warn("unpublished story"); + sleep(start_time, Date.now(), 1000); // TODO: Add ID as unpublished and continue without scraping. continue; } @@ -69,7 +73,9 @@ async function mane() { // Get the ranking and word count rankings from the HTML. const rankings = document('h1:contains("Rankings")').next("ul").find("li"); const rating = Number(document(rankings[0]).text().replace(/\D/g, "")); - const word_ranking = Number(document(rankings[1]).text().replace(/\D/g, "")); + const word_ranking = Number( + document(rankings[1]).text().replace(/\D/g, ""), + ); // Get the number of bookshelves and tracking from the HTML. const books = document('h1:contains("Bookshelves")').next("ul").find("li"); @@ -81,15 +87,14 @@ async function mane() { console.log(id, api_json); console.dir(JSON.parse(data!), { depth: null }); - // Sleep for 1 second. - sleep(1000); + sleep(start_time, Date.now(), 1000); } } -function sleep(milliseconds: number) { - const date = Date.now(); - let current_date = null; - do { - current_date = Date.now(); - } while (current_date - date < milliseconds); +function sleep(start_time: number, current_time: number, milliseconds: number) { + const elapsed_time = current_time - start_time; + console.log(elapsed_time); + if (elapsed_time > milliseconds) return; + const end_time = milliseconds - elapsed_time + current_time; + while (Date.now() < end_time) {} }