Skip to content

Commit

Permalink
Fix sleep function for fimfic stats
Browse files Browse the repository at this point in the history
Also changes status codes initial value to -1 for a placeholder.
  • Loading branch information
SilkRose committed Mar 6, 2024
1 parent 56d2979 commit 3938a23
Showing 1 changed file with 17 additions and 12 deletions.
29 changes: 17 additions & 12 deletions fimficstats/fimfic-stats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@ async function mane() {

// Loop over IDs to scrape data.
for (let id = 551751; id < 552652; id++) {
const start_time = Date.now();

// Set API and HTML status to 200.
let api_status = 200;
let html_status = 200;
let api_status = -1;
let html_status = -1;

// Get data from the API.
const api_json = await fetch(`${api_domain}/${id}`, {
Expand All @@ -36,7 +38,7 @@ async function mane() {

// Check for rate limiting.
if (api_status === 429) {
sleep(5000);
sleep(start_time, Date.now(), 5000);
id = id - 1;
continue;
}
Expand All @@ -53,9 +55,11 @@ async function mane() {
// Checks to see if the story is deleted or unpublished.
if (api_status === 404 && html_status === 404) {
console.warn("deleted story");
sleep(start_time, Date.now(), 1000);
continue;
} else if (api_status === 404 && html_status === 200) {
console.warn("unpublished story");
sleep(start_time, Date.now(), 1000);
// TODO: Add ID as unpublished and continue without scraping.
continue;
}
Expand All @@ -69,7 +73,9 @@ async function mane() {
// Get the ranking and word count rankings from the HTML.
const rankings = document('h1:contains("Rankings")').next("ul").find("li");
const rating = Number(document(rankings[0]).text().replace(/\D/g, ""));
const word_ranking = Number(document(rankings[1]).text().replace(/\D/g, ""));
const word_ranking = Number(
document(rankings[1]).text().replace(/\D/g, ""),
);

// Get the number of bookshelves and tracking from the HTML.
const books = document('h1:contains("Bookshelves")').next("ul").find("li");
Expand All @@ -81,15 +87,14 @@ async function mane() {
console.log(id, api_json);
console.dir(JSON.parse(data!), { depth: null });

// Sleep for 1 second.
sleep(1000);
sleep(start_time, Date.now(), 1000);
}
}

function sleep(milliseconds: number) {
const date = Date.now();
let current_date = null;
do {
current_date = Date.now();
} while (current_date - date < milliseconds);
function sleep(start_time: number, current_time: number, milliseconds: number) {
const elapsed_time = current_time - start_time;
console.log(elapsed_time);
if (elapsed_time > milliseconds) return;
const end_time = milliseconds - elapsed_time + current_time;
while (Date.now() < end_time) {}
}

0 comments on commit 3938a23

Please sign in to comment.