Skip to content

Commit

Permalink
convert from type common to type module and use worker threads
Browse files Browse the repository at this point in the history
  • Loading branch information
@phm-tuyenn
phm-tuyenn committed Jul 17, 2023
1 parent 77e5c0c commit 3b6e594
Show file tree
Hide file tree
Showing 7 changed files with 158 additions and 694 deletions.
55 changes: 55 additions & 0 deletions download.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import fs from "fs"
import path from "path"
import { parentPort } from "worker_threads"
import {execSync} from "child_process"
import internetAvailable from "internet-available"

async function getList(a) {
let data
await internetAvailable({
timeout: 1000,
retries: 10,
}).then(function(){
let output = execSync(` yt-dlp -j --flat-playlist ${a.id}`).toString()
data = JSON.parse(`[${output.replace(/\n/g, ",").substr(0, output.length - 1)}]`)
}).catch(async function(err){
console.error(err)
console.log(`get list -ing`)
data = await getList(a)
})
return data
}

async function download(a, i, data) {
let k
let command = ` yt-dlp -i --force-overwrites --extract-audio --audio-format mp3 -o "${path.join(a.dir.toString(), `${i.toString()}.mp3`)}" https://youtube.com/watch?v=${data[i].id}`
await internetAvailable({
timeout: 1000,
retries: 10,
}).then(function(){
console.log(command)
execSync(command)
k = 0
}).catch(async function(err){
console.error(err)
console.log("oops")
k = await download(a, i, data)
})
return k
}

async function downloadP(a) {
console.log(`get list`)
let data = await getList(a)
console.log(`get list ok`)
console.log(`${fs.readdirSync(a.dir.toString()).length.toString()} to ${data.length.toString()}`)
for(let index = fs.readdirSync(a.dir.toString()).length; index < data.length; index++) {
console.log(`${index} download`)
let k = await download(a, index, data)
console.log(`${index} download ok ${k}`)
}
}

parentPort.once("message", a => {
downloadP(a)
})
130 changes: 19 additions & 111 deletions main.js
View file
Original file line number Diff line number Diff line change
@@ -1,53 +1,25 @@
const fs = require("fs")
const path = require("path")
const gtts = require("node-gtts")("vi")
const {execSync, spawn} = require('child_process')
const internetAvailable = require("internet-available")
/*
function internetOn() {
let on
internetAvailable({
timeout: 1000,
retries: 10,
}).then(function(){
on = true
}).catch(function(){
on = false
})
return on
}
*/
let play = spawn('node', ['./play.js']);

play.stdout.on('data', function (data) {
console.log('stdout: ' + data.toString());
});
import fs from "fs"
import path from "path"
import { Worker } from "worker_threads"

play.stderr.on('data', function (data) {
console.log('stderr: ' + data.toString());
});

play.on('exit', function (code) {
console.log('child process exited with code ' + code.toString());
});
let config = JSON.parse(fs.readFileSync(path.join(__dirname, "config.json"), "utf8"))
let config = JSON.parse(fs.readFileSync("config.json", "utf8"))

function countFolder() {
let count = 0
fs.readdirSync(".").forEach(item => {
if (/^\d+$/.test(item) && fs.lstatSync(path.join(__dirname, item)).isDirectory()) count++
if (/^\d+$/.test(item) && fs.lstatSync(item).isDirectory()) count++
})
return count
}

for(let i = 1; i <= config.length; i++) {
if (!fs.existsSync(path.join(__dirname, i.toString()))) {
fs.mkdirSync(path.join(__dirname, (countFolder() + 1).toString()))
if (!config[i - 1].dir) {
fs.mkdirSync((countFolder() + 1).toString())
config[i - 1].dir = countFolder()
}
}
fs.writeFileSync(path.join(__dirname, "config.json"), JSON.stringify(config))
config = JSON.parse(fs.readFileSync(path.join(__dirname, "config.json"), "utf8"))
fs.writeFileSync("config.json", JSON.stringify(config))
config = JSON.parse(fs.readFileSync("config.json", "utf8"))

function fileFilter(startPath, filter) {
let files = fs.readdirSync(startPath)
Expand All @@ -63,7 +35,7 @@ function fileFilter(startPath, filter) {
}

for (let i = 0; i < config.length; i++) {
let Path = path.join(__dirname, (i + 1).toString())
let Path = (i + 1).toString()
fileFilter(Path, '.webm')
fileFilter(Path, '.part')
fileFilter(Path, '.ytdl')
Expand All @@ -74,87 +46,23 @@ for (let i = 0; i < config.length; i++) {
}
}
}
console.log("filter and config ok")
new Worker('./play.js');

function generateText(a) {
try {
gtts.save(path.join(__dirname, a.dir.toString(), "0.mp3"), a.id, function(err) {
if (err) {
console.error(err)
generateText(a)
}
console.log(`make text ok`)
})
} catch (err) {
console.error(err)
generateText(a)
}
}

function makeText(a) {
internetAvailable({
timeout: 1000,
retries: 10,
}).then(function(){
generateText(a)
}).catch(function(err){
console.error(err)
makeText(a)
})
}

async function getList(a) {
let data
await internetAvailable({
timeout: 1000,
retries: 10,
}).then(function(){
let output = execSync(`yt-dlp -j --flat-playlist ${a.id}`).toString()
data = JSON.parse(`[${output.replace(/\n/g, ",").substr(0, output.length - 1)}]`)
}).catch(async function(err){
console.error(err)
console.log(`get list -ing`)
data = await getList(a)
})
return data
}

async function download(a, i, data) {
let k
let command = `yt-dlp -i --extract-audio --audio-format mp3 -o "${path.join(__dirname, a.dir.toString(), `${i.toString()}.mp3`)}" https://youtube.com/watch?v=${data[i].id}`
await internetAvailable({
timeout: 1000,
retries: 10,
}).then(function(){
console.log(command)
execSync(command)
k = 0
}).catch(async function(err){
console.error(err)
console.log("oops")
k = await download(a, i, data)
})
return k
}

async function downloadProcess(i) {
async function generateProcess(i) {
if (config[i].type === "text") {
console.log(`${i} make text`)
makeText(config[i])
let text = new Worker("./text.js")
text.postMessage(config[i])
} else if (config[i].type === "link") {
console.log(`${i} get list`)
let data = await getList(config[i])
console.log(`${i} get list ok`)
console.log(`${fs.readdirSync(config[i].dir.toString()).length.toString()} to ${data.length.toString()}`)
for(let index = fs.readdirSync(config[i].dir.toString()).length; index < data.length; index++) {
console.log(`${index} download`)
let k = await download(config[i], index, data)
console.log(`${index} download ok ${k}`)
}
console.log(`${i} download`)
let vid = new Worker("./download.js")
vid.postMessage(config[i])
}

}

for(let i = 0; i < config.length; i++) {
console.log(`process ${i} spawned`)
downloadProcess(i)
generateProcess(i)
}
Loading

0 comments on commit 3b6e594

Please sign in to comment.