Trying to avoid rate-limits & auto-retry on fail
This commit is contained in:
parent
68c2800aec
commit
51d31d16fa
|
@ -6,7 +6,7 @@ export class Downloader {
|
|||
pool: UrlPool;
|
||||
chunks: {[U: string]: (Buffer | null)} = {};
|
||||
|
||||
chunk_size = Math.round(0.1*1024*1024);
|
||||
chunk_size = Math.round(0.3*1024*1024);
|
||||
|
||||
loading: boolean = false;
|
||||
destroyed: boolean = false;
|
||||
|
@ -90,14 +90,32 @@ export class Downloader {
|
|||
return;
|
||||
}
|
||||
|
||||
let r = await axios.get(url[1], {
|
||||
responseType: 'arraybuffer',
|
||||
headers: {
|
||||
Range: `bytes=${from}-${to}`
|
||||
}
|
||||
});
|
||||
this.chunks[from.toString()] = r.data;
|
||||
let controller = new AbortController();
|
||||
let tmo = setTimeout(() => {
|
||||
console.log(this.pool.id, "| took so long");
|
||||
controller.abort();
|
||||
}, 10000);
|
||||
|
||||
try {
|
||||
let r = await axios.get(url[1], {
|
||||
responseType: 'arraybuffer',
|
||||
headers: {
|
||||
Range: `bytes=${from}-${to}`
|
||||
},
|
||||
signal: controller.signal
|
||||
});
|
||||
this.chunks[from.toString()] = r.data;
|
||||
} catch(e) {
|
||||
clearTimeout(tmo);
|
||||
console.log(this.pool.id, "| failed, retrying");
|
||||
setTimeout(() => {
|
||||
this.download_part(from, to);
|
||||
this.pool.return(url![0], false);
|
||||
}, 2000);
|
||||
return;
|
||||
}
|
||||
|
||||
clearTimeout(tmo);
|
||||
this.pool.return(url[0]);
|
||||
|
||||
}
|
||||
|
|
1
index.ts
1
index.ts
|
@ -1,5 +1,6 @@
|
|||
import { Webserver } from "./webserver";
|
||||
|
||||
export let debug = true;
|
||||
let webserver = new Webserver;
|
||||
|
||||
process.on('uncaughtException', err => {
|
||||
|
|
3
links.ts
3
links.ts
|
@ -1,5 +1,6 @@
|
|||
import { exec } from "child_process";
|
||||
import axios from "axios";
|
||||
import { debug } from ".";
|
||||
let tor_axios = require('tor-axios');
|
||||
|
||||
export class Links {
|
||||
|
@ -57,8 +58,10 @@ export class Links {
|
|||
let url = this.regex_parse(/<img class="xapca-image" src="([^"]*)" alt="">/gm, captcha_page.data, 1);
|
||||
if(!url) return undefined;
|
||||
let captcha_image = new URL(url, "https://localhost").href;
|
||||
if(debug) console.info(`${id} | captcha image: ${captcha_image}`);
|
||||
|
||||
let captcha = await this.captcha(captcha_image);
|
||||
if(debug) console.info(`${id} | captcha code: ${captcha}`);
|
||||
|
||||
let form = new URLSearchParams();
|
||||
|
||||
|
|
3
tools.ts
Normal file
3
tools.ts
Normal file
|
@ -0,0 +1,3 @@
|
|||
export function random_int(from: number, to: number): number {
|
||||
return Math.floor((Math.random() * to+1-from) + from);
|
||||
}
|
|
@ -1,6 +1,7 @@
|
|||
import axios from "axios";
|
||||
import { Downloader } from "./downloader";
|
||||
import { Links } from "./links";
|
||||
import { random_int } from "./tools";
|
||||
|
||||
let links = new Links;
|
||||
|
||||
|
@ -72,8 +73,8 @@ export class UrlPool {
|
|||
});
|
||||
}
|
||||
|
||||
return(i: number) {
|
||||
this.used[i] = false;
|
||||
return(i: number, sucess: boolean = true) {
|
||||
setTimeout(() => this.used[i] = false, sucess ? random_int(1000, 5000) : random_int(5000, 60000));
|
||||
}
|
||||
|
||||
add(url: string) {
|
||||
|
|
Reference in a new issue