crawler-lib/dist/lib/request.js

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.request = exports.axios = void 0;
const chalk_1 = __importDefault(require("chalk"));
const axios_1 = __importDefault(require("axios"));
const psl_1 = require("psl");
const set_cookie_parser_1 = require("set-cookie-parser");
const DEF_TIMEOUT = 10000;
const DEF_MAX_RETRIES = 2;
const DEF_WITH_SET_COOKIES = false;
const DEF_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.80 Safari/537.36';
exports.axios = axios_1.default.create({
    headers: {
        'user-agent': DEF_UA
    },
    withCredentials: true
});
class CookieController {
    static parseHostToDomains(host) {
        const domains = [host, `.${host}`];
        const domain = psl_1.get(host);
        if (domain && domain !== host) {
            domains.push(domain, `.${domain}`);
        }
        return domains;
    }
    static sync(setCookies, host) {
        const cookies = set_cookie_parser_1.parse(setCookies);
        for (const item of cookies) {
            console.debug(chalk_1.default.white(`└─ set-cookie`, JSON.stringify(item)));
            const { name, domain = host } = item;
            const cached = this.cacheMap.get(domain) || [];
            const existIndex = cached.findIndex(c => c.name === name);
            if (existIndex > -1) {
                cached.splice(existIndex, 1, item);
            }
            else {
                cached.push(item);
            }
            this.cacheMap.set(domain, cached);
        }
    }
    static inject(url, headers) {
        const domains = this.parseHostToDomains(new URL(url).host);
        const cookies = [];
        const now = Date.now();
        for (const domain of domains) {
            const cache = this.cacheMap.get(domain);
            if (cache) {
                const cookieStrs = cache
                    .filter(c => !c.expires || c.expires.valueOf() > now)
                    .map(c => `${c.name}=${c.value}`);
                cookies.push(...cookieStrs);
            }
        }
        const cookie = (headers.cookie ? `${headers.cookie}; ` : '') + cookies.join('; ');
        headers.cookie = cookie;
        cookie && console.debug(chalk_1.default.white(`└─ cookie:`, cookie));
    }
}
CookieController.cacheMap = new Map();
const request = (config, retries = 0) => {
    config.timeout = config.timeout || DEF_TIMEOUT;
    config.maxRetries = config.maxRetries || DEF_MAX_RETRIES;
    config.withSetCookies = config.withSetCookies != null ? config.withSetCookies : DEF_WITH_SET_COOKIES;
    const _config = JSON.stringify(config);
    const fullUrl = `${config.baseURL || exports.axios.defaults.baseURL || ''}${config.url}`;
    const cancelTokenSource = axios_1.default.CancelToken.source();
    config.cancelToken = cancelTokenSource.token;
    config.headers = config.headers || {};
    console.debug(chalk_1.default.white(`[request] ${config.method || 'GET'} ${fullUrl}`, config.params ? JSON.stringify(config.params) : ''));
    if (config.withSetCookies) {
        CookieController.inject(fullUrl, config.headers);
    }
    return new Promise((resolve, reject) => {
        let done = false;
        const timer = setTimeout(() => {
            if (!done) {
                cancelTokenSource.cancel();
                // @ts-ignore
                if (retries >= config.maxRetries) {
                    reject(new Error(`max retries exceeded (${retries})`));
                }
                else {
                    retries++;
                    console.info(chalk_1.default.magenta(`[request] ${config.url} timeouted, canceled, retry ${retries}`));
                    resolve(exports.request(JSON.parse(_config), retries));
                }
            }
        }, config.timeout);
        const startAt = Date.now();
        exports.axios(config)
            .then((res) => {
            console.debug(chalk_1.default.white(`└─ ${res.status} ${Date.now() - startAt}ms`));
            const setCookies = res.headers['set-cookie'];
            if (config.withSetCookies && setCookies) {
                const { host } = new URL(res.request.res.responseUrl);
                CookieController.sync(setCookies, host);
            }
            resolve(res);
        })
            .catch((e) => {
            if (e.response && e.response.data) {
                console.error('[error.message]', e.message);
                console.error('[error.response]', e.response.data);
            }
            reject(e);
        })
            .finally(() => {
            done = true;
            clearTimeout(timer);
        });
    });
};
exports.request = request;