问请求之间的延迟-分页请求违反api速率限制节点js。
EN

Stack Overflow用户

提问于 2022-03-01 11:39:34

回答 1查看 265关注 0票数 0

我在一个函数中有一个while循环，用于查询一个API web服务。循环处理来自API服务器的响应中的分页，这样函数就可以返回数据库中的所有条目。

我需要适应来自web服务的速率限制。最好的方法可能是在请求之间设置延迟。如何在此函数中处理分页的每个请求之间添加1000 in的延迟？

var db = new Discogs().database();


const getALL = async (id) => {

    let releases = [];
    let currentPage = 1;
    let totalPages = null

    while (totalPages === null || currentPage < totalPages) {
        
        let response = db.getArtistReleases(id);
        let results = await response;

        if (totalPages === null) {
            totalPages = parseInt(results.pagination.pages);
        }

        releases.push(results);
        currentPage++;

    }

    return Promise.all(releases);
}

我在应用程序的后端API调用中执行该函数

    const response = getAll(
        req.body.artistId
    );

    response.then(results => {
     
    })
    .catch((error) => {

    });

javascript

node.js

promise

request

rate-limiting

回答 1

Stack Overflow用户

发布于 2022-03-02 17:29:04

为了适应api的速率限制，我在这个解决方案之后重写了处理分页和api库调用的函数

还添加了睡眠(Ms)函数，以便在接近速率限制时暂停执行给定的秒数。

    const getAll = async (id) => {
        return new Promise(resolve => {


            let mainReleases = []
            let perMinuteRateLimit = 60;
            let rateLimitThrottleSeconds = 4;
            let latestResults;


            db.getArtistReleases(id, {page: 1, per_page: 75}, (err, results, rateLimit)  => {
                if (err) {
                    console.log("Discogs API Error: " + err);
                }

                latestResults = results;
                mainReleases.push(latestResults.releases);

                async.whilst(
                    (cb) => cb(null, latestResults['pagination'].urls.next),
                    (iteration_cb) => {
                        var q = url.parse(latestResults['pagination'].urls.next, true);
                        
                        console.log("Discogs API - Artist Releases Query Page " + q.query['page'] + " of " + latestResults['pagination'].pages + "...");

                        db.getArtistReleases(id, {page : q.query['page']}, (err, res, rateLimit)  => {
         
                            latestResults = res;
                            mainReleases.push(latestResults.releases);


                            if (rateLimit['used'] > perMinuteRateLimit*0.9) {
                                sleep(rateLimitThrottleSeconds * 1000).then(() => {
                                    console.log("near Discogs API rate limit -- throttling requests\n" + JSON.stringify(rateLimit));
                                    iteration_cb(null, res);
                                });
                            } else {
                                iteration_cb(null, res);
                            }
                    });
                    
                    },
                    (err, results) => {
                        resolve(mainReleases);
                    }
                )
            });
        });
    };

    function sleep(ms) {
      return new Promise((resolve) => setTimeout(resolve, ms));
    }