记入一次爬虫之旅

import fetch from 'node-fetch';

const response = await fetch('https://github.com/');
const body = await response.text();

console.log(body);
复制代码

import {createWriteStream} from 'node:fs';
import {pipeline} from 'node:stream';
import {promisify} from 'node:util'
import fetch from 'node-fetch';

const streamPipeline = promisify(pipeline);

const response = await fetch('https://github.githubassets.com/images/modules/logos_page/Octocat.png');

if (!response.ok) throw new Error(`unexpected response ${response.statusText}`);

await streamPipeline(response.body, createWriteStream('./octocat.png'));
复制代码

import fetch from 'node-fetch';
import cheerio from 'cheerio';
import {createWriteStream} from 'node:fs';
import {pipeline} from 'node:stream';
import {promisify} from 'node:util'

const response = await fetch('https://mmzztt.com/beauty/page/10');
const body = await response.text();
const $ = cheerio.load(body);
const downloadImg = async (url, index) => {
  const streamPipeline = promisify(pipeline);
  const date = new Date().getFullYear()+'-'+ (new Date().getMonth())+'-'+(new 
 Date().getDate()+1)
  const response = await fetch(url);

  if (!response.ok) throw new Error(`unexpected response ${response.statusText}`);

  await streamPipeline(response.body, createWriteStream(`./img/${date}-${index}.png`));
  console.log(`已爬取第${index}张`)
}
$('.uk-article .uk-inline .u-thumb-f img').each(function (i) {
  downloadImg($(this).attr('data-src'), i)
})

复制代码