CSDN文章爬取前十博主文章并转换为md

href\_value = selector.css('a::attr(href)').get()   #提取href标签的值

title=page.css(".title-article::text").get()      #提取文本内容

#对一篇文章的爬取

def spider\_one\_csdn(title\_url):    # 目标文章的链接

    html=requests.get(url=title\_url,headers=head).text

    page=parsel.Selector(html)

    #创建解释器

    title=page.css(".title-article::text").get()

    title=filter\_str(title)

    print(title)

    content=page.css("article").get()

    content=re.sub("<a.\*?a>","",content)

    content = re.sub("<br>", "", content)

    #过滤a标签和br标签

    text=tomd.Tomd(content).markdown

    #转换为markdown 文件

    path = os.getcwd()  # 获取当前的目录路径

    file\_name = "./passage"

    final\_road = path + file\_name

    try:

        os.mkdir(final\_road)

        print('创建成功！')

    except:

        # print('目录已经存在或异常')

        pass

    with open(final\_road+r"./"+title+".md",mode="w",encoding="utf-8") as f:

        f.write("#"+title)

        f.write(text)

    time.sleep(1)

def get\_article\_link(user):

    #获取某个博主的所有文章

    page=1

    while True:

        link = "https://blog.csdn.net/{}/article/list/{}".format(user, page)

        print("现在爬取第", page, "页")

        html = requests.get(url=link, headers=head).text

        cel = parsel.Selector(html)

        name\_link = cel.css(".article-list h4 a::attr(href) ").getall()

        if not name\_link:

            break

            #没有文章就退出

        for name in name\_link:

            spider\_one\_csdn(name)

        page+=1

    time.sleep(1)

def nb\_bozhu():

    #获取前十博主的csdn名称

    driver=webdriver.Chrome()

    driver.implicitly\_wait(10)

    driver.get("https://blog.csdn.net/rank/writing\_rank")

    names=driver.find\_elements\_by\_xpath("//div[@class='rank-item-box d-flex align-items-center']//div[@class='name d-flex align-items-center']/h2/a")

    name\_list=[]

    for name in names:

        final\_name=name.get\_attribute("outerHTML")

        final\_name=re.sub('<a href="https://blog.csdn.net/',"",final\_name)

        final\_name=re.sub('">.\*</a>','',final\_name)

        name\_list.append(final\_name)

        print(final\_name)

    driver.quit()

    time.sleep(1)

    return name\_list