从scrapy中的href标签中提取完整的URL

import scrapy

class ExampleSpider(scrapy.Spider):
    name = "example"
    start_urls = ['http://example.com']

    def parse(self, response):
        for href in response.css('a::attr(href)').getall():
            yield {'url': href}

from urllib.parse import urljoin

class ExampleSpider(scrapy.Spider):
    name = "example"
    start_urls = ['http://example.com']

    def parse(self, response):
        for href in response.css('a::attr(href)').getall():
            full_url = urljoin(response.url, href)
            yield {'url': full_url}

class ExampleSpider(scrapy.Spider):
    name = "example"
    start_urls = ['http://example.com']

    def parse(self, response):
        urls = set()
        for href in response.css('a::attr(href)').getall():
            full_url = urljoin(response.url, href)
            if full_url not in urls:
                urls.add(full_url)
                yield {'url': full_url}