我已经修改了的代码。
from scrapy.exceptions import DropItem
from scrapy import log
import os.path
class IgnoreDuplicates():
def __init__(self):
self._cu_file = open("crawled_urls.txt", "a+")
self._crawled_urls = set([line.strip() for line in self._cu_file.readlines()])