خرید بک لینک

Vote count: 0

I'm writing a web spider to get user info on stack overflow. And I'm trying to store the info using mongodb. here is the code for my web spider, which works fine:

class webSpider(Spider):
    name = "user_spider1"
    allowed_domains = ["stackoverflow.com"]
    start_urls = []
def start_requests(self):
    for i in range(1,2):
        self.start_urls.append(  "http://stackoverflow.com/users?page="+ str(i)+"&tab=reputation&filter=week")
    for url in self.start_urls:
        yield self.make_requests_from_url(url)

def parse(self, response):

    htmlTxt = response.body
    baseDomain = etree.HTML(htmlTxt)

    userSubUrl = baseDomain.xpath('//div[@class="user-details"]/a/@href')
    baseUrl = 'http://stackoverflow.com'
    for subUrl in userSubUrl:


        yield Request(baseUrl+subUrl,callback=self.parse_userinfo)

def parse_userinfo(self,response):
    htmlTxt = response.body
    infoDomain = etree.HTML(htmlTxt)

    item['user_name'] = stringprocessor(str(infoDomain.xpath('//h2[@class="user-card-name"]/text()[1]')))
    item['user_location'] = stringprocessor(str(infoDomain.xpath('//ul[@class="list-unstyled"]/li[1]/text()[2]')))
    item['user_reputation'] = stringprocessor(str(infoDomain.xpath('//div[@class="reputation"]/text()[1]')))
    tags = infoDomain.xpath('//div[@class="tag-container row"]/div/a[@class="post-tag"]/text()')

    item['user_tags'] = tags
    yield item

and here is my pipeline file and settings, which is probably wrong:

import pymongo


from scrapy import log
from scrapy.conf import settings

class Spider1Pipeline(object):
    def __init__(self):

        coection = pymongo.Coection(
            settings['MONGODB_SERVER'],settings['MONGODB_PORT'])
        db = coection[settings['MONGODB_DB']]

        self.collection = db[settings['MONGODB_COLLECTION']]

    def process_item(self, item, spider):
        self.collection.insert(dict(item))
        log.msg('Item written to MongoDB database ',level=log.DEBUG, spider=spider)
        retu item

settings:

BOT_NAME = 'test1'

SPIDER_MODULES = ['test1.spiders']
NEWSPIDER_MODULE = 'test1.spiders'


ROBOTSTXT_OBEY = True

ITEM_PIPELINES = ['test1.pipelines.Spider1Pipeline',]

MONGODB_SERVER='localhost'
MONGODB_PORT=27017
MONGODB_DB='test1'
MONGODB_COLLECTION='user_info'

the error I get is like this:

AttributeError: 'list' object has no attribute 'iteritems'

I'm really confused here. Plz help me out here.

asked 2 mins ago

برچسب: نویسنده: استخدام کار تاريخ: يکشنبه 2 خرداد 1395 ساعت: 21:49

صفحه بندی