Aug-05-2020, 02:06 AM
I am struggling with how to properly extract an oddly formatted date that exists on the page in one location using Python Scrapy.
What needs to be changed to have the date included on every output row in yyyy-mm-dd format?
Problematic code lines:
Here is the full spider.
What needs to be changed to have the date included on every output row in yyyy-mm-dd format?
Problematic code lines:
data2 = response.xpath('//span[@class="tab"]/text()').get().replace(". ", "-") date = datetime.datetime.strptime(data2, "%d-%m-%Y").strftime("%Y-%m-%d")Sample output appears to contain one character for date. Example: {'match_id': '1893065', 'date': '0'}
Here is the full spider.
import scrapy import datetime import re from datetime import timedelta class Tennis_ExplorerSpider(scrapy.Spider): name = 'tennis_explorer' allowed_domains = ['tennisexplorer.com'] def daterange(start_date, end_date): for n in range(int((end_date - start_date).days)): yield start_date + timedelta(n) start_date = datetime.datetime.today() - datetime.timedelta(days=1) end_date = datetime.datetime.today() + datetime.timedelta(days=1) start_urls = [] start_url='https://www.tennisexplorer.com/matches/?type=all&year=' for single_date in daterange(start_date, end_date): start_urls.append(single_date.strftime(start_url+"%Y&month=%m&day=%d&timezone=-6")) def parse(self, response): #Extracting the content using xpath self.logger.debug('callback "parse": got response %r' % response) data = response.xpath('//table[@class="result"]//a[contains(@href,"match-detail")]/@href').extract() match_id =[re.sub('^.+=','',el) for el in data] data2 = response.xpath('//span[@class="tab"]/text()').get().replace(". ", "-") date = datetime.datetime.strptime(data2, "%d-%m-%Y").strftime("%Y-%m-%d") #Give the extracted content row wise for item in zip(match_id, date): #create a dictionary to store the scraped info scraped_info = { 'match_id' : item[0], 'date' : item[1] } #yield or give the scraped info to scrapy yield scraped_info