Hi everyone I am having some truoble storing the cookies after the authentication. I managed to authenticate and here is a dummy user account to play with. What I want to do is after authenticating go to another part of the site by changin the url and not by clicking in it to save time.
class ExampleSpider(scrapy.Spider):
name = 'example'
start_urls = ["https://ident.familysearch.org/cis-web/oauth2/v3/authorization?client_secret=K7ymz%2FugczIps3SKrEjpp41QmXcDGK0BrGkSnGIunpHwQ0pq1rs%2FoAcsLaouJ0zbPaFtJ5eG7MdZ64LjiB3y5rMVFURm%2B18fjRPvje%2BeCwHiqBVqgu5i2gXzaHgaCi33ylnqSw%2F0h9uuLjo0myEJ%2FFDUhm8W9iTxDdKTonoYqbGkQ%2FwdfqPjKwLtXRjlEXq5mpkdkQZciVY4MDiSBaZJ3DfKZihyveKZSp3dC%2FJ3nH7Q%2FUjWpg8RP4fvR%2BN4ValGmrC9imWLOPlt5dwKOPsTF8z9AiOyaGOvxEn5rYzf4iNZ35Hr7X9p87BRhKvTRSuQJr24y4aPpC33EAr7QrmgZw%3D%3D&response_type=code&redirect_uri=https%3A%2F%2Fwww.familysearch.org%2Fauth%2Ffamilysearch%2Fcallback&state=%2F&client_id=3Z3L-Z4GK-J7ZS-YT3Z-Q4KY-YN66-ZX5K-176R",
"https://www.familysearch.org/search/record/results?count=20&q.anyDate.from=1500&q.anyDate.to=1650&q.anyPlace=Lima"]
def start_requests(self):
script = """
function main(splash,args)
splash:init_cookies(splash.args.cookies)
splash:set_viewport_size(1366, 768)
splash:set_user_agent('Splash bot')
local url = splash.args.url
splash:go(url)
splash:wait(5)
local form = splash: select('form[name=eventForm]')
local values = {
userName = 'abdc199',
password = 'github199',
}
assert(form:fill(values))
assert(form:submit())
assert(splash:wait(10))
return {
html=splash:html(),
cookies=splash.get_cookies()
}
end
"""
yield SplashRequest(url=self.start_urls[0],
callback=self.after_login,
session_id='1',
endpoint='execute',
args={
'lua_source': script,
'wait': 5})
def after_login(self, response):
# check login succeed before going on
if "authentication failed" in response.text:
self.logger.error("Login failed")
return
else:
url = "https://www.familysearch.org/search/record/results?count=20&q.anyDate.from=1500&q.anyDate.to=1650&q.anyPlace=Lima"
yield SplashRequest(url, self.parse_url, session_id='1', args={'wait':5})
def parse_url(self, response):
yield {
'response':response.text
}
Hi everyone I am having some truoble storing the cookies after the authentication. I managed to authenticate and here is a dummy user account to play with. What I want to do is after authenticating go to another part of the site by changin the url and not by clicking in it to save time.
Here is my code