Wednesday, 4 September 2013

Retrieve HTML from webpage in PyQt

Retrieve HTML from webpage in PyQt

I currently have a python script running, logging in the given user to the
web page and navigating to a different page on the site after logging.
What I'm aiming for i to get the raw HTML of the final page after it is
finished loading. I have tried different variation of the Render class but
it seems to throw a "frame is not an attribute of Render" error. Like I
said my main goal is just to get the HTML of the page. What am I doing
incorrect?
url = "https://firstwebpage.com/"
url3 = "https://finaldestinationpage.com" #the page that I want the HTML from
username = "username"
password = "password"
import sys, signal
from PyQt4.QtCore import *
from PyQt4.QtGui import *
from PyQt4.QtWebKit import *
class Render(QWebPage):
def __init__(self, app, url):
QWebPage.__init__(self)
self.loadFinished.connect(self._loadFinished)
self.mainFrame().load(QUrl(url))
app.exec_()
def _loadFinished(self, result):
self.frame = self.mainFrame()
def JSEval(code):
return webpage.mainFrame().evaluateJavaScript(code)
def onLoadStarted():
print("Loading started: %s" % webpage.mainFrame().url().toString())
def onLoadFinished(result):
print("Loading finished: %s" % webpage.mainFrame().url().toString())
if not result:
print("Request failed")
return
JSEval("_form = document.getElementsByName('loginForm')[0];")
JSEval("_form.username.value='%s';" % username \
+ "_form.password.value='%s';" % password \
+ "_form.submit();")
print("Login data sent")
if webpage.mainFrame().url().toString() == url3:
r = Render(app,url3)
html = r.frame.toHtml() #Here is where the "frame" error comes in
print(html)
app = QApplication(sys.argv)
signal.signal(signal.SIGINT, signal.SIG_DFL)
webpage = QWebPage()
webpage.connect(webpage, SIGNAL("loadFinished(bool)"), onLoadFinished)
webpage.connect(webpage, SIGNAL("loadStarted()"), onLoadStarted)
webpage.mainFrame().load(QUrl(url)) #where user is initally logged in
webpage.mainFrame().load(QUrl(url3))
web = QWebView()
web.setPage(webpage)
web.show()
sys.exit(app.exec_())

No comments:

Post a Comment