# HTMLReport.py
#
# Copyright (C) 2012 Carlos Garcia Campos
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from __future__ import absolute_import, division, print_function
from backends import get_backend, get_all_backends
from Config import Config
import os
import errno
import subprocess
class HTMLPrettyDiff:
def write(self, test, outdir, actual, expected, diff):
raise NotImplementedError
def _create_diff_for_test(self, outdir, test):
diffdir = os.path.join(outdir, 'html', test)
try:
os.makedirs(diffdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
except:
raise
return diffdir
class HTMLPrettyDiffImage(HTMLPrettyDiff):
def write(self, test, outdir, result, actual, expected, diff):
def get_relative_path(path):
return '../' * len(path.split('/')) + path
html = """
%s
Difference between images: diff
Loading...
""" % (test, get_relative_path(diff), get_relative_path(actual), expected)
diffdir = self._create_diff_for_test(outdir, test)
pretty_diff_name = result + '-pretty-diff.html'
pretty_diff = os.path.abspath(os.path.join(diffdir, pretty_diff_name))
f = open(pretty_diff, 'w')
f.write(html)
f.close()
return os.path.join(test, pretty_diff_name)
class HTMLPrettyDiffText(HTMLPrettyDiff):
def write(self, test, outdir, result, actual, expected, diff):
import difflib
actual_file = open(os.path.join(outdir, actual), 'r')
expected_file = open(expected, 'r')
html = difflib.HtmlDiff().make_file(actual_file.readlines(),
expected_file.readlines(),
"Actual", "Expected", context=True)
actual_file.close()
expected_file.close()
diffdir = self._create_diff_for_test(outdir, test)
pretty_diff_name = result + '-pretty-diff.html'
pretty_diff = os.path.abspath(os.path.join(diffdir, pretty_diff_name))
f = open(pretty_diff, 'w')
f.write(html)
f.close()
return os.path.join(test, pretty_diff_name)
def create_pretty_diff(backend):
if backend.get_diff_ext() == '.diff.png':
return HTMLPrettyDiffImage()
# Disable pretty diff for Text files for now, since HtmlDiff().make_file() is
# entering in an infinite loop with some files. We need to either fix that somehow or
# find a different way to generate pretty diffs of text files.
#if backend.get_diff_ext() == '.diff':
# return HTMLPrettyDiffText()
return None
class BackendTestResult:
def __init__(self, test, refsdir, outdir, backend, results):
self._test = test
self._refsdir = refsdir
self._outdir = outdir
self._backend = backend
self.config = Config()
self._results = []
ref_path = os.path.join(self._refsdir, self._test)
if not backend.has_md5(ref_path):
return
ref_names = backend.get_ref_names(ref_path)
for result in results:
basename = os.path.basename(result)
if basename in ref_names:
self._results.append(basename)
def is_failed(self):
return len(self._results) > 0
def is_crashed(self):
return self._backend.is_crashed(os.path.join(self._outdir, self._test))
def is_failed_to_run(self):
return self._backend.is_failed(os.path.join(self._outdir, self._test))
def get_stderr(self):
return self._backend.get_stderr(os.path.join(self._outdir, self._test))
def get_failed_html(self):
html = ""
for result in self._results:
actual = os.path.join(self._test, result)
actual_path = os.path.join(self._outdir, actual)
expected = os.path.join(self._refsdir, self._test, result)
if self.config.abs_paths:
expected = os.path.abspath(expected)
html += "actual expected " % (actual, expected)
if self._backend.has_diff(actual_path):
diff = actual + self._backend.get_diff_ext()
html += "diff " % (diff)
if self.config.pretty_diff:
pretty_diff = create_pretty_diff(self._backend)
if pretty_diff:
html += "pretty diff " % (pretty_diff.write (self._test, self._outdir, result, actual, expected, diff))
html += "\n"
if html:
return "\n" % (html)
return ""
class TestResult:
def __init__(self, docsdir, refsdir, outdir, resultdir, results, backends):
self._refsdir = refsdir
self._outdir = outdir
self.config = Config()
self._test = resultdir[len(self._outdir):].lstrip('/')
self._doc = os.path.join(docsdir, self._test)
self._results = {}
for backend in backends:
ref_path = os.path.join(self._refsdir, self._test)
if not backend.has_md5(ref_path) and not backend.is_crashed(ref_path) and not backend.is_failed(ref_path):
continue
self._results[backend] = BackendTestResult(self._test, refsdir, outdir, backend, results)
def get_test(self):
return self._test
def is_failed(self):
for backend in self._results:
if self._results[backend].is_failed():
return True
return False;
def get_failed_html(self):
html = ""
for backend in self._results:
if self._results[backend].is_crashed() or self._results[backend].is_failed_to_run():
continue
backend_html = self._results[backend].get_failed_html()
if not backend_html:
continue
html += "%s " % (backend.get_name())
stderr = self._results[backend].get_stderr()
if os.path.exists(stderr):
stderr_name = stderr[len(self._outdir):].lstrip('/')
html += "stderr" % (stderr_name)
html += "\n%s" % (backend_html)
if html:
return "\nTop\n" % (self._test, self._doc, self._test, html)
return ""
def get_crashed_html(self):
html = ""
for backend in self._results:
if not self._results[backend].is_crashed():
continue
html += "%s (%s)\n" % (self._doc, self._test, backend.get_name())
if html:
return "\n" % (html)
return ""
def get_failed_to_run_html(self):
html = ""
for backend in self._results:
status = self._results[backend].is_failed_to_run()
if not status:
continue
html += "%s [Status: %d] (%s)\n" % (self._doc, self._test, status, backend.get_name())
if html:
return "\n" % (html)
return ""
class HTMLReport:
def __init__(self, docsdir, refsdir, outdir):
self._docsdir = docsdir
self._refsdir = refsdir
self._outdir = outdir
self._htmldir = os.path.join(outdir, 'html')
self.config = Config()
try:
os.makedirs(self._htmldir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
except:
raise
def create(self, launch_browser):
html = ""
if os.path.exists(os.path.join(self._outdir, '.exited_early')):
html += "Testing exited early
"
if self.config.backends:
backends = [get_backend(name) for name in self.config.backends]
else:
backends = get_all_backends()
results = {}
for root, dirs, files in os.walk(self._outdir, False):
if not files:
continue
if not root.lower().endswith('.pdf'):
continue
if root.startswith(self._htmldir):
continue
results[root] = TestResult(self._docsdir, self._refsdir, self._outdir, root, files, backends)
failed_anchors = []
failed = ""
crashed = ""
failed_to_run = ""
for test_name, test in sorted(results.items()):
if test.is_failed():
failed_anchors.append(test.get_test())
failed += test.get_failed_html()
crashed += test.get_crashed_html()
failed_to_run += test.get_failed_to_run_html()
if failed:
failed = "\n%s" % (failed)
if crashed:
crashed = "\n%s" % (crashed)
if failed_to_run:
failed_to_run = "\n%s" % (failed_to_run)
if failed or crashed or failed_to_run:
html += "\n"
html += failed + crashed + failed_to_run + ""
report_index = os.path.join(self._htmldir, 'index.html')
with open(report_index, 'w') as f:
f.write(html)
if launch_browser:
subprocess.Popen(['xdg-open', report_index])