Commit 50b6ae51 authored by cyrush's avatar cyrush

added tools for creating redmine reports

git-svn-id: http://visit.ilight.com/svn/visit/trunk/src@11801 18c085ea-50e0-402c-830e-de6fd14e8384
parent 1f8e3229
#!/usr/bin/env python
#
# file: setup.py
# author: cdh
#
# distutils setup script for the pyredmine module.
#
from distutils.core import setup
import setup_tests
setup(name='pyrmine',
version='0.1',
author = 'Cyrus Harrison',
author_email = 'cyrush@llnl.gov',
description='Python Module for (limited) Redmine interaction.',
package_dir = {'pyrmine':'src'},
packages=['pyrmine'],
cmdclass = { 'test': setup_tests.ExecuteTests})
\ No newline at end of file
#!/usr/bin/env python
#
# file: setup_tests.py
# author: cdh
#
# Provides 'ExecuteTests' a distutils command class that automatically
# collects and launches unittest test cases from scripts in the 'tests'
# subdir.
#
# I created this simple one file module b/c nose & setuptools are not always
# available/installable on all of the systems I develop python modules on.
#
# To use this class in a distutils setup project:
# A) Create test scripts in the subdir 'tests'
# B) In your setup.py:
# #import the test module
# import setup_tests
# # Add the custom command to the 'cmdclass' dictonary in your setup call:
# setup( ...
# cmdclass = { 'test': setup_tests.ExecuteTests})
#
# C) Invoke 'python setup.py test' to run tests
#
from distutils.core import Command
from unittest import TextTestRunner, TestLoader
import os, glob, sys
class ExecuteTests(Command):
description = "locate and execute scripts containing unit tests."
user_options = [ ('verbose', None, "Verbose output"),
('tests-dir=', None, "Directory containg tests")]
boolean_options = ['verbose']
negative_opt = {'quiet' : 'verbose'}
def initialize_options(self):
self.verbose = 1
self.tests_dir = "tests"
def finalize_options(self):
self.tests_dir = os.path.join(os.getcwd(),self.tests_dir)
def run(self):
"""
Finds and executes unit tests in the 'tests' subdir.
Because TestLoader imports the tests as a module this method
automatically creates/updates the 'tests/__init__.py' to
import all python scripts in the 'tests' subdir.
"""
self.run_command('build')
sys.path.insert(0,os.path.join(os.getcwd(),"build","lib"))
self.tests = []
# make sure the 'tests' subdir actually exists.
if not os.path.isdir(self.tests_dir):
print "ExecuteTests: <Error> 'tests' subdir not found!"
else:
self.find_tests()
self.gen_tests_init()
# create a test suite.
tests = TestLoader().loadTestsFromNames([t[0] for t in self.tests])
# run the test suite if it actually contains test cases.
run_verbosity = 2
if self.verbose == 0:
run_verbosity = 0
if tests.countTestCases() > 0:
runner = TextTestRunner(verbosity=run_verbosity)
runner.run(tests)
else:
print "ExecuteTests: <Warning> No test cases found!"
sys.path.pop(0)
def find_tests(self):
"""
Helper called by 'run' to find python scripts in the 'tests' subdir.
"""
for f in glob.glob(os.path.join(self.tests_dir,"*.py")):
if not f.endswith('__init__.py'):
test = os.path.splitext(os.path.basename(f))[0]
test = '.'.join(['tests',test])
self.tests.append( (test,f))
if len(self.tests) > 0 and self.verbose:
print "Detected Test Scripts:"
for t in self.tests:
print " %s @ %s)" % (t[0],t[1])
def gen_tests_init(self):
"""
Helper called by 'run' to update the 'tests/__init__.py' file.
Note: If no python scripts exist in the 'tests' subdir
'tests/__init__.py' will be removed.
"""
tests_init = os.path.join(self.tests_dir,"__init__.py")
if len(self.tests) > 0:
f = open(tests_init,"w")
f.write("# module header auto generated by "
"setup_tests.ExecuteTests\n")
# import each test script as part of the 'tests' module.
for t in self.tests:
f.write("import %s\n" % t[0])
elif os.path.exists(tests_init):
# remove the module init file if no test scripts are found.
os.remove(tests_init)
\ No newline at end of file
#!/usr/bin/env python
#
# file: Connection.py
# author: Cyrus Harrison <cyrush@llnl.gov>
# created: 6/1/2010
# purpose:
# Provides a 'Connection' class that interacts with a redmine instance to
# extract results from redmine queries.
#
import urllib2,urllib,csv,getpass,warnings
from collections import namedtuple
from Issue import *
try:
import pyPdf
except:
print "Warning: pyrmine requires the 'pyPdf' ",
print "module for full pdf functionality."
class Connection(object):
def __init__(self,base_url):
"""
Creates a redmine connection object to redmine instance at the given
url.
"""
self.urls = {}
if base_url[-1] == "/":
base_url = base_url[:-1]
self.urls["base"] = base_url
self.urls["login"] = "%s/login/" % base_url
self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
def login(self,uname=None,passwd=None):
"""
Login handshake.
If username & passwd are not given this function asks for them via
stdout/stdin.
"""
if uname is None:
uname = raw_input("username:")
if passwd is None:
passwd = getpass.getpass("password:")
f = self.opener.open(self.urls["login"])
data = f.read()
f.close()
split_key = '<input name="authenticity_token" type="hidden" value="'
data = data.split(split_key)[1]
atok= data.split('" />')[0]
params = dict(username=uname,
password=passwd,
authenticity_token=atok)
params = urllib.urlencode(params)
f = self.opener.open(self.urls["login"], params)
data = f.readlines()
f.close()
def open_base_url(self,url):
"""
Constructs and opens an url relative to the base of this connection.
"""
url = "%s/%s" % (self.urls["base"],url)
return self.opener.open(url)
def open_project_url(self,project,url):
"""
Constructs and opens a project url relative to the base of this
connection.
"""
url = "%s/projects/%s/%s" % (self.urls["base"] ,project)
return self.opener.open(url)
def fetch_issues(self,project,query_id=-1,iclass=Issue):
"""
Executes a query and returns a set of Issues holding
the results.
You can specify which class is used to wrap returned issues via 'iclass'.
"""
issues_url = "%s/projects/%s/issues.csv" % (self.urls["base"] ,project)
if int(query_id) >= 0:
params = {}
params['query_id'] = str(query_id)
issues_url += "?" + urllib.urlencode(params)
print "[executing query: %s]" % issues_url
f = self.opener.open(issues_url)
csv_reader = csv.reader(f)
issues = [ row for row in csv_reader]
fields = [self.__format_field_name(val) for val in issues[0]]
issues = issues[1:]
print "[query returned %d issues]" % len(issues)
IssueTuple = namedtuple("Issue",fields)
issues = [iclass(IssueTuple(*i),self) for i in issues]
return fields,issues
def save_query_pdf(self,project,query_id,output_file):
"""
Collects pdfs of all issues returned by a query and combines them into
a single output pdf.
"""
fields,issues = self.fetch_issues(project,query_id)
nissues = len(issues)
if nissues == 0:
print "[query returned no issues -",
print " skipping creation of '%s']" % output_file
return
# try to ingore some deprecation warnings from pyPdf
with warnings.catch_warnings():
warnings.simplefilter("ignore")
opdf = pyPdf.PdfFileWriter()
for i in issues:
print "[downloading issue %s]" % i.id
idata = i.fetch_pdf_buffer()
ipdf = pyPdf.PdfFileReader(idata)
for p in range(ipdf.numPages):
opdf.addPage(ipdf.getPage(p))
print "[creating %s]" % output_file
opdf.write(file(output_file,"wb"))
def __format_field_name(self,name):
"""
Helper that makes sure field names comply w/ rules required for
creating a 'namedtuple' object.
"""
name = name.lower().replace(" ","_")
if name == "#":
name = "id"
name = name.replace("%","percent")
return name
#!/usr/bin/env python
#
# file: base.py
# author: Cyrus Harrison <cyrush@llnl.gov>
# created: 6/1/2010
# purpose:
# Provides an 'Issue' class that wraps issue data returned by redmine queries.
#
import cStringIO
import xml.etree.ElementTree as etree
from Connection import *
class Issue(object):
"""
Wraps an issue returned from a redmine query.
"""
def __init__(self,data,conn):
self.data = data
self.conn = conn
def as_dict(self):
"""
Returns issue data from namedtuple as a python dictonary.
"""
return data._asdict()
def save_pdf(self,output_file):
"""
Downloads a rendered pdf of an issue to a file.
"""
buff = self.fetch_pdf_buffer(self.data.id)
ofile = open(output_file,"wb")
ofile.write(buff.read())
f.close()
def fetch_pdf_buffer(self):
"""
Downloads a rendered pdf of an issue & wraps this data with a
StringIO object suitable for use with pyPdf.
"""
issue_url = "issues/%s.pdf" % self.data.id
f = self.conn.open_base_url(issue_url)
return cStringIO.StringIO(f.read())
def fetch_updates(self):
"""
Returns a list with the contents of each update pulled from the
issue's atom feed.
"""
issue_url = "issues/%s.atom" % self.data.id
f = self.conn.open_base_url(issue_url)
root = etree.fromstring(f.read())
updates = []
for entry in root.findall('{http://www.w3.org/2005/Atom}entry'):
update = {}
for c in entry:
if c.tag == "{http://www.w3.org/2005/Atom}updated":
update["date"] = c.text
if c.tag == "{http://www.w3.org/2005/Atom}author":
update["author"] = c.find("{http://www.w3.org/2005/Atom}name").text
if c.tag == "{http://www.w3.org/2005/Atom}content":
update["content"] = c.text
updates.append(update)
return updates
#!/usr/bin/env python
#
# file: __init__.py
# author: Cyrus Harrison <cyrush@llnl.gov>
# created: 6/1/2010
#
#
from Connection import Connection
from Issue import Issue
#!/usr/bin/env python
#
# file: test_issues.py
# author: Cyrus Harrison (cyrush@llnl.gov)
# created: 6/01/2010
#
#
import unittest,os,sys
import pyrmine
class TestIssues(unittest.TestCase):
def test_csv(self):
rconn = pyrmine.Connection("http://www.redmine.org/")
fields,issues = rconn.fetch_issues("redmine")
self.assertTrue(len(fields) > 0)
self.assertTrue(len(issues) > 0)
if __name__ == '__main__':
unittest.main()
#!/usr/bin/env python
# file: visit_redmine_query_report.py
# author: Cyrus Harrison (cyrush@llnl.gov)
# purpose:
# Generates issue report pdfs from VisIt redmine queries.
# See http://www.visitusers.org/index.php?title=Redmine_Reports for more info.
#
import pyrmine,sys,os,datetime,logging
import ho.pisa as pisa
class VisItIssue(pyrmine.Issue):
"""
Extents standard issue class to provide VisIt specific html output.
"""
def __init__(self,data,conn):
pyrmine.Issue.__init__(self,data,conn)
def to_html(self):
"""
Creates an html representation of a VisIt issue.
"""
i = self.data._asdict()
res = "<h1># %s (%s) %s</h1>\n" % (i["id"],i["tracker"],i["subject"])
res += "<table><tr><td>\n"
for key in ["status","author","created","updated","priority","assigned_to"]:
res += "<b>%s</b>: %s<br>\n" % (key,self.__format_blank(i[key]))
res+="</td><td>\n"
if i["tracker"] == "Feature":
for key in ["expected_use","impact"]:
res += "<b>%s</b>: %s<br>\n" % (key,self.__format_blank(i[key]))
elif i["tracker"] == "Bug":
for key in ["likelihood","severity","found_in_version"]:
res += "<b>%s</b>: %s<br>\n" % (key,self.__format_blank(i[key]))
for key in ["os","support_group","estimated_time"]:
res += "<b>%s</b>: %s<br>\n" % (key,self.__format_blank(i[key]))
res +="</td></tr><table>\n"
res +="<p><b>Description:</b><br>\n"
res += i["description"].replace("\n","<br>") + "</p>\n<p>"
updates = self.fetch_updates()
if len(updates) == 0:
res += "<b>(No Updates)</b><br>\n"
else:
if len(updates) == 1:
res += "<b>History: (1 Update)</b><br>\n"
else:
res += "<b>History: (%d Updates)</b><br>\n" % len(updates)
for u in updates:
res += "<blockquote>\n<b>%s</b> (%s)<br>" % (u["author"],u["date"])
res += "%s\n</blockquote>\n" % u["content"]
res += "</p><hr>\n"
return res
def __format_blank(self,txt):
"""
Returns (unset) to indicate a blank field value.
"""
if txt.strip() == "":
return "(unset)"
return txt
class VisItReportGenerator(object):
"""
Creates html/pdf reports from redmine queries.
"""
def __init__(self):
base_url = 'https://visitbugs.ornl.gov/'
print "[connecting to redmine instance @ %s]" % base_url
self.conn = pyrmine.Connection(base_url);
self.conn.login()
def generate_query_report(self,query_id,obase):
obase = os.path.abspath(obase)
ohtml = obase + ".html"
opdf = obase + ".pdf"
print "[fetching issues]"
fields,issues = self.conn.fetch_issues("visit",query_id,VisItIssue)
print "[generating %s]" % ohtml
fhtml = open(ohtml,"w")
fhtml.write(self.__generate_html_start())
for i in issues:
fhtml.write(i.to_html())
fhtml.write(self.__generate_html_end())
fhtml.close()
print "[generating %s]" % opdf
pdf = pisa.CreatePDF( open(ohtml, "rb"),open(opdf, "wb"))
def __generate_html_start(self):
"""
Generates the start of html doc, creates style sheet & header.
"""
res = """
<html>
<style>
@page {
size: letter;
margin: 1cm;
@frame footer {
-pdf-frame-content: footerContent;
bottom: .5cm;
margin-left: .5cm;
margin-right: .5cm;
height: .5cm;
}
}
h1 {
font-size: 135%;
}
td,
p {
font-size: 130%;
}
li {
font-size: 125%;
}
</style>
<body>
<table><tr><td>
<h2> VisIt Redmine issue report </h2>
</td><td align="right">
<h5> Generated:
"""
res += str(datetime.datetime.now())
res += "</h5>\n</td></tr>\n</table>\n<hr>\n"
return res
def __generate_html_end(self):
"""
Generates the end of html doc, creates page # footer.
"""
return """
<div id="footerContent" align="right">
Page #<pdf:pagenumber>
</div>
</body>
</html>
"""
def parse_args():
if len(sys.argv) < 3:
print "usage: visit_redmine_query_report.py [query_id] [output_base]"
sys.exit(-1)
qid, ofile = sys.argv[1:3]
return qid,ofile
if __name__ == "__main__":
logging.basicConfig(level=logging.ERROR)
query_id, obase = parse_args()
vrq = VisItReportGenerator()
vrq.generate_query_report(query_id,obase)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment