Large restructuration

10 files changed, 448 insertions, 591 deletions

diff --git a/.gitignore b/.gitignore
index 97ee5c5..4869765 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,4 @@
 *pyc
 *~
 *#
-tmp
-out
+tools/tmp
diff --git a/TODO b/TODO
index 725adb3..68ef59c 100644
--- a/TODO
+++ b/TODO
@@ -1,18 +1,19 @@
 Implementation
 ==============
 - Implement Report
+- Implement GUI
+- Implement kernel testing framwork
 - Handle "best kernel not found"
 - Add further error handling
-- Implement find_cve() (return bugid)
-- Create a GENERIC-MAP-NOMATCH entry in the cve xml files
 - Implement hardend/xen intervall
 
 Cleanup and Rework
 ==================
+- Rework cron.py
 - Remove unused code and find better ways
 - Check lookaround of 'grp_all'
-- Rework interval class and all interval functions (expand?)
-- Rework cves.refs and find a solution for <ref source url/>
+- Rework interval class
+- Rework cves.refs
 
 Dokumentation
 =============
@@ -23,7 +24,6 @@ Dokumentation
 Whiteboard changes
 ==================
 - Move arch into whiteboard e.g. {x86, amd64}
-- [kernel +<version] -> [+kernel <version]
 
 Summary changes
 ===============
diff --git a/collector.py b/collector.py
deleted file mode 100755
index 30f3272..0000000
--- a/collector.py
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env python
-# kernel-check -- Kernel security information
-# Copyright 2009-2009 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-import getopt
-import os
-import sys
-import time
-
-import kernellib as lib
-
-
-def main(argv):
-    'Main function'
-
-    try:
-        opts, args = getopt.getopt(argv, 'd:fh:sv',
-        ['delay=', 'force', 'help', 'skip', 'verbose'])
-    except getopt.GetoptError:
-        usage()
-        return
-
-    for opt, arg in opts:
-        if opt in ('-d', '--delay'):
-            if arg.isdigit():
-                lib.DELAY = int(arg)
-        elif opt in ('-f', '--force'):
-            lib.FORCE = True
-        elif opt in ('-h', '--help'):
-            usage()
-            return
-        elif opt in ('-s', '--skip'):
-            lib.SKIP = True
-        elif opt in ('-v', '--verbose'):
-            lib.VERBOSE = True
-
-    for directory in lib.DIR:
-        if not os.path.isdir(lib.DIR[directory]):
-            os.makedirs(lib.DIR[directory])
-
-    print 'Reading available genpatches...'
-    try:
-        read_patches = lib.read_genpatch_file(lib.DIR['out'])
-    except IOError:
-        read_patches = list()
-
-    print 'Parsing genpatches from portage...'
-    found_patches = lib.parse_genpatch_list(lib.PORTDIR)
-
-    new_patches = 0
-    for item in found_patches:
-        if item not in read_patches:
-            read_patches.append(item)
-            new_patches += 1
-
-    if (new_patches):
-        lib.write_genpatch_file(lib.DIR['out'], read_patches)
-        print 'Added %i new genpatches!' % new_patches
-
-    print '\nReceiving the latest xml file from the nvd...'
-    lib.receive_nvd_recent(lib.DIR['nvd'])
-
-    if not lib.SKIP:
-        print 'Receiving earlier xml files from the nvd...'
-        lib.receive_nvd_all(lib.DIR['nvd'])
-
-    print 'Creating the nvd dictionary...'
-    nvd_dict = lib.parse_nvd_dict(lib.DIR['nvd'])
-
-    print 'Receiving the kernel vulnerability list from bugzilla...'
-    lib.receive_bugzilla_list(lib.DIR['tmp'])
-
-    buglist = lib.parse_bugzilla_list(lib.DIR['tmp'])
-    print 'Found %i kernel vulnerabilities!' % len(buglist)
-
-    print '\nCreating the xml files...'
-
-    created_files = 0
-    for item in buglist:
-        try:
-            lib.receive_bugzilla_bug(lib.DIR['bug'], item)
-            vul = lib.parse_bugzilla_dict(lib.DIR['bug'], item)
-            vul = lib.search_nvd_dict(nvd_dict, vul)
-            lib.write_cve_file(lib.DIR['out'], vul)
-
-            created_files += 1
-            time.sleep(lib.DELAY)
-
-        except lib.InvalidWhiteboardError, e:
-            print '\n[%s] Invalid whiteboard' % item
-            print '%s' % e.value
-
-        except lib.InvalidCveError, e:
-            print '\n[%s] Invalid CVE' % item
-            print '%s' % e.value
-
-        except lib.NvdEntryError, e:
-            print '\n[%s] No Nvd Entry' % item
-            print '%s' % e.value
-
-        except lib.CveDuplicateError, e:
-            print '\n[%s] CVE Duplicate' % item
-            print '%s' % e.value
-
-    print '\nCreated %i xml files!' % created_files
-
-
-def usage():
-    'Prints the usage screen'
-
-    print 'Usage: %s [OPTION]...' % sys.argv[0][:-3]
-    print 'Kernel security information %s\r\n' % lib.VERSION
-    print '  -d, --delay [ticks]  add delay to xml file creation'
-    print '  -f, --force          force update of xml files'
-    print '  -h, --help           display help information'
-    print '  -s, --skip           skip update of prior nvd files'
-    print '  -v, --verbose        display additional information'
-
-
-if __name__ == '__main__':
-    main(sys.argv[1:])
-
diff --git a/kernel-check.py b/kernel-check.py
index 9c7e563..a6d6051 100755
--- a/kernel-check.py
+++ b/kernel-check.py
@@ -9,7 +9,7 @@ import sys
 import textwrap
 import os
 
-import kernellib as lib
+import lib.kernellib as lib
 
 info = portage.output.EOutput().einfo
 warn = portage.output.EOutput().ewarn
@@ -61,8 +61,10 @@ def main(argv):
           (kernel.version, kernel.revision))))
     info('Kernel source  : %s' % color('GOOD', kernel.source))
 
-    kernel.genpatch = lib.get_genpatch(lib.read_genpatch_file(lib.DIR['out']),
-                                                        kernel)
+
+    genpatches = lib.parse_genpatch_list(lib.PORTDIR)
+
+    kernel.genpatch = lib.get_genpatch(genpatches, kernel)
 
     if kernel.genpatch is not None:
         info('Gen(too)patch  : %s' % color('GOOD', '%s %s' %
@@ -79,6 +81,7 @@ def main(argv):
 
     print '\n>>> Reading all kernel vulnerabilities'
 
+    """
     supported = list()
     for item in lib.SUPPORTED:
         best = (lib.all_version(item))
@@ -88,6 +91,7 @@ def main(argv):
                     i.genpatch = lib.get_genpatch(lib.read_genpatch_file(
                                                   lib.DIR['out']), i)
                 supported.append(i)
+    """
 
     kernel_eval = lib.eval_cve_files(lib.DIR['out'], kernel, arch)
     if not kernel_eval:
@@ -109,6 +113,7 @@ def main(argv):
         color('BAD', str(len(kernel_eval.affected))))
     print_summary(kernel_eval.affected)
 
+    """
     info('You have the following choices: ')
     print ''
 
@@ -148,6 +153,7 @@ def main(argv):
                       color('BAD', str(len(kernel_eval.affected))),
                       color('BAD', str(len(comparison.new)))))
                 print ''
+    """
 
     print_information()
     print_beta()
@@ -244,17 +250,15 @@ def print_beta():
     'Prints a beta warning message'
 
     print('')
-    error('%s You are using a very early version of kernel-check.' %
+    error('%s You are using a early version of kernel-check.' %
           color('BAD', 'IMPORTANT'))
     error('Please note that this tool might not operate as expected.')
-    error('Moreover the given information are most likely incorrect.')
 
 
 def print_information():
     'Prints an information message'
 
     info('To print more information about a vulnerability try:')
-    info('')
     info('   $ %s -s [bugid|cve]' % sys.argv[0])
 
 
diff --git a/lib/__init__.py b/lib/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/__init__.py
diff --git a/guidexml.py b/lib/guidexml.py
index d6babb8..d6babb8 100755
--- a/guidexml.py
+++ b/lib/guidexml.py
diff --git a/kernellib.py b/lib/kernellib.py
index fe63a3e..d320c20 100755
--- a/kernellib.py
+++ b/lib/kernellib.py
@@ -27,19 +27,8 @@ BUGORDER = ['bugid', 'reporter', 'reported', 'status', 'arch', 'affected']
 CVEORDER = ['cve', 'published', 'desc', 'severity', 'vector', 'score', 'refs']
 
 REGEX = {
-    'bugzilla'   : re.compile(r'(?<=bug.cgi\?id=)\d*'),
     'gp_version' : re.compile(r'(?<=K_GENPATCHES_VER\=\").+(?=\")'),
     'gp_want'    : re.compile(r'(?<=K_WANT_GENPATCHES\=\").+(?=\")'),
-    'grp_all'    : re.compile(r'(?<=\()[ (]*CVE-(\d{4})' \
-                              r'([-,(){}|, \d]+)(?=\))'),
-    'grp_split'  : re.compile(r'(?<=\D)(\d{4})(?=\D|$)'),
-    'm_nomatch'  : re.compile(r'.*GENERIC-MAP-NOMATCH.*'),
-    'wb_match'   : re.compile(r'\s*\[\s*([^ +<=>]+)\s*(\+?)' \
-                              r'\s*([<=>]{1,2})\s*([^ <=>\]' \
-                              r']+)\s*(?:([<=>]{1,2})\s*([^' \
-                              r' \]]+))?\s*\]\s*(.*)'),
-    'wb_version' : re.compile(r'^(?:\d{1,2}\.){0,3}\d{1,2}' \
-                              r'(?:[-_](?:r|rc)?\d{1,2})*$'),
     'k_version'  : re.compile(r'^((?:\d{1,2}\.){0,4}\d{1,2})(-.*)?$'),
     'rc_kernel'  : re.compile(r'^rc\d{1,3}$'),
     'git_kernel' : re.compile(r'^git(\d{1,3})$'),
@@ -61,23 +50,23 @@ KERNEL_TYPES = [
     'vserver', 'win4lin', 'wolk-dev', 'wolk', 'xbox', 'xen', 'xfs'
 ]
 
-VERSION = '0.3.8'
+VERSION = '0.3.9'
 NOCVE = 'GENERIC-MAP-NOMATCH'
 NOCVEDESC = 'This GENERIC identifier is not specific to any vulnerability. '\
             'GENERIC-MAP-NOMATCH is used by products, databases, and ' \
             'services to specify when a particular vulnerability element ' \
             'does not map to a corresponding CVE entry.'
 CVES = dict()
-PORTDIR = portage.settings['PORTDIR']
 DEBUG = False
 VERBOSE = False
 FORCE = False
 SKIP = False
 DELAY = 0
 FILEPATH = os.path.dirname(os.path.realpath(__file__))
+PORTDIR = portage.settings['PORTDIR']
 DIR = {
     'tmp' : os.path.join(FILEPATH, 'tmp'),
-    'out' : os.path.join(FILEPATH, 'out'),
+    'out' : os.path.join(PORTDIR, 'metadata', 'kernel'),
     'bug' : os.path.join(FILEPATH, 'tmp', 'bug'),
     'nvd' : os.path.join(FILEPATH, 'tmp', 'nvd')
 }
@@ -87,22 +76,6 @@ def BUG_ON(msg):
         print 'DEBUG line %s in %s(): %s' % (inspect.stack()[1][2],
               inspect.stack()[1][3], msg)
 
-class InvalidWhiteboardError(Exception):
-    def __init__(self, value):
-        self.value = value
-
-class InvalidCveError(Exception):
-    def __init__(self, value):
-        self.value = value
-
-class CveDuplicateError(Exception):
-    def __init__(self, value):
-        self.value = value
-
-class NvdEntryError(Exception):
-    def __init__(self, value):
-        self.value = value
-
 class Evaluation:
     """Evaluation class
 
@@ -311,21 +284,6 @@ class Interval:
         return interval
 
 
-def interval_to_xml(interval, root):
-    'Formats an interval for xml output'
-
-    intnode = et.Element('interval')
-    intnode.set('source', interval.name)
-
-    root.append(intnode)
-
-    for item in ('lower', 'upper'):
-        if getattr(interval, item):
-            node = et.SubElement(intnode, item)
-            node.text = getattr(interval, item)
-            node.set('inclusive', str(getattr(interval, item + '_i')).lower())
-
-
 def interval_from_xml(root):
     'Returns an interval from xml'
 
@@ -395,33 +353,7 @@ def is_in_interval(interval, kernel, bugid=None):
 
     return True
 
-
-def extract_genpatch(ebuild, directory, sources):
-    'Returns a genpatch from an ebuild'
-
-    pkg = portage.versions.catpkgsplit('sys-kernel/%s' % ebuild[:-7])
-
-    with open(os.path.join(directory, sources, ebuild), 'r') as ebuild_file:
-        content = ebuild_file.read()
-
-    try:
-        genpatch_v = REGEX['gp_version'].findall(content)[0]
-        genpatch_w = REGEX['gp_want'].findall(content)[0]
-    except:
-        return None
-
-    kernel = Kernel(pkg[1].replace('-sources', ''))
-    kernel.version = pkg[2]
-    kernel.revision = pkg[3]
-
-    genpatch = Genpatch(pkg[2] + '-' + genpatch_v)
-    genpatch.kernel = kernel
-    genpatch.base = ('base' in genpatch_w)
-    genpatch.extras = ('extras' in genpatch_w)
-
-    return genpatch
-
-
+#TODO Add inline get_genpatch
 def parse_genpatch_list(directory):
     'Returns a list containing all genpatches from portage'
 
@@ -440,55 +372,6 @@ def parse_genpatch_list(directory):
     return patches
 
 
-def read_genpatch_file(directory):
-    'Read the genpatch file created by collector'
-
-    patches = list()
-    filename = os.path.join(directory, 'genpatches.xml')
-
-    try:
-        with open(filename, 'r+') as xml_data:
-            memory_map = mmap.mmap(xml_data.fileno(), 0)
-            root = et.parse(memory_map).getroot()
-
-    except SyntaxError:
-        return list()
-
-    except IOError:
-        return list()
-
-    for tree in root:
-        kernel = extract_version(tree.get('kernel'))
-
-        if kernel is None:
-            continue
-
-        genpatch = Genpatch(tree.get('version'))
-        genpatch.kernel = kernel
-        genpatch.base = (tree.get('base') == 'true')
-        genpatch.extras = (tree.get('extras') == 'true')
-
-        patches.append(genpatch)
-
-    return patches
-
-
-def write_genpatch_file(directory, patches):
-    'Write the genpatch file with all genpatches'
-
-    filename = os.path.join(directory, 'genpatches.xml')
-    root = et.Element('patches')
-
-    for item in patches:
-        genpatch = et.SubElement(root, 'genpatch')
-        genpatch.set('kernel', repr(item.kernel))
-        genpatch.set('version', item.version)
-        genpatch.set('base', str(item.base).lower())
-        genpatch.set('extras', str(item.extras).lower())
-
-    write_xml(root, filename)
-
-
 def get_genpatch(patches, kernel):
     'Returns the genpatch for a specific kernel'
 
@@ -499,142 +382,30 @@ def get_genpatch(patches, kernel):
     return None
 
 
-def parse_bugzilla_list(directory):
-    'Returns a list containing all bugzilla kernel bugs'
-
-    filename = os.path.join(directory, 'bugzilla.xml')
-
-    with open(filename, 'r+') as buglist_file:
-        memory_map = mmap.mmap(buglist_file.fileno(), 0)
-
-    buglist = REGEX['bugzilla'].findall(memory_map.read(-1))
-
-    return buglist
-
-
-def parse_bugzilla_dict(directory, bugid):
-    'Returns a vulnerability class containing information about a bug'
-
-    filename = os.path.join(directory, bugid)
-
-    try:
-        with open(filename, 'r+') as xml_data:
-            memory_map = mmap.mmap(xml_data.fileno(), 0)
-            root = et.parse(memory_map).getroot()[0]
-
-    except IOError: #FIXME Handle Exception
-        return
-
-    vul = Vulnerability(bugid)
-
-    try:
-        vul.cvelist = extract_cves(root.find('short_desc').text)
-        if not vul.cvelist:
-            raise InvalidCveError(root.find('short_desc').text)
-
-        for item in vul.cvelist:
-            if item != NOCVE:
-                if item not in CVES:
-                    CVES[item] = vul.bugid
-                else:
-                    raise CveDuplicateError(CVES[item])
+def extract_genpatch(ebuild, directory, sources):
+    'Returns a genpatch from an ebuild'
 
-        vul.arch = root.find('rep_platform').text.lower()
-        vul.reported = root.find('creation_ts').text
-        vul.reporter = root.find('reporter').text.lower()
-        vul.status = root.find('bug_status').text.lower()
+    pkg = portage.versions.catpkgsplit('sys-kernel/%s' % ebuild[:-7])
 
-    except AttributeError:
-        #TODO Error
-        pass
+    with open(os.path.join(directory, sources, ebuild), 'r') as ebuild_file:
+        content = ebuild_file.read()
 
     try:
-        vul.affected = interval_from_whiteboard(
-                       root.find('status_whiteboard').text)
-
-    except AttributeError:
-        raise InvalidWhiteboardError('Empty')
-
-    return vul
-
-
-def search_nvd_dict(nvd, vul):
-    'Adds all matching cves found in the nvd dicitonay to vul'
-
-    cves = list()
-    for item in vul.cvelist:
-        if item == NOCVE:
-            vul.cves = list()
-            return vul
-
-        try:
-            cves.append(nvd[item])
-        except KeyError:
-            raise NvdEntryError(item)
-
-    vul.cves = cves
-
-    return vul
-
-
-def parse_nvd_dict(directory):
-    'Returns a dictionary from the National Vulnerability Database'
-
-    nvd = dict()
-
-    for nvdfile in os.listdir(directory):
-        filename = os.path.join(directory, nvdfile)
-
-        with open(filename, 'r+') as xml_data:
-            memory_map = mmap.mmap(xml_data.fileno(), 0)
-            root = et.parse(memory_map).getroot()
-
-        namespace = root.tag[:-3]
-
-        for tree in root:
-            cve = Cve(tree.get('name'))
-            cve.published = tree.get('published')
-            cve.severity = tree.get('severity')
-            cve.vector = tree.get('CVSS_vector')
-            cve.score = tree.get('CVSS_score')
-
-            #FIXME
-            desc = tree.find('%sdesc/%sdescript/' % (namespace, namespace))
-            if desc is not None:
-                cve.desc = desc.text
-
-            #TODO Rework!
-            reftree = tree.find(namespace + 'refs')
-            reftree.tag = reftree.tag.replace(namespace, '')
-            for elem in reftree.findall('.//*'):
-                elem.tag = elem.tag.replace(namespace, '')
-            bugref = et.SubElement(reftree, 'ref')
-            bugref.set('source', 'GENTOO')
-            bugref.set('url', 'https://bugs.gentoo.org/show_bug.cgi?id=%s' %
-                        cve.cve)
-            bugref.text = 'Gentoo %s' % cve.cve
-
-            cve.refs = reftree
-
-            nvd[cve.cve] = cve
-
-    return nvd
-
-
-def extract_cves(string):
-    'Returns a list containing all CVEs of a particular string'
-
-    cves = list()
-    string = string.replace('CAN', 'CVE')
+        genpatch_v = REGEX['gp_version'].findall(content)[0]
+        genpatch_w = REGEX['gp_want'].findall(content)[0]
+    except:
+        return None
 
-    if string in REGEX['m_nomatch'].findall(string):
-        return [NOCVE]
+    kernel = Kernel(pkg[1].replace('-sources', ''))
+    kernel.version = pkg[2]
+    kernel.revision = pkg[3]
 
-    for (year, split_cves) in REGEX['grp_all'].findall(string):
-        for cve in REGEX['grp_split'].findall(split_cves):
-            cves.append('CVE-%s-%s' % (year, cve))
+    genpatch = Genpatch(pkg[2] + '-' + genpatch_v)
+    genpatch.kernel = kernel
+    genpatch.base = ('base' in genpatch_w)
+    genpatch.extras = ('extras' in genpatch_w)
 
-    return cves
+    return genpatch
 
 
 def parse_cve_files(directory):
@@ -695,15 +466,14 @@ def eval_cve_files(directory, kernel, arch):
 
     return evaluation
 
-
-def is_affected(interval_list, kernel, item): #TODO Remove item
+#TODO Remove item
+def is_affected(interval_list, kernel, item):
     'Returns true if a kernel is affected'
 
     kernel_gentoo = (kernel.source == 'gentoo' and kernel.genpatch is not None)
     kernel_affected = False
     kernel_linux_affected = False
     kernel_gp_affected  = False
-    #TODO kernel_gp_exp_affected  = False
     linux_interval = False
     gentoo_interval = False
 
@@ -810,125 +580,6 @@ def read_cve_file(directory, bugid):
 
     return vul
 
-
-def write_cve_file(directory, vul):
-    'Write a bug file containing all important information for kernel-check'
-
-    filename = os.path.join(directory, vul.bugid + '.xml')
-
-    root = et.Element('vulnerability')
-    bugroot = et.SubElement(root, 'bug')
-
-    for element in BUGORDER:
-        if element == 'affected':
-                affectedroot = et.SubElement(bugroot, 'affected')
-                for item in vul.affected:
-                    interval_to_xml(item, affectedroot)
-        else:
-            node = et.SubElement(bugroot, element)
-            node.text = getattr(vul, element)
-
-    for cve in vul.cves:
-        cveroot = et.SubElement(root, 'cve')
-        for element in CVEORDER:
-            if element == 'refs':
-                cveroot.append(cve.refs)
-            else:
-                node = et.SubElement(cveroot, element)
-                node.text = getattr(cve, element)
-
-    write_xml(root, filename)
-
-
-def write_xml(root, filename):
-    'Write root to a xml file'
-
-    with open(filename, 'w') as xmlout:
-        __indent__(root)
-        doc = et.ElementTree(root)
-        doc.write(xmlout, encoding='utf-8')
-
-
-def __indent__(node, level=0):
-    'Indents xml layout for printing'
-
-    i = '\n' + level * ' ' * 4
-    if len(node):
-        if not node.text or not node.text.strip():
-            node.text = i + ' ' * 4
-        if not node.tail or not node.tail.strip():
-            node.tail = i
-        for node in node:
-            __indent__(node, level + 1)
-        if not node.tail or not node.tail.strip():
-            node.tail = i
-    else:
-        if level and (not node.tail or not node.tail.strip()):
-            node.tail = i
-
-
-def interval_from_whiteboard(whiteboard):
-    'Returns a list of intervals within a whiteboard string'
-
-    expand = False #TODO
-    upper_inc = None
-    upper = None
-    lower_inc = None
-    lower = None
-
-    affected = list()
-
-    while len(whiteboard.strip()) > 0:
-        match = REGEX['wb_match'].match(whiteboard)
-        if not match:
-            raise InvalidWhiteboardError(whiteboard)
-
-        name = match.group(1)
-        exp = match.group(2)
-        comp1 = match.group(3)
-        vers1 = match.group(4)
-        comp2 = match.group(5)
-        vers2 = match.group(6)
-
-        if exp == '+':
-            expand = True
-
-        if comp1 == '=' or comp1 == '==':
-            lower_inc = True
-            upper_inc = True
-            lower = vers1
-            upper = vers1
-
-            if not REGEX['wb_version'].match(vers1):
-                raise InvalidWhiteboardError(whiteboard)
-        else:
-            for (char, version) in ((comp1, vers1), (comp2, vers2)):
-                if char == '<':
-                    upper_inc = False
-                    upper = version
-                elif char == '<=' or char == '=<':
-                    upper_inc = True
-                    upper = version
-                elif char == '>':
-                    lower_inc = False
-                    lower = version
-                elif char == '>=' or char == '=>':
-                    lower_inc = True
-                    lower = version
-                elif char:
-                    raise InvalidWhiteboardError(whiteboard)
-
-                if version and not REGEX['wb_version'].match(version):
-                    raise InvalidWhiteboardError(whiteboard)
-
-        affected.append(Interval(name, lower, upper, lower_inc,
-                                 upper_inc, expand))
-
-        whiteboard  = match.group(7)
-
-    return affected
-
-
 #TODO Use Exceptions
 def extract_version(release):
     'Extracts revision, source and version out of a release tag'
@@ -987,82 +638,3 @@ def all_version(source):
 
     return versions
 
-
-#TODO Remove BUG_ON; use Exceptions
-def receive_file(directory, path, xml_file,
-                 max_age = datetime.timedelta(0, 59*60)):
-    'Generic download function'
-
-    filename = os.path.join(directory, xml_file)
-
-    if not FORCE:
-        if os.path.exists(filename):
-            age = datetime.datetime.now() - \
-                  datetime.datetime.fromtimestamp(os.path.getmtime(filename))
-            if age < max_age:
-                BUG_ON('File %s - %sKB is recent enough [%s]' %
-                      (filename, os.path.getsize(filename)/1024,
-                      str(age)[:-7]))
-                return
-    try:
-        with closing(cStringIO.StringIO()) as data:
-            with closing(urllib.urlopen(path + xml_file)) as resource:
-                data.write(resource.read())
-
-            with open(filename, 'w') as output:
-                output.write(data.getvalue())
-
-    except IOError:
-        BUG_ON('Download failed!') #FIXME Handle exception
-
-    BUG_ON('File %s - %sKB received' %
-          (filename, os.path.getsize(filename)/1024))
-
-
-def receive_nvd_recent(directory):
-    'Download the latest CVEs file from the National Vulnerability Database'
-
-    url = 'http://nvd.nist.gov/download/'
-
-    receive_file(directory, url, 'nvdcve-recent.xml')
-
-
-def receive_nvd_all(directory):
-    'Download all earlier CVEs files from the National Vulnerability Database'
-
-    url = 'http://nvd.nist.gov/download/'
-    year = datetime.datetime.now().year
-
-    if year < 2002 or year > 2020:
-        year = 2020
-
-    for i in xrange(2002, year + 1):
-        receive_file(directory, url, 'nvdcve-%s.xml' % str(i),
-                     max_age = datetime.timedelta(1))
-
-
-def receive_bugzilla_list(directory):
-    'Download a list containing all Bugzilla kernel bugs'
-
-    status = ['NEW', 'ASSIGNED', 'REOPENED', 'RESOLVED', 'VERIFIED', 'CLOSED']
-    resolution = ['FIXED', 'LATER', 'TEST-REQUEST', 'UPSTREAM', '---']
-
-    url = ['https://bugs.gentoo.org/buglist.cgi?',
-            'query_format=advanced&component=Kernel']
-
-    for i in status:
-        url.append('&bug_status=' + i)
-    for i in resolution:
-        url.append('&resolution=' + i)
-    url.append('#')
-
-    receive_file(directory, ''.join(url), 'bugzilla.xml')
-
-
-def receive_bugzilla_bug(directory, bugid):
-    'Download the xml file of a particular Bugzilla kernel bug'
-
-    url = 'https://bugs.gentoo.org/show_bug.cgi?ctype=xml&id='
-
-    receive_file(directory, url, bugid)
-
diff --git a/tools/cron.py b/tools/cron.py
new file mode 100755
index 0000000..01e04af
--- /dev/null
+++ b/tools/cron.py
@@ -0,0 +1,405 @@
+#!/usr/bin/env python
+# kernel-check -- Kernel security information
+# Copyright 2009-2009 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from contextlib import closing
+import xml.etree.cElementTree as et
+import cStringIO
+import datetime
+import logging
+import mmap
+import os
+import portage
+import re
+import sys
+import time
+import urllib
+
+
+class CronError(Exception):
+    def __init__(self, value):
+        self.value = value
+
+NOCVE = 'GENERIC-MAP-NOMATCH'
+NOCVEDESC = 'This GENERIC identifier is not specific to any vulnerability. '\
+            'GENERIC-MAP-NOMATCH is used by products, databases, and ' \
+            'services to specify when a particular vulnerability element ' \
+            'does not map to a corresponding CVE entry.'
+DELAY = 0.2
+SKIP = True
+MINYEAR = 2002
+MAXYEAR = 2020
+NVDURL = 'http://nvd.nist.gov/'
+BZURL = 'https://bugs.gentoo.org/'
+STATE = ['NEW', 'ASSIGNED', 'REOPENED', 'RESOLVED', 'VERIFIED', 'CLOSED']
+RESOLUTION = ['FIXED', 'LATER', 'TEST-REQUEST', 'UPSTREAM', '---']
+BUGORDER = ['bugid', 'reporter', 'reported', 'status', 'arch', 'affected']
+CVEORDER = ['cve', 'published', 'desc', 'severity', 'vector', 'score', 'refs']
+FILEPATH = os.path.dirname(os.path.realpath(__file__))
+PORTDIR = portage.settings['PORTDIR']
+LOGFILE = None #os.path.join(FILEPATH, 'cron.log')
+DIR = {
+    'tmp' : os.path.join(FILEPATH, 'tmp'),
+    'out' : os.path.join(PORTDIR, 'metadata', 'kernel'),
+    'bug' : os.path.join(FILEPATH, 'tmp', 'bug'),
+    'nvd' : os.path.join(FILEPATH, 'tmp', 'nvd')
+}
+REGEX = {
+    'bugzilla'   : re.compile(r'(?<=bug.cgi\?id=)\d*'),
+    'grp_all'    : re.compile(r'(?<=\()[ (]*CVE-(\d{4})' \
+                              r'([-,(){}|, \d]+)(?=\))'),
+    'm_nomatch'  : re.compile(r'.*GENERIC-MAP-NOMATCH.*'),
+    'grp_split'  : re.compile(r'(?<=\D)(\d{4})(?=\D|$)'),
+    'wb_match'   : re.compile(r'\s*\[\s*([^ +<=>]+)\s*([' \
+                              r'<=>]{1,2})\s*([^ <=>\]]+' \
+                              r')\s*(?:([<=>]{1,2})\s*([' \
+                              r'^ \]]+))?\s*\]\s*(.*)'),
+    'wb_version' : re.compile(r'^(?:\d{1,2}\.){0,3}\d{1,2}' \
+                              r'(?:[-_](?:r|rc)?\d{1,2})*$')
+}
+CVES = dict()
+logging.basicConfig(format='[%(asctime)s] %(levelname)-6s : %(message)s',
+                    datefmt='%H:%M:%S', filename=LOGFILE, level=logging.DEBUG)
+
+
+def main(argv):
+    'Main function'
+
+    logging.info('Running cron...')
+
+    current_year = datetime.datetime.now().year
+    if current_year < MINYEAR or current_year > MAXYEAR:
+        current_year = MAXYEAR
+
+    for directory in DIR:
+        if not os.path.isdir(DIR[directory]):
+            os.makedirs(DIR[directory])
+
+    logging.info('Receiving the latest xml file from the nvd')
+
+    receive_file(DIR['nvd'], [NVDURL, 'download/'],'nvdcve-recent.xml')
+
+    if not SKIP:
+        logging.info('Receiving earlier xml files from the nvd')
+
+        for year in xrange(MINYEAR, current_year + 1):
+            receive_file(DIR['nvd'], [NVDURL, 'download/'],
+                         'nvdcve-%s.xml' % str(year))
+
+    logging.info('Receiving the kernel vulnerability list from bugzilla')
+
+    url = [BZURL, 'buglist.cgi?query_format=advanced&component=Kernel']
+
+    for item in STATE:
+        url.append('&bug_status=' + item)
+    for item in RESOLUTION:
+        url.append('&resolution=' + item)
+    url.append('#')
+
+    receive_file(DIR['tmp'], url, 'bugzilla.xml')
+
+    filename = os.path.join(DIR['tmp'], 'bugzilla.xml')
+    with open(filename, 'r+') as buglist_file:
+        memory_map = mmap.mmap(buglist_file.fileno(), 0)
+        buglist = REGEX['bugzilla'].findall(memory_map.read(-1))
+
+    logging.info('Found %i kernel vulnerabilities' % len(buglist))
+
+    logging.info('Creating the nvd dictionary')
+    nvd_dict = parse_nvd_dict(DIR['nvd'])
+
+    logging.info('Creating the xml files')
+
+    created_files = 0
+    for item in buglist:
+        try:
+            receive_file(DIR['bug'], [BZURL, 'show_bug.cgi?ctype=xml&id='],
+                         item)
+
+            vul = parse_bugzilla_dict(DIR['bug'], item)
+
+            for cve in vul['cvelist']:
+                if cve == NOCVE:
+                    vul['cves'] = [NOCVE]
+                    break; #TODO
+                else:
+                    try:
+                        vul['cves'].append(nvd_dict[cve])
+                    except KeyError:
+                        raise CronError('No Nvd entry: ' + cve)
+
+            write_cve_file(DIR['out'], vul)
+            created_files += 1
+            time.sleep(DELAY)
+
+        except CronError, e:
+            logging.error('[%s] %s' % (item, e.value))
+
+    logging.info('Created %i xml files' % created_files)
+
+
+def receive_file(directory, url, xml_file):
+    'Generic download function'
+
+    filename = os.path.join(directory, xml_file)
+    url.append(xml_file)
+
+    try:
+        with closing(cStringIO.StringIO()) as data:
+            with closing(urllib.urlopen(''.join(url))) as resource:
+                data.write(resource.read())
+
+            with open(filename, 'w') as output:
+                output.write(data.getvalue())
+
+    except IOError:
+        logging.error('File %s - Download failed!' % filename)
+
+    logging.debug('File %s - %sKB received' %
+          (filename, os.path.getsize(filename)/1024))
+
+
+def parse_nvd_dict(directory):
+    'Returns a dictionary from the National Vulnerability Database'
+
+    nvd = dict()
+
+    for nvdfile in os.listdir(directory):
+        filename = os.path.join(directory, nvdfile)
+        try:
+            with open(filename, 'r+') as xml_data:
+                memory_map = mmap.mmap(xml_data.fileno(), 0)
+
+        except SyntaxError:
+            continue
+
+        root = et.parse(memory_map).getroot()
+        namespace = root.tag[:-3]
+
+        for tree in root:
+            cve = {
+                'cve'      : tree.get('name'),
+                'published' : tree.get('published'),
+                'severity'  : tree.get('severity'),
+                'vector'    : tree.get('CVSS_vector'),
+                'score'     : tree.get('CVSS_score')
+            }
+
+            desc = tree.find('%sdesc/%sdescript/' % (namespace, namespace))
+            if desc is not None:
+                cve['desc'] = desc.text
+
+            reftree = tree.find(namespace + 'refs')
+            reftree.tag = reftree.tag.replace(namespace, '')
+            for elem in reftree.findall('.//*'):
+                elem.tag = elem.tag.replace(namespace, '')
+
+            bugref = et.SubElement(reftree, 'ref')
+            bugref.set('source', 'GENTOO')
+            bugref.set('url', '%sshow_bug.cgi?id=%s' % (BZURL, cve['cve']))
+            bugref.text = 'Gentoo %s' % cve['cve']
+
+            cve['refs'] = reftree
+
+            nvd[cve['cve']] = cve
+
+    return nvd
+
+def parse_bugzilla_dict(directory, bugid):
+    'Returns a vulnerability class containing information about a bug'
+
+    filename = os.path.join(directory, bugid)
+
+    try:
+        with open(filename, 'r+') as xml_data:
+            memory_map = mmap.mmap(xml_data.fileno(), 0)
+            root = et.parse(memory_map).getroot()[0]
+
+    except IOError:
+        return
+
+    string = str()
+
+    try:
+        string = root.find('short_desc').text
+    except AttributeError:
+        CronError('No Cve')
+
+    try:
+        cvelist = list()
+        string = string.replace('CAN', 'CVE')
+
+        if string in REGEX['m_nomatch'].findall(string):
+            cvelist = [NOCVE]
+
+        for (year, split_cves) in REGEX['grp_all'].findall(string):
+            for cve in REGEX['grp_split'].findall(split_cves):
+                cvelist.append('CVE-%s-%s' % (year, cve))
+
+        vul = {
+            'bugid'    : bugid,
+            'cvelist'  : cvelist,
+            'cves'     : list(),
+            'arch'     : root.find('rep_platform').text.lower(),
+            'reporter' : root.find('reporter').text.lower(),
+            'reported' : root.find('creation_ts').text,
+            'status'   : root.find('bug_status').text.lower(),
+        }
+
+        for item in vul['cvelist']:
+            if item != NOCVE:
+                if item not in CVES:
+                    CVES[item] = vul.bugid
+                else:
+                    raise CronError('Duplicate: ' + CVES[item])
+
+    except AttributeError:
+        pass
+
+    try:
+        wb = root.find('status_whiteboard').text;
+        vul['affected'] = interval_from_whiteboard(wb)
+
+        if vul['affected'] == None:
+            raise CronError('Invalid whiteboard: ' + wb)
+
+    except AttributeError:
+        raise CronError('Empty whiteboard')
+
+    return vul
+
+
+def interval_from_whiteboard(whiteboard):
+    'Returns a list of intervals within a whiteboard string'
+
+    upper_inc = None
+    upper = None
+    lower_inc = None
+    lower = None
+
+    affected = list()
+
+    while len(whiteboard.strip()) > 0:
+        match = REGEX['wb_match'].match(whiteboard)
+        if not match:
+            return None
+
+        name = match.group(1)
+        comp1 = match.group(2)
+        vers1 = match.group(3)
+        comp2 = match.group(4)
+        vers2 = match.group(5)
+        whiteboard = match.group(6)
+
+        if comp1 == '=' or comp1 == '==':
+            lower_inc = True
+            upper_inc = True
+            lower = vers1
+            upper = vers1
+
+            if not REGEX['wb_version'].match(vers1):
+                return None
+        else:
+            for (char, version) in ((comp1, vers1), (comp2, vers2)):
+
+                if char == '<':
+                    upper_inc = False
+                    upper = version
+                elif char == '<=' or char == '=<':
+                    upper_inc = True
+                    upper = version
+                elif char == '>':
+                    lower_inc = False
+                    lower = version
+                elif char == '>=' or char == '=>':
+                    lower_inc = True
+                    lower = version
+                elif char:
+                    return None
+
+                if version and not REGEX['wb_version'].match(version):
+                    return None
+
+        interval = {
+            'name'      : name,
+            'lower'     : lower,
+            'upper'     : upper,
+            'lower_inc' : lower_inc,
+            'upper_inc' : upper_inc
+        }
+
+        affected.append(interval)
+
+    return affected
+
+
+def write_cve_file(directory, vul):
+    'Write a bug file containing all important information for kernel-check'
+
+    filename = os.path.join(directory, vul['bugid'] + '.xml')
+
+    root = et.Element('vulnerability')
+    bugroot = et.SubElement(root, 'bug')
+
+    for element in BUGORDER:
+        if element == 'affected':
+                affectedroot = et.SubElement(bugroot, 'affected')
+                for item in vul['affected']:
+                        intnode = et.Element('interval')
+                        intnode.set('source', item['name'])
+
+                        affectedroot.append(intnode)
+
+                        for i in ('lower', 'upper'):
+                            if item[i]:
+                                node = et.SubElement(intnode, i)
+                                node.text = item[i]
+                                node.set('inclusive',
+                                str(item[i + '_inc']).lower())
+        else:
+            node = et.SubElement(bugroot, element)
+            node.text = vul[element]
+
+    for cve in vul['cves']:
+        cveroot = et.SubElement(root, 'cve')
+        if cve == NOCVE:
+            node = et.SubElement(cveroot, 'cve')
+            node.text = NOCVE
+            node = et.SubElement(cveroot, 'desc')
+            node.text = NOCVEDESC
+        else:
+            for element in CVEORDER:
+                if element == 'refs':
+                    cveroot.append(cve[element])
+                else:
+                    node = et.SubElement(cveroot, element)
+                    node.text = cve[element]
+
+    with open(filename, 'w') as xmlout:
+        __indent__(root)
+        doc = et.ElementTree(root)
+        doc.write(xmlout, encoding='utf-8')
+
+
+def __indent__(node, level=0):
+    'Indents xml layout for printing'
+
+    i = '\n' + level * ' ' * 4
+    if len(node):
+        if not node.text or not node.text.strip():
+            node.text = i + ' ' * 4
+        if not node.tail or not node.tail.strip():
+            node.tail = i
+        for node in node:
+            __indent__(node, level + 1)
+        if not node.tail or not node.tail.strip():
+            node.tail = i
+    else:
+        if level and (not node.tail or not node.tail.strip()):
+            node.tail = i
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
+
diff --git a/findcommit.sh b/tools/findcommit.sh
index 88c5857..770078b 100755
--- a/findcommit.sh
+++ b/tools/findcommit.sh
@@ -47,11 +47,11 @@ for B in $BRANCHES ; do
                     WHITEBOARD="$WHITEBOARD [linux <${VERSION}]"
             else
                 if [ "${LASTFIXEDBRANCH}" == "${VERSION/-rc*/}" ] ; then
-                    WHITEBOARD="${WHITEBOARD} ${BAD}[linux >=
-                                ${LASTFIXEDBRANCH} <${VERSION}]${NORMAL}"
+                    WHITEBOARD="${WHITEBOARD} ${BAD}[linux >=" \
+                               "${LASTFIXEDBRANCH} <${VERSION}]${NORMAL}"
                 else
-                    WHITEBOARD="${WHITEBOARD} [linux >=
-                                ${LASTFIXEDBRANCH} <${VERSION}]"
+                    WHITEBOARD="${WHITEBOARD} [linux >=" \
+                               "${LASTFIXEDBRANCH} <${VERSION}]"
                 fi
             fi
             LASTFIXEDBRANCH=$NEXTBRANCH
diff --git a/testsuite.py b/tools/testsuite.py
index 1a9287c..1a9287c 100755
--- a/testsuite.py
+++ b/tools/testsuite.py