added checksum support to package database fetch

4 files changed, 61 insertions, 11 deletions

diff --git a/contrib/manage_pkgdb.py b/contrib/manage_pkgdb.py
index f1e0583..5d3ba88 100755
--- a/contrib/manage_pkgdb.py
+++ b/contrib/manage_pkgdb.py
@@ -15,6 +15,7 @@ from __future__ import print_function
 
 import datetime
 import feedparser
+import json
 import optparse
 import os
 import re
@@ -32,6 +33,8 @@ if os.path.exists(os.path.join(current_dir, '..', 'g_octave')):
     sys.path.insert(0, os.path.join(current_dir, '..'))
 
 from g_octave import description_tree
+from g_octave.checksum import sha1_compute
+
 
 class Git:
 
@@ -63,6 +66,25 @@ class SfUpdates:
         if self.feed.bozo == 1:
             raise self.feed.bozo_exception
         self.entries = self.feed.entries
+        self._db = description_tree.DescriptionTree()
+
+    def _compute_manifest(self, db_dir):
+        manifest = {}
+        for category in self._db.pkg_list:
+            for pkg in self._db.pkg_list[category]:
+                description = os.path.join(
+                    db_dir,
+                    'octave-forge',
+                    self._db.categories[pkg['name']],
+                    pkg['name'],
+                    '%s-%s.DESCRIPTION' % (pkg['name'], pkg['version'])
+                )
+                manifest[pkg['name']+'-'+pkg['version']] = sha1_compute(description)
+        try:
+            with open(os.path.join(self._repo_dir, 'manifest.json'), 'w') as fp:
+                json.dump(manifest, fp, indent=2, sort_keys=True)
+        except:
+            pass
 
     def _save_timestamp(self):
         try:
@@ -109,14 +131,13 @@ class SfUpdates:
         #   version,
         #   category
         # }
-        db = description_tree.DescriptionTree()
         entries = {}
-        for category in db.pkg_list:
-            for pkg in db.pkg_list[category]:
+        for category in self._db.pkg_list:
+            for pkg in self._db.pkg_list[category]:
                 entries['%s-%s.tar.gz' % (pkg['name'], pkg['version'])] = {
                     'name': pkg['name'],
                     'version': pkg['version'],
-                    'category': unicode(db.categories[pkg['name']]),
+                    'category': unicode(self._db.categories[pkg['name']]),
                 }
         return entries
 
@@ -194,6 +215,7 @@ class SfUpdates:
                     with closing(src_tar.extractfile(f)) as fp_tar:
                         with open(description, 'w') as fp:
                             shutil.copyfileobj(fp_tar, fp)
+        self._compute_manifest(db_dir)
         self._save_timestamp()
 
 
diff --git a/g_octave/checksum.py b/g_octave/checksum.py
index 79fe969..c5ead44 100644
--- a/g_octave/checksum.py
+++ b/g_octave/checksum.py
@@ -5,7 +5,7 @@
     ~~~~~~~~~~~
     
     This module implements functions for compute/generate SHA1 checksums
-    for files.
+    for DESCRIPTION files.
     
     :copyright: (c) 2010 by Rafael Goncalves Martins
     :license: GPL-2, see LICENSE for more details.
@@ -16,15 +16,34 @@ from __future__ import absolute_import
 __all__ = [
     'sha1_compute',
     'sha1_check',
+    'sha1_check_db',
 ]
 
-from .compat import open
 from hashlib import sha1
+import json
+import os
+
+from .config import Config
+config = Config(True)
 
 
 def sha1_compute(filename):
     with open(filename) as fp:
         return sha1(fp.read()).hexdigest()
 
-def sha1_check(filename, checksum):
-    return sha1_compute(filename) == checksum
+def sha1_check(db, p):
+    description = db[p]
+    manifest = {}
+    with open(os.path.join(config.db, 'manifest.json')) as fp:
+        manifest = json.load(fp)
+    if p not in manifest:
+        return False
+    return manifest[p] == description.sha1sum()
+
+def sha1_check_db(db):
+    for cat in db.pkg_list:
+        for pkg in db.pkg_list[cat]:
+            p = pkg['name']+'-'+pkg['version']
+            if not sha1_check(db, p):
+                return False
+    return True
diff --git a/g_octave/fetch.py b/g_octave/fetch.py
index b8a8d6c..a0d7d5a 100644
--- a/g_octave/fetch.py
+++ b/g_octave/fetch.py
@@ -20,6 +20,7 @@ __all__ = ['fetch']
 from .config import Config
 conf = Config(True) # fetch phase
 
+from .description_tree import DescriptionTree
 from .exception import FetchException
 from .compat import py3k, open as open_
 
@@ -40,7 +41,7 @@ import tarfile
 from contextlib import closing
 
 def clean_db():
-    for f in ['timestamp', 'info.json', 'patches', 'octave-forge']:
+    for f in ['timestamp', 'info.json', 'patches', 'octave-forge', 'manifest.json']:
         current = os.path.join(conf.db, f)
         if os.path.isdir(current):
             shutil.rmtree(current)
diff --git a/scripts/g-octave b/scripts/g-octave
index 69ac115..89fb1d1 100755
--- a/scripts/g-octave
+++ b/scripts/g-octave
@@ -48,6 +48,8 @@ current_dir = os.path.dirname(os.path.realpath(__file__))
 if os.path.exists(os.path.join(current_dir, '..', 'g_octave')):
     sys.path.insert(0, os.path.join(current_dir, '..'))
 
+from g_octave.checksum import sha1_check_db
+from g_octave.description_tree import DescriptionTree
 from g_octave.log import Log
 log = Log('g-octave')
 
@@ -281,7 +283,14 @@ def main():
                 log.info('No updates available')
                 out.einfo('No updates available')
             updates.extract()
-
+            
+            log.info('Checking SHA1 checksums ...')
+            out.ebegin('Checking SHA1 checksums ...')
+            if sha1_check_db(DescriptionTree()):
+                out.eend(0)
+            else:
+                out.eend(1)
+            
             return os.EX_OK
     else:
         log.info('You can\'t fetch package databases.')
@@ -293,7 +302,6 @@ def main():
     conf = Config()
 
     from g_octave.description import Description
-    from g_octave.description_tree import DescriptionTree
     from g_octave.ebuild import Ebuild, EbuildException
     from g_octave.overlay import create_overlay