aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorAlexander Bersenev <bay@hackerdom.ru>2011-07-19 22:44:25 +0000
committerAlexander Bersenev <bay@hackerdom.ru>2011-07-19 22:44:25 +0000
commit5460a8dbba7f775c974de886399b90a83e800805 (patch)
tree7409aea435dc798fbb723615569ad78f7efffdc4 /src
parenttests (diff)
downloadautodep-5460a8dbba7f775c974de886399b90a83e800805.tar.gz
autodep-5460a8dbba7f775c974de886399b90a83e800805.tar.bz2
autodep-5460a8dbba7f775c974de886399b90a83e800805.zip
more work on pre-dependency analisys
Diffstat (limited to 'src')
-rw-r--r--src/autodep/__init__.py0
-rw-r--r--src/autodep/package_utils/__init__.py0
-rw-r--r--src/autodep/package_utils/portage_log_parser.py77
-rw-r--r--src/autodep/package_utils/portage_misc_functions.py84
-rw-r--r--src/autodep/package_utils/portage_utils.py (renamed from src/autodep/logfs/portage_utils.py)2
-rwxr-xr-xsrc/autodep/showfsevents.py123
6 files changed, 247 insertions, 39 deletions
diff --git a/src/autodep/__init__.py b/src/autodep/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/autodep/__init__.py
diff --git a/src/autodep/package_utils/__init__.py b/src/autodep/package_utils/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/autodep/package_utils/__init__.py
diff --git a/src/autodep/package_utils/portage_log_parser.py b/src/autodep/package_utils/portage_log_parser.py
new file mode 100644
index 0000000..3854996
--- /dev/null
+++ b/src/autodep/package_utils/portage_log_parser.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python2
+# parse log and get packages actually merged
+
+import re
+import time
+
+# the log path seems to be always on that path
+# see <portage_lib_path>/pym/_emerge/emergelog.py
+log_path='/var/log/emerge.log'
+
+def get_list_of_merged_packages(starttime=0,endtime=-1):
+ ret=[]
+ try:
+ log=open(log_path)
+
+ found_begining=False
+ current_package=''
+ current_package_num=0
+ total_packages_num=0
+
+ expect_start=True
+ expect_end=False
+
+ for line in log:
+ if ':' not in line: # skipping bad strings
+ continue
+
+ msgtime,msgtext=line.split(':',1)
+ msgtime,msgtext=int(msgtime),msgtext.strip()
+ if msgtime<starttime:
+ continue
+ if endtime!=-1 and msgtime>endtime:
+ continue
+
+ if msgtext.startswith("Started emerge on: "):
+ # doing an additional check: we have msg like:
+ # 1310909507: Started emerge on: Jul 17, 2011 13:31:47
+ # we want to make sure that two variants of time is not
+ # distinguish more than on 2 days(local timezone may be changed)
+ # we protect self from malformed and broken log files
+ msg, date = msgtext.split(": ",1)
+ msgtime2=time.mktime(time.strptime(date,"%b %d, %Y %H:%M:%S"))
+ if abs(msgtime-msgtime2)>60*60*24*2:
+ continue
+ found_begining=True
+ if not found_begining:
+ continue
+
+ if expect_start and msgtext.startswith(">>> emerge "):
+ # string example: >>> emerge (1 of 1) sys-process/lsof-4.84 to /
+ m=re.search(r">>> emerge \((\d+) of (\d+)\) (\S+) to",msgtext)
+ if m:
+ pkgnum,total_pkgnum,pkgname=m.group(1),m.group(2), m.group(3)
+ if total_packages_num==0:
+ total_packages_num=total_pkgnum
+ elif total_packages_num!=total_pkgnum:
+ continue
+ current_package_num=pkgnum
+ current_package=pkgname
+ expect_start=False
+ expect_end=True
+ elif expect_end and msgtext.startswith(
+ "::: completed emerge (" + current_package_num + " of " +
+ total_packages_num + ") " + current_package):
+ ret.append(current_package)
+ if total_packages_num==current_package_num:
+ break
+ expect_start=True
+ expect_end=False
+
+ log.close()
+ return ret
+
+ except IOError,e:
+ print "Error while opening logfile. %s" % e
+ return []
+ \ No newline at end of file
diff --git a/src/autodep/package_utils/portage_misc_functions.py b/src/autodep/package_utils/portage_misc_functions.py
new file mode 100644
index 0000000..4d63de4
--- /dev/null
+++ b/src/autodep/package_utils/portage_misc_functions.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python2
+# Thanks to Zac Medico <zmedico@gentoo.org> for working example of using an api
+
+import portage
+from portage.dbapi._expand_new_virt import expand_new_virt
+
+# to not use own emerge option parser. Options may change but I hope
+# parse_opts function will always be there
+from _emerge.main import parse_opts
+
+
+class portage_api:
+ def __init__(self):
+ self.settings=portage.config(clone=portage.settings)
+ self.vartree=portage.db[portage.root]['vartree']
+ self.vardb=self.vartree.dbapi
+ self.portdb=portage.portdb
+ self.metadata_keys = [k for k in portage.auxdbkeys if not k.startswith("UNUSED_")]
+ self.use=self.settings["USE"]
+
+ # recursive dependency getter
+ def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]):
+ #pkg="kde-meta"
+ #print self.vardb.match("<sys-apps/paludis-0.26.0_alpha5")
+ #metadata = dict(zip(self.metadata_keys, self.vardb.aux_get(pkg, self.metadata_keys)))
+
+ ret=set()
+
+ pkg = self.portdb.xmatch("bestmatch-visible", pkg)
+ if not pkg:
+ return ret
+
+ #print pkg
+
+ known_packages=set()
+ unknown_packages={pkg}
+
+ while unknown_packages:
+ p=unknown_packages.pop()
+ #print "proceeding "+p
+ if p in known_packages:
+ continue
+ known_packages.add(p)
+
+ #print self.metadata_keys, p,self.portdb.aux_get(p, self.metadata_keys)
+ metadata = dict(zip(self.metadata_keys, self.vardb.aux_get(p, self.metadata_keys)))
+ #print "proceeding2 "+p
+
+ dep_str = " ".join(metadata[k] for k in dep_type)
+
+ success, atoms = portage.dep_check(dep_str, None, self.settings, myuse=self.use.split(),
+ trees=portage.db, myroot=self.settings["ROOT"])
+
+ #print atoms
+ if not success:
+ continue
+
+ for atom in atoms:
+ atomname = self.vartree.dep_bestmatch(atom)
+ #print atomname
+ if not atomname:
+ continue
+
+ for unvirt_pkg in expand_new_virt(self.vartree.dbapi,'='+atomname):
+ for pkg in self.vartree.dep_match(unvirt_pkg):
+ ret.add(pkg)
+ unknown_packages.add(pkg)
+ return ret
+
+ # returns all packages from system set. They are always implicit dependencies
+ def get_system_packages_list(self):
+ ret=[]
+ for atom in self.settings.packages:
+ for pre_pkg in self.vartree.dep_match(atom):
+ for unvirt_pkg in expand_new_virt(self.vartree.dbapi,'='+pre_pkg):
+ for pkg in self.vartree.dep_match(unvirt_pkg):
+ ret.append(pkg)
+ return ret
+
+ # call emerge arguments parser
+ def parse_emerge_args(self,args):
+ action, opts, files = parse_opts(args, silent=True)
+ return action, opts, files
+
diff --git a/src/autodep/logfs/portage_utils.py b/src/autodep/package_utils/portage_utils.py
index a03f6af..c98ce1d 100644
--- a/src/autodep/logfs/portage_utils.py
+++ b/src/autodep/package_utils/portage_utils.py
@@ -59,3 +59,5 @@ def getfilesbypackage(packagename):
return ret
+
+ \ No newline at end of file
diff --git a/src/autodep/showfsevents.py b/src/autodep/showfsevents.py
index 68f8425..960d038 100755
--- a/src/autodep/showfsevents.py
+++ b/src/autodep/showfsevents.py
@@ -4,20 +4,41 @@ import optparse
import os
import sys
+import time
import logfs.fstracer
-import logfs.portage_utils
+import package_utils.portage_utils
+import package_utils.portage_misc_functions
+import package_utils.portage_log_parser
+
+portage_api=package_utils.portage_misc_functions.portage_api()
+portage_api=package_utils.portage_misc_functions.portage_api()
+
+runtime_vars={} # This is here mainly for grouping. We are trying to
+ # get as much data about an environment as possible
+runtime_vars["starttime"]=int(time.time())
+#print package_utils.portage_log_parser.get_list_of_merged_packages(1244256830)
+
+#quit(1)
+
+
+#system_packages = deps_finder.get_system_packages_list()
+#print "sys-libs/glibc-2.13-r2" in system_packages
+#print deps_finder.get_deps('bash')
+
+#print(runtime_vars["starttime"])
+#quit(1)
args_parser=optparse.OptionParser("%prog [options] <command>")
args_parser.add_option("-b", "--block",action="store", type="string",
dest="packages", default="", help="block an access to files from this packages")
-args_parser.add_option("-u", "--unknown", action="store_true", dest="show_unknown",
- default=False, help="show unknown stage and files from unknown package")
-args_parser.add_option("-v", action="store_true", dest="verbose",
- default=False, help="show accessed files")
-args_parser.add_option("-n","--notfound", action="store_true", dest="show_notfound",
- default=False, help="show not founded files")
-
+args_parser.add_option("-f","--files", action="store_true", dest="show_files",
+ default=False, help="show accessed files and not founded files")
+args_parser.add_option("-v","--verbose", action="store_true", dest="verbose",
+ default=False, help="show non-important packages, "
+ "show unknown package and unknown stage")
+args_parser.add_option("-C","--nocolor",action="store_true", dest="nocolor",
+ default=False, help="don't output color")
args_parser.add_option("--hooklib",action="store_const", dest="approach",
const="hooklib", help="use ld_preload logging approach(default)")
@@ -26,9 +47,7 @@ args_parser.add_option("--fusefs",action="store_const", dest="approach",
args_parser.set_defaults(approach="hooklib")
args_parser.epilog="Example: %s -b lsof,cowsay emerge bash" % (os.path.basename(sys.argv[0]))
-
args_parser.disable_interspersed_args()
-
(options, args) = args_parser.parse_args()
#print options
#print args
@@ -37,6 +56,16 @@ if len(args)==0:
args_parser.print_help()
exit(1)
+if args[0]=="emerge":
+ runtime_vars["is_emerge"]=True
+ emergeaction ,emergeopts, emergefiles=portage_api.parse_emerge_args(args[1:])
+ runtime_vars["emerge_parameters"]=(emergeaction ,emergeopts, emergefiles)
+ if len(emergefiles)>1:
+ print "Please, install packages one by one to get more accurate reports"
+else:
+ runtime_vars["is_emerge"]=False
+
+
filter_function=lambda eventname,filename,stage: True
# handling --block
@@ -44,7 +73,7 @@ if options.packages:
packages=options.packages.split(",")
files_to_block=[]
for package in packages:
- files_in_package=logfs.portage_utils.getfilesbypackage(package)
+ files_in_package=package_utils.portage_utils.getfilesbypackage(package)
if len(files_in_package)==0:
print "Bad package name: %s. Exiting" % package
exit(1)
@@ -57,6 +86,29 @@ if options.packages:
events=logfs.fstracer.getfsevents(args[0], args,approach=options.approach,filterproc=filter_function)
print "Program finished, analyzing dependencies"
+runtime_vars["endtime"]=int(time.time())
+
+if runtime_vars["is_emerge"]:
+ # try to get information about packages merged sucessfully
+ try:
+ pkgs=package_utils.portage_log_parser.get_list_of_merged_packages(
+ runtime_vars["starttime"],runtime_vars["endtime"]
+ )
+ if len(pkgs) > 1:
+ print "Several packages were installed. The report will be inaccurate"
+ runtime_vars["pkgs_installed"]=pkgs
+ runtime_vars["deps_buildtime"]=[]
+ runtime_vars["deps_all"]=[]
+ for pkg in pkgs:
+ runtime_vars["deps_buildtime"]+=portage_api.get_deps(pkg,["DEPEND"])
+ runtime_vars["deps_all"]+=portage_api.get_deps(pkg,["DEPEND","RDEPEND"])
+
+ print runtime_vars["deps_buildtime"]
+ print runtime_vars["deps_all"]
+ except:
+ print "Non-critical error while parsing logfile of emerge"
+ runtime_vars["is_emerge"]=False # shutting down all emerge handling logic
+ pass
# get unique filenames
filenames={}
@@ -70,19 +122,17 @@ for stage in events:
filenames=filenames.keys();
# temporary disabled
-file_to_package=logfs.portage_utils.getpackagesbyfiles(filenames)
+file_to_package=package_utils.portage_utils.getpackagesbyfiles(filenames)
#file_to_package={}
#print events
-# this part is completly unreadable. It converting one complex struct(returned by getfsevents) to
-# another complex struct which good for generating output
-# old struct is also used during output
+# This part is completly unreadable.
+# It converting one complex struct(returned by getfsevents) to another complex
+# struct which good for generating output.
+#
+# Old struct is also used during output
-#events_converted_for_output={}
packagesinfo={}
-#events_converted_for_output=packagesinfo
-#otherfilesinfo={}
-#events_converted_for_output["otherfilesinfo"]=otherfilesinfo
for stage in sorted(events):
succ_events=events[stage][0]
@@ -99,10 +149,6 @@ for stage in sorted(events):
stageinfo=packagesinfo[package]
if not stage in stageinfo:
stageinfo[stage]={}
-# else:
-# stageinfo=otherfilesinfo
-# if not stage in stageinfo:
-# stageinfo[stage]={}
filesinfo=stageinfo[stage]
if not filename in filesinfo:
@@ -119,10 +165,6 @@ for stage in sorted(events):
stageinfo=packagesinfo[package]
if not stage in stageinfo:
stageinfo[stage]={}
- #else:
- # stageinfo=otherfilesinfo
- # if not stage in stageinfo:
- # stageinfo[stage]={}
filesinfo=stageinfo[stage]
if not filename in filesinfo:
@@ -131,35 +173,38 @@ for stage in sorted(events):
#print events_converted_for_output
-# explicit check for launching with non-emerge application
-was_emerge_process=False
-for package in packagesinfo:
- if len(packagesinfo[package].keys())>1:
- was_emerge_process=True
- break
-
# generating output
stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7,
"install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13}
+deps_finder=package_utils.portage_misc_functions.portage_api()
+system_packages = deps_finder.get_system_packages_list()
+
# print information grouped by package
for package in sorted(packagesinfo):
# not showing special directory package
if package=="directory":
continue
- if package=="unknown" and not options.show_unknown:
+ if package=="unknown" and not options.verbose:
continue
+
+ if package in system_packages and not options.verbose:
+ continue
+
+ is_attention_pkg=runtime_vars["is_emerge"] and package not in runtime_vars["deps_all"]
+
stages=[]
for stage in sorted(packagesinfo[package].keys(), key=stagesorder.get):
- if stage!="unknown" or options.show_unknown or not was_emerge_process:
+ if stage!="unknown" or options.verbose or not runtime_vars["is_emerge"]:
stages.append(stage)
if len(stages)!=0:
- print "%-40s: %s"%(package,stages)
+
+ print "%s %-40s: %s"%(is_attention_pkg,package,stages)
# show information about accessed files
- if options.verbose:
+ if options.show_files:
filenames={}
for stage in stages:
for filename in packagesinfo[package][stage]:
@@ -184,7 +229,7 @@ for package in sorted(packagesinfo):
print " %-56s %-21s" % (filename,action[event_info])
# print not founded files with stages
-if options.show_notfound:
+if options.show_files:
filenames={}
print "\nNot founded files:"
for stage in sorted(events, key=stagesorder.get):