Get Gentoo!
gentoo.org sites
gentoo.org Wiki Bugs Forums Packages
Planet Archives Sources
Infra Status
aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--pkg/portage/repository/category.go140
-rw-r--r--pkg/portage/repository/commit.go215
-rw-r--r--pkg/portage/repository/mask.go65
-rw-r--r--pkg/portage/repository/package.go160
-rw-r--r--pkg/portage/repository/use.go78
-rw-r--r--pkg/portage/repository/version.go123
-rw-r--r--pkg/portage/update.go110
-rw-r--r--pkg/portage/utils/git.go22
8 files changed, 461 insertions, 452 deletions
diff --git a/pkg/portage/repository/category.go b/pkg/portage/repository/category.go
index b0b1e95..d88009e 100644
--- a/pkg/portage/repository/category.go
+++ b/pkg/portage/repository/category.go
@@ -4,7 +4,6 @@ package repository
import (
"encoding/xml"
- "io/ioutil"
"os"
"regexp"
"soko/pkg/config"
@@ -21,95 +20,112 @@ func isCategory(path string) bool {
return isCategory
}
-// UpdateCategory updates the category in the database in case
-// the given path points to a category description
-func UpdateCategory(path string) {
-
- splittedLine := strings.Split(path, "\t")
+// UpdateCategories updates the categories in the database for each
+// given path that points to a category description
+func UpdateCategories(paths []string) {
+ deleted := map[string]*models.Category{}
+ modified := map[string]*models.Category{}
+
+ for _, path := range paths {
+ splittedLine := strings.Split(path, "\t")
+
+ if len(splittedLine) != 2 {
+ if len(splittedLine) == 1 && isCategory(path) {
+ if cat := updateModifiedCategory(path); cat != nil {
+ modified[cat.Name] = cat
+ }
+ }
+ continue
+ }
- if len(splittedLine) != 2 {
- if len(splittedLine) == 1 && isCategory(path) {
- updateModifiedCategory(path)
+ status := splittedLine[0]
+ changedFile := splittedLine[1]
+
+ if !isCategory(changedFile) {
+ continue
+ } else if status == "D" {
+ cat := updateDeletedCategory(changedFile)
+ deleted[cat.Name] = cat
+ } else if status == "A" || status == "M" {
+ if cat := updateModifiedCategory(changedFile); cat != nil {
+ modified[cat.Name] = cat
+ }
}
- return
}
- status := splittedLine[0]
- changedFile := splittedLine[1]
+ if len(deleted) > 0 {
+ rows := make([]*models.Category, 0, len(deleted))
+ for _, row := range deleted {
+ rows = append(rows, row)
+ }
+ res, err := database.DBCon.Model(&rows).Delete()
+ if err != nil {
+ logger.Error.Println("Error during deleting categories", err)
+ } else {
+ logger.Info.Println("Deleted", res.RowsAffected(), "categories")
+ }
+ }
- if isCategory(changedFile) && status == "D" {
- updateDeletedCategory(changedFile)
- } else if isCategory(changedFile) && (status == "A" || status == "M") {
- updateModifiedCategory(changedFile)
+ if len(modified) > 0 {
+ rows := make([]*models.Category, 0, len(modified))
+ for _, row := range modified {
+ rows = append(rows, row)
+ }
+ res, err := database.DBCon.Model(&rows).OnConflict("(name) DO UPDATE").Insert()
+ if err != nil {
+ logger.Error.Println("Error during updating categories", err)
+ } else {
+ logger.Info.Println("Updated", res.RowsAffected(), "categories")
+ }
}
}
// updateDeletedCategory deletes a category from the database
-func updateDeletedCategory(changedFile string) {
- splitted := strings.Split(changedFile, "/")
- id := splitted[0]
-
- category := &models.Category{Name: id}
- _, err := database.DBCon.Model(category).WherePK().Delete()
-
- if err != nil {
- logger.Error.Println("Error during deleting category " + id)
- logger.Error.Println(err)
- }
-
+func updateDeletedCategory(changedFile string) *models.Category {
+ name, _, _ := strings.Cut(changedFile, "/")
+ return &models.Category{Name: name}
}
// updateModifiedCategory adds a category to the database or
// updates it. To do so, it parses the metadata from metadata.xml
-func updateModifiedCategory(changedFile string) {
- splitted := strings.Split(changedFile, "/")
- id := splitted[0]
-
- catmetadata := GetCatMetadata(config.PortDir() + "/" + changedFile)
- description := ""
+func updateModifiedCategory(changedFile string) *models.Category {
+ name, _, _ := strings.Cut(changedFile, "/")
- for _, longdescription := range catmetadata.Longdescriptions {
- if longdescription.Lang == "en" {
- description = strings.TrimSpace(longdescription.Content)
- }
+ xmlFile, err := os.Open(config.PortDir() + "/" + changedFile)
+ if err != nil {
+ logger.Error.Println("Error during reading category metadata", err)
+ return nil
}
+ defer xmlFile.Close()
- category := &models.Category{
- Name: id,
- Description: description,
+ var catMetadata CatMetadata
+ err = xml.NewDecoder(xmlFile).Decode(&catMetadata)
+ if err != nil {
+ logger.Error.Println("Error during category", changedFile, "decoding", err)
+ return nil
}
- _, err := database.DBCon.Model(category).OnConflict("(name) DO UPDATE").Insert()
-
- if err != nil {
- logger.Error.Println("Error during updating category " + id)
- logger.Error.Println(err)
+ var description string
+ for _, longDescription := range catMetadata.LongDescriptions {
+ if longDescription.Lang == "en" || longDescription.Lang == "" {
+ description = strings.TrimSpace(longDescription.Content)
+ }
}
-}
-// GetCatMetadata reads and parses the category
-// metadata from the metadata.xml file
-func GetCatMetadata(path string) Catmetadata {
- xmlFile, err := os.Open(path)
- if err != nil {
- logger.Error.Println("Error during reading category metadata")
- logger.Error.Println(err)
+ return &models.Category{
+ Name: name,
+ Description: description,
}
- defer xmlFile.Close()
- byteValue, _ := ioutil.ReadAll(xmlFile)
- var catmetadata Catmetadata
- xml.Unmarshal(byteValue, &catmetadata)
- return catmetadata
}
// Descriptions of the category metadata.xml format
-type Catmetadata struct {
+type CatMetadata struct {
XMLName xml.Name `xml:"catmetadata"`
- Longdescriptions []Longdescription `xml:"longdescription"`
+ LongDescriptions []LongDescription `xml:"longdescription"`
}
-type Longdescription struct {
+type LongDescription struct {
XMLName xml.Name `xml:"longdescription"`
Lang string `xml:"lang,attr"`
Content string `xml:",chardata"`
diff --git a/pkg/portage/repository/commit.go b/pkg/portage/repository/commit.go
index f9ccc7f..615009f 100644
--- a/pkg/portage/repository/commit.go
+++ b/pkg/portage/repository/commit.go
@@ -14,6 +14,15 @@ import (
"time"
)
+var (
+ // arrays for collecting date for batch dump into database
+ commits []*models.Commit
+ packages []*models.Package
+ keywordChanges = map[string]*models.KeywordChange{}
+ packagesCommit []*models.CommitToPackage
+ versionsCommits []*models.CommitToVersion
+)
+
// UpdateCommits incrementally imports all new commits. New commits are
// determined by retrieving the last commit in the database (if present)
// and parsing all following commits. In case no last commit is present
@@ -21,26 +30,30 @@ import (
func UpdateCommits() string {
logger.Info.Println("Start updating commits")
- latestCommit, PrecedingCommitsOffset := utils.GetLatestCommitAndPreceeding()
+ latestCommit, precedingCommitsOffset := utils.GetLatestCommitAndPreceding()
+
+ for precedingCommits, rawCommit := range utils.GetCommits(latestCommit, "HEAD") {
+ latestCommit = processCommit(precedingCommits, precedingCommitsOffset, rawCommit)
- for PrecedingCommits, rawCommit := range utils.GetCommits(latestCommit, "HEAD") {
- latestCommit = processCommit(PrecedingCommits, PrecedingCommitsOffset, rawCommit)
+ if len(commits) > 10000 {
+ dumpToDatabase()
+ }
}
+ dumpToDatabase()
logger.Info.Println("Finished updating commits")
return latestCommit
}
// processCommit parses a single commit log output and updates it into the database
-func processCommit(PrecedingCommits int, PrecedingCommitsOffset int, rawCommit string) string {
-
+func processCommit(PrecedingCommits, PrecedingCommitsOffset int, rawCommit string) string {
commitLines := strings.Split(rawCommit, "\n")
if len(commitLines) < 8 {
return ""
}
- logProgess(PrecedingCommits)
+ logProgress(PrecedingCommits)
id := strings.TrimSpace(strings.ReplaceAll(commitLines[0], "commit ", ""))
authorName := strings.TrimSpace(strings.Split(strings.ReplaceAll(commitLines[1], "Author: ", ""), "<")[0])
@@ -59,7 +72,7 @@ func processCommit(PrecedingCommits int, PrecedingCommitsOffset int, rawCommit s
changedFiles := processChangedFiles(PrecedingCommits, PrecedingCommitsOffset, commitLines, id)
- commit := &models.Commit{
+ commits = append(commits, &models.Commit{
Id: id,
PrecedingCommits: PrecedingCommitsOffset + PrecedingCommits + 1,
AuthorName: authorName,
@@ -70,26 +83,16 @@ func processCommit(PrecedingCommits int, PrecedingCommitsOffset int, rawCommit s
CommitterDate: committerDate,
Message: message,
ChangedFiles: changedFiles,
- }
-
- _, err := database.DBCon.Model(commit).OnConflict("(id) DO UPDATE").Insert()
-
- if err != nil {
- logger.Error.Println("Error during updating commit: " + id)
- logger.Error.Println(err)
- }
+ })
return id
}
// processChangedFiles parses files that have changed in the commit and links the
// commit to packages and package versions
-func processChangedFiles(PrecedingCommits int, PrecedingCommitsOffset int, commitLines []string, id string) *models.ChangedFiles {
- var addedFiles []*models.ChangedFile
- var modifiedFiles []*models.ChangedFile
- var deletedFiles []*models.ChangedFile
+func processChangedFiles(PrecedingCommits, PrecedingCommitsOffset int, commitLines []string, id string) *models.ChangedFiles {
+ var addedFiles, modifiedFiles, deletedFiles []*models.ChangedFile
for _, commitLine := range commitLines {
-
line := strings.Split(commitLine, "\t")
if len(line) < 2 {
continue
@@ -99,25 +102,18 @@ func processChangedFiles(PrecedingCommits int, PrecedingCommitsOffset int, commi
path := strings.TrimSpace(line[1])
if strings.HasPrefix(status, "M") {
-
- modifiedFiles = addChangedFile(modifiedFiles, path, "M")
+ modifiedFiles = append(modifiedFiles, &models.ChangedFile{Path: path, ChangeType: "M"})
createKeywordChange(id, path, commitLine)
-
} else if strings.HasPrefix(commitLine, "D") {
-
- deletedFiles = addChangedFile(deletedFiles, path, "D")
-
+ deletedFiles = append(deletedFiles, &models.ChangedFile{Path: path, ChangeType: "D"})
} else if strings.HasPrefix(commitLine, "A") {
-
- addedFiles = addChangedFile(addedFiles, path, "A")
+ addedFiles = append(addedFiles, &models.ChangedFile{Path: path, ChangeType: "A"})
updateFirstCommitOfPackage(path, commitLine, PrecedingCommitsOffset+PrecedingCommits+1)
createAddedKeywords(id, path, commitLine)
-
}
linkCommitToPackage(commitLine, path, id)
linkCommitToVersion(commitLine, path, id)
-
}
return &models.ChangedFiles{
@@ -127,8 +123,8 @@ func processChangedFiles(PrecedingCommits int, PrecedingCommitsOffset int, commi
}
}
-// logProgess logs the progress of a loop
-func logProgess(counter int) {
+// logProgress logs the progress of a loop
+func logProgress(counter int) {
if counter%1000 == 0 {
logger.Info.Println("Processed commits: " + strconv.Itoa(counter))
} else if counter == 1 {
@@ -137,67 +133,46 @@ func logProgess(counter int) {
}
}
-func linkCommitToPackage(commitLine string, path string, id string) {
- var commitToPackage *models.CommitToPackage
+func linkCommitToPackage(commitLine, path, id string) {
if (len(strings.Split(commitLine, "/")) >= 3) &&
(strings.HasPrefix(commitLine, "M") ||
strings.HasPrefix(commitLine, "D") ||
strings.HasPrefix(commitLine, "A")) {
- pathParts := strings.Split(strings.ReplaceAll(path, ".ebuild", ""), "/")
+ pathParts := strings.Split(strings.TrimSuffix(path, ".ebuild"), "/")
- commitToPackageId := id + "-" + pathParts[0] + "/" + strings.Split(commitLine, "/")[1]
- commitToPackage = &models.CommitToPackage{
- Id: commitToPackageId,
+ packageAtom := pathParts[0] + "/" + strings.Split(commitLine, "/")[1]
+ packagesCommit = append(packagesCommit, &models.CommitToPackage{
+ Id: id + "-" + packageAtom,
CommitId: id,
- PackageAtom: pathParts[0] + "/" + strings.Split(commitLine, "/")[1],
- }
-
- _, err := database.DBCon.Model(commitToPackage).OnConflict("(id) DO NOTHING").Insert()
-
- if err != nil {
- logger.Error.Println("Error during updating CommitToPackage: " + commitToPackageId)
- logger.Error.Println(err)
- }
-
+ PackageAtom: packageAtom,
+ })
}
}
-func linkCommitToVersion(commitLine string, path string, id string) {
- var commitToVersion *models.CommitToVersion
+func linkCommitToVersion(commitLine, path, id string) {
if (strings.HasPrefix(commitLine, "M") ||
strings.HasPrefix(commitLine, "D") ||
strings.HasPrefix(commitLine, "A")) &&
- len(strings.Split(strings.ReplaceAll(path, ".ebuild", ""), "/")) == 3 &&
+ len(strings.Split(strings.TrimSuffix(path, ".ebuild"), "/")) == 3 &&
strings.HasSuffix(strings.TrimSpace(strings.Split(commitLine, "\t")[1]), ".ebuild") {
- pathParts := strings.Split(strings.ReplaceAll(path, ".ebuild", ""), "/")
+ pathParts := strings.Split(strings.TrimSuffix(path, ".ebuild"), "/")
- commitToVersionId := id + "-" + pathParts[0] + "/" + pathParts[2]
- commitToVersion = &models.CommitToVersion{
- Id: commitToVersionId,
+ versionId := pathParts[0] + "/" + pathParts[2]
+ versionsCommits = append(versionsCommits, &models.CommitToVersion{
+ Id: id + "-" + versionId,
CommitId: id,
- VersionId: pathParts[0] + "/" + pathParts[2],
- }
-
- _, err := database.DBCon.Model(commitToVersion).OnConflict("(id) DO NOTHING").Insert()
-
- if err != nil {
- logger.Error.Println("Error during updating CommitToVersion: " + commitToVersionId)
- logger.Error.Println(err)
- }
-
+ VersionId: versionId,
+ })
}
}
-func createKeywordChange(id string, path string, commitLine string) {
-
- if !strings.HasSuffix(path, ".ebuild") || !(len(strings.Split(commitLine, "/")) >= 3) {
+func createKeywordChange(id, path, commitLine string) {
+ if !strings.HasSuffix(path, ".ebuild") || !(strings.Count(commitLine, "/") >= 2) {
return
}
- var change *models.KeywordChange
-
raw_lines, err := utils.Exec(config.PortDir(), "git", "show", id, "--", path)
if err != nil {
if exitError, ok := err.(*exec.ExitError); !ok || exitError.ExitCode() != 1 {
@@ -206,23 +181,19 @@ func createKeywordChange(id string, path string, commitLine string) {
}
}
- var keywords_old []string
- var keywords_new []string
+ var keywords_old, keywords_new []string
for _, line := range raw_lines {
if strings.HasPrefix(line, "-KEYWORDS=") {
- keywords_old = strings.Split(strings.ReplaceAll(strings.ReplaceAll(line, "-KEYWORDS=", ""), "\"", ""), " ")
-
+ keywords_old = strings.Split(strings.ReplaceAll(strings.TrimPrefix(line, "-KEYWORDS="), "\"", ""), " ")
} else if strings.HasPrefix(line, "+KEYWORDS") {
- keywords_new = strings.Split(strings.ReplaceAll(strings.ReplaceAll(line, "+KEYWORDS=", ""), "\"", ""), " ")
+ keywords_new = strings.Split(strings.ReplaceAll(strings.TrimPrefix(line, "+KEYWORDS="), "\"", ""), " ")
}
}
- var added_keywords []string
- var stabilized_keywords []string
+ var added_keywords, stabilized_keywords []string
if keywords_old != nil && keywords_new != nil {
-
for _, keyword := range keywords_new {
if !utils.Contains(keywords_old, keyword) {
added_keywords = append(added_keywords, keyword)
@@ -233,10 +204,10 @@ func createKeywordChange(id string, path string, commitLine string) {
}
}
- pathParts := strings.Split(strings.ReplaceAll(path, ".ebuild", ""), "/")
+ pathParts := strings.Split(strings.TrimSuffix(path, ".ebuild"), "/")
keywordChangeId := id + "-" + strings.TrimSpace(strings.Split(commitLine, "\t")[1])
- change = &models.KeywordChange{
+ keywordChanges[keywordChangeId] = &models.KeywordChange{
Id: keywordChangeId,
CommitId: id,
VersionId: pathParts[0] + "/" + pathParts[2],
@@ -245,21 +216,12 @@ func createKeywordChange(id string, path string, commitLine string) {
Stabilized: stabilized_keywords,
All: keywords_new,
}
-
- _, err := database.DBCon.Model(change).OnConflict("(id) DO UPDATE").Insert()
-
- if err != nil {
- logger.Error.Println("Error updating Keyword change: " + keywordChangeId)
- logger.Error.Println(err)
- }
-
}
}
func createAddedKeywords(id string, path string, commitLine string) {
- var change *models.KeywordChange
if strings.HasSuffix(strings.TrimSpace(strings.Split(commitLine, "\t")[1]), ".ebuild") &&
- (len(strings.Split(commitLine, "/")) >= 3) {
+ (strings.Count(commitLine, "/") >= 2) {
raw_lines, err := utils.Exec(config.PortDir(), "git", "show", id, "--", path)
if err != nil {
@@ -272,12 +234,11 @@ func createAddedKeywords(id string, path string, commitLine string) {
for _, line := range raw_lines {
if strings.HasPrefix(line, "+KEYWORDS=") {
-
- pathParts := strings.Split(strings.ReplaceAll(path, ".ebuild", ""), "/")
- keywords := strings.Split(strings.ReplaceAll(strings.ReplaceAll(line, "+KEYWORDS=", ""), "\"", ""), " ")
+ pathParts := strings.Split(strings.TrimSuffix(path, ".ebuild"), "/")
+ keywords := strings.Split(strings.ReplaceAll(strings.TrimPrefix(line, "+KEYWORDS="), "\"", ""), " ")
keywordChangeId := id + "-" + strings.TrimSpace(strings.Split(commitLine, "\t")[1])
- change = &models.KeywordChange{
+ keywordChanges[keywordChangeId] = &models.KeywordChange{
Id: keywordChangeId,
CommitId: id,
VersionId: pathParts[0] + "/" + pathParts[2],
@@ -285,14 +246,6 @@ func createAddedKeywords(id string, path string, commitLine string) {
Added: keywords,
All: keywords,
}
-
- _, err := database.DBCon.Model(change).OnConflict("(id) DO UPDATE").Insert()
-
- if err != nil {
- logger.Error.Println("Error updating Keyword change: " + keywordChangeId)
- logger.Error.Println(err)
- }
-
}
}
@@ -301,26 +254,60 @@ func createAddedKeywords(id string, path string, commitLine string) {
func updateFirstCommitOfPackage(path string, commitLine string, precedingCommits int) {
// Added Package
- if strings.HasSuffix(path, "metadata.xml") && len(strings.Split(path, "/")) == 3 {
-
+ if strings.HasSuffix(path, "metadata.xml") && strings.Count(commitLine, "/") == 2 {
atom := strings.Split(path, "/")[0] + "/" + strings.Split(path, "/")[1]
- addedpackage := &models.Package{
+ packages = append(packages, &models.Package{
Atom: atom,
PrecedingCommits: precedingCommits,
+ })
+ }
+}
+
+func dumpToDatabase() {
+ logger.Info.Println("Writing to database")
+
+ if len(keywordChanges) > 0 {
+ rows := make([]*models.KeywordChange, 0, len(keywordChanges))
+ for _, keywordChange := range keywordChanges {
+ rows = append(rows, keywordChange)
}
+ _, err := database.DBCon.Model(&rows).OnConflict("(id) DO UPDATE").Insert()
+ if err != nil {
+ logger.Error.Println("Error during updating KeywordChange", err)
+ }
+ keywordChanges = map[string]*models.KeywordChange{}
+ }
- _, err := database.DBCon.Model(addedpackage).Column("preceding_commits").WherePK().Update()
+ if len(packages) > 0 {
+ _, err := database.DBCon.Model(&packages).Column("preceding_commits").Update()
if err != nil {
- logger.Error.Println("Error updating precedingCommits (" + strconv.Itoa(precedingCommits) + ") of package: " + atom)
- logger.Error.Println(err)
+ logger.Error.Println("Error during updating precedingCommits", err)
}
+ packages = nil
+ }
+ if len(packagesCommit) > 0 {
+ _, err := database.DBCon.Model(&packagesCommit).OnConflict("(id) DO NOTHING").Insert()
+ if err != nil {
+ logger.Error.Println("Error during updating CommitToPackage", err)
+ }
+ packagesCommit = nil
+ }
+
+ if len(versionsCommits) > 0 {
+ _, err := database.DBCon.Model(&versionsCommits).OnConflict("(id) DO NOTHING").Insert()
+ if err != nil {
+ logger.Error.Println("Error during updating CommitToVersion", err)
+ }
+ versionsCommits = nil
+ }
+
+ if len(commits) > 0 {
+ _, err := database.DBCon.Model(&commits).OnConflict("(id) DO UPDATE").Insert()
+ if err != nil {
+ logger.Error.Println("Error during updating commits:", err)
+ }
+ commits = nil
}
-}
-func addChangedFile(changedFiles []*models.ChangedFile, path string, status string) []*models.ChangedFile {
- return append(changedFiles, &models.ChangedFile{
- Path: path,
- ChangeType: status,
- })
}
diff --git a/pkg/portage/repository/mask.go b/pkg/portage/repository/mask.go
index 10d8609..3e1df83 100644
--- a/pkg/portage/repository/mask.go
+++ b/pkg/portage/repository/mask.go
@@ -13,6 +13,7 @@ package repository
import (
"regexp"
+ "soko/pkg/config"
"soko/pkg/database"
"soko/pkg/logger"
"soko/pkg/models"
@@ -36,14 +37,15 @@ func UpdateMask(path string) {
splittedLine := strings.Split(path, "\t")
var status, changedFile string
- if len(splittedLine) == 2 {
+ switch len(splittedLine) {
+ case 2:
status = splittedLine[0]
changedFile = splittedLine[1]
- } else if len(splittedLine) == 1 {
+ case 1:
// This happens in case of a full update
status = "A"
changedFile = splittedLine[0]
- } else {
+ default:
// should not happen
return
}
@@ -54,7 +56,7 @@ func UpdateMask(path string) {
// delete all existing masks before parsing the file again
// in future we might implement a incremental version here
- deleteAllMasks()
+ database.TruncateTable[models.Mask]("versions")
for _, packageMask := range getMasks(changedFile) {
parsePackageMask(packageMask)
@@ -62,6 +64,8 @@ func UpdateMask(path string) {
}
}
+var versionNumber = regexp.MustCompile(`-[0-9]`)
+
// versionSpecifierToPackageAtom returns the package atom from a given version specifier
func versionSpecifierToPackageAtom(versionSpecifier string) string {
gpackage := strings.ReplaceAll(versionSpecifier, ">", "")
@@ -70,9 +74,7 @@ func versionSpecifierToPackageAtom(versionSpecifier string) string {
gpackage = strings.ReplaceAll(gpackage, "~", "")
gpackage = strings.Split(gpackage, ":")[0]
-
- versionnumber := regexp.MustCompile(`-[0-9]`)
- gpackage = versionnumber.Split(gpackage, 2)[0]
+ gpackage = versionNumber.Split(gpackage, 2)[0]
return gpackage
}
@@ -109,7 +111,7 @@ func parsePackageMask(packageMask string) {
packageMaskLine, packageMaskLines := packageMaskLines[0], packageMaskLines[1:]
author, authorEmail, date := parseAuthorLine(packageMaskLine)
- reason := ""
+ var reason string
packageMaskLine, packageMaskLines = packageMaskLines[0], packageMaskLines[1:]
for strings.HasPrefix(packageMaskLine, "#") {
reason = reason + " " + strings.Replace(packageMaskLine, "# ", "", 1)
@@ -119,7 +121,7 @@ func parsePackageMask(packageMask string) {
packageMaskLines = append(packageMaskLines, packageMaskLine)
for _, version := range packageMaskLines {
- useflag := &models.Mask{
+ mask := &models.Mask{
Author: author,
AuthorEmail: authorEmail,
Date: date,
@@ -127,11 +129,9 @@ func parsePackageMask(packageMask string) {
Versions: version,
}
- _, err := database.DBCon.Model(useflag).OnConflict("(versions) DO UPDATE").Insert()
-
+ _, err := database.DBCon.Model(mask).OnConflict("(versions) DO UPDATE").Insert()
if err != nil {
- logger.Error.Println("Error while inserting/updating package mask entry")
- logger.Error.Println(err)
+ logger.Error.Println("Error while inserting/updating package mask entry", err)
}
}
}
@@ -141,11 +141,10 @@ func parsePackageMask(packageMask string) {
// get all mask entries from the package.mask file
func getMasks(path string) []string {
var masks []string
- lines, err := utils.ReadLines(path)
+ lines, err := utils.ReadLines(config.PortDir() + "/" + path)
if err != nil {
- logger.Error.Println("Could not read Masks file. Abort masks import")
- logger.Error.Println(err)
+ logger.Error.Println("Could not read Masks file. Abort masks import", err)
return masks
}
@@ -161,15 +160,14 @@ func getMasks(path string) []string {
// Calculate all versions that are currently
// masked and update the MaskToVersion Table
func CalculateMaskedVersions() {
-
// clean up all masked versions before recalculating them
- deleteAllMasksToVersion()
+ database.TruncateTable[models.MaskToVersion]("id")
var masks []*models.Mask
err := database.DBCon.Model(&masks).Select()
if err != nil && err != pg.ErrNoRows {
- logger.Error.Println("Failed to retrieve package masks. Aborting update")
- logger.Error.Println(err)
+ logger.Error.Println("Failed to retrieve package masks. Aborting update", err)
+ return
}
for _, mask := range masks {
@@ -178,7 +176,7 @@ func CalculateMaskedVersions() {
var versions []*models.Version
if strings.HasPrefix(versionSpecifier, "=") {
- versions = exaktVersion(versionSpecifier, packageAtom)
+ versions = exactVersion(versionSpecifier, packageAtom)
} else if strings.HasPrefix(versionSpecifier, "<=") {
versions = comparedVersions("<=", versionSpecifier, packageAtom)
} else if strings.HasPrefix(versionSpecifier, "<") {
@@ -247,8 +245,8 @@ func allRevisions(versionSpecifier string, packageAtom string) []*models.Version
return versions
}
-// exaktVersion returns the exact version specified in the versionSpecifier
-func exaktVersion(versionSpecifier string, packageAtom string) []*models.Version {
+// exactVersion returns the exact version specified in the versionSpecifier
+func exactVersion(versionSpecifier string, packageAtom string) []*models.Version {
var versions []*models.Version
database.DBCon.Model(&versions).
Where("id = ?", strings.Replace(versionSpecifier, "=", "", 1)).
@@ -293,26 +291,7 @@ func maskVersions(versionSpecifier string, versions []*models.Version) {
_, err := database.DBCon.Model(maskToVersion).OnConflict("(id) DO UPDATE").Insert()
if err != nil {
- logger.Error.Println("Error while inserting mask to version entry")
- logger.Error.Println(err)
+ logger.Error.Println("Error while inserting mask to version entry", err)
}
}
}
-
-// deleteAllMasks deletes all entries in the mask table
-func deleteAllMasksToVersion() {
- var masks []*models.MaskToVersion
- database.DBCon.Model(&masks).Select()
- for _, mask := range masks {
- database.DBCon.Model(mask).WherePK().Delete()
- }
-}
-
-// deleteAllMasks deletes all entries in the mask table
-func deleteAllMasks() {
- var masks []*models.Mask
- database.DBCon.Model(&masks).Select()
- for _, mask := range masks {
- database.DBCon.Model(mask).WherePK().Delete()
- }
-}
diff --git a/pkg/portage/repository/package.go b/pkg/portage/repository/package.go
index 4263b1c..2c51caa 100644
--- a/pkg/portage/repository/package.go
+++ b/pkg/portage/repository/package.go
@@ -4,7 +4,6 @@ package repository
import (
"encoding/xml"
- "io/ioutil"
"os"
"regexp"
"soko/pkg/config"
@@ -21,89 +20,136 @@ func isPackage(path string) bool {
return isPackage
}
-// UpdatePackage updates the package in the database in case
-// the given path points to a package description
-func UpdatePackage(path string) {
-
- splittedLine := strings.Split(path, "\t")
+// UpdatePackages updates the packages in the database for each
+// given path that points to a package description
+func UpdatePackages(paths []string) {
+ deleted := map[string]*models.Package{}
+ modified := map[string]*models.Package{}
+
+ for _, path := range paths {
+ splittedLine := strings.Split(path, "\t")
+
+ if len(splittedLine) != 2 {
+ if len(splittedLine) == 1 && isPackage(path) {
+ if pkg := updateModifiedPackage(path); pkg != nil {
+ modified[pkg.Atom] = pkg
+ }
+ }
+ continue
+ }
- if len(splittedLine) != 2 {
- if len(splittedLine) == 1 && isPackage(path) {
- updateModifiedPackage(path)
+ status := splittedLine[0]
+ changedFile := splittedLine[1]
+
+ if !isPackage(changedFile) {
+ continue
+ } else if status == "D" {
+ pkg := updateDeletedPackage(changedFile)
+ deleted[pkg.Atom] = pkg
+ } else if status == "A" || status == "M" {
+ if pkg := updateModifiedPackage(changedFile); pkg != nil {
+ modified[pkg.Atom] = pkg
+ }
}
- return
}
- status := splittedLine[0]
- changedFile := splittedLine[1]
+ if len(deleted) > 0 {
+ rows := make([]*models.Package, 0, len(deleted))
+ for _, row := range deleted {
+ rows = append(rows, row)
+ }
+ res, err := database.DBCon.Model(&rows).Delete()
+ if err != nil {
+ logger.Error.Println("Error during deleting packages", err)
+ } else {
+ logger.Info.Println("Deleted", res.RowsAffected(), "packages")
+ }
+ }
- if isPackage(changedFile) && status == "D" {
- updateDeletedPackage(changedFile)
- } else if isPackage(changedFile) && (status == "A" || status == "M") {
- updateModifiedPackage(changedFile)
+ if len(modified) > 0 {
+ rows := make([]*models.Package, 0, len(modified))
+ for _, row := range modified {
+ rows = append(rows, row)
+ }
+ res, err := database.DBCon.Model(&rows).OnConflict("(atom) DO UPDATE").
+ Set("atom = EXCLUDED.atom").
+ Set("category = EXCLUDED.category").
+ Set("name = EXCLUDED.name").
+ Set("longdescription = EXCLUDED.longdescription").
+ Set("maintainers = EXCLUDED.maintainers").
+ Insert()
+ if err != nil {
+ logger.Error.Println("Error during updating packages", err)
+ } else {
+ logger.Info.Println("Updated", res.RowsAffected(), "packages")
+ }
}
}
// updateDeletedPackage deletes a package from the database
-func updateDeletedPackage(changedFile string) {
+func updateDeletedPackage(changedFile string) *models.Package {
splitted := strings.Split(changedFile, "/")
category := splitted[0]
packagename := splitted[1]
atom := category + "/" + packagename
- gpackage := &models.Package{Atom: atom}
- _, err := database.DBCon.Model(gpackage).WherePK().Delete()
-
- if err != nil {
- logger.Error.Println("Error during deleting package " + atom)
- logger.Error.Println(err)
- }
+ return &models.Package{Atom: atom}
}
// updateModifiedPackage adds a package to the database or
// updates it. To do so, it parses the metadata from metadata.xml
-func updateModifiedPackage(changedFile string) {
+func updateModifiedPackage(changedFile string) *models.Package {
splitted := strings.Split(changedFile, "/")
category := splitted[0]
packagename := splitted[1]
atom := category + "/" + packagename
- pkgmetadata := GetPkgMetadata(config.PortDir() + "/" + atom + "/metadata.xml")
- var maintainers []*models.Maintainer
+ xmlFile, err := os.Open(config.PortDir() + "/" + atom + "/metadata.xml")
+ if err != nil {
+ logger.Error.Println("Error during reading package metadata", err)
+ return nil
+ }
+ defer xmlFile.Close()
+ var pkgMetadata PkgMetadata
+ err = xml.NewDecoder(xmlFile).Decode(&pkgMetadata)
+ if err != nil {
+ logger.Error.Println("Error during package", changedFile, "decoding", err)
+ return nil
+ }
- for _, maintainer := range pkgmetadata.MaintainerList {
- maintainer := &models.Maintainer{
+ maintainers := make([]*models.Maintainer, len(pkgMetadata.MaintainerList))
+ for i, maintainer := range pkgMetadata.MaintainerList {
+ maintainers[i] = &models.Maintainer{
Name: maintainer.Name,
Type: maintainer.Type,
Email: maintainer.Email,
Restrict: maintainer.Restrict,
}
- maintainers = append(maintainers, maintainer)
}
- longDescription := ""
- for _, l := range pkgmetadata.LongdescriptionList {
+ var longDescription string
+ for _, l := range pkgMetadata.LongDescriptionList {
if l.Language == "" || l.Language == "en" {
longDescription = l.Content
}
}
- remoteIds := []models.RemoteId{}
- for _, r := range pkgmetadata.Upstream.RemoteIds {
- remoteIds = append(remoteIds, models.RemoteId{
+ remoteIds := make([]models.RemoteId, len(pkgMetadata.Upstream.RemoteIds))
+ for i, r := range pkgMetadata.Upstream.RemoteIds {
+ remoteIds[i] = models.RemoteId{
Type: r.Type,
Id: r.Content,
- })
+ }
}
upstream := models.Upstream{
RemoteIds: remoteIds,
- Doc: pkgmetadata.Upstream.Doc,
- BugsTo: pkgmetadata.Upstream.BugsTo,
- Changelog: pkgmetadata.Upstream.Changelog,
+ Doc: pkgMetadata.Upstream.Doc,
+ BugsTo: pkgMetadata.Upstream.BugsTo,
+ Changelog: pkgMetadata.Upstream.Changelog,
}
- gpackage := &models.Package{
+ return &models.Package{
Atom: atom,
Category: category,
Name: packagename,
@@ -111,42 +157,14 @@ func updateModifiedPackage(changedFile string) {
Maintainers: maintainers,
Upstream: upstream,
}
-
- _, err := database.DBCon.Model(gpackage).OnConflict("(atom) DO UPDATE").
- Set("atom = EXCLUDED.atom").
- Set("category = EXCLUDED.category").
- Set("name = EXCLUDED.name").
- Set("longdescription = EXCLUDED.longdescription").
- Set("maintainers = EXCLUDED.maintainers").
- Insert()
-
- if err != nil {
- logger.Error.Println("Error during updating package " + atom)
- logger.Error.Println(err)
- }
-}
-
-// GetPkgMetadata reads and parses the package
-// metadata from the metadata.xml file
-func GetPkgMetadata(path string) Pkgmetadata {
- xmlFile, err := os.Open(path)
- if err != nil {
- logger.Error.Println("Error during reading package metadata")
- logger.Error.Println(err)
- }
- defer xmlFile.Close()
- byteValue, _ := ioutil.ReadAll(xmlFile)
- var pkgmetadata Pkgmetadata
- xml.Unmarshal(byteValue, &pkgmetadata)
- return pkgmetadata
}
// Descriptions of the package metadata.xml format
-type Pkgmetadata struct {
+type PkgMetadata struct {
XMLName xml.Name `xml:"pkgmetadata"`
MaintainerList []Maintainer `xml:"maintainer"`
- LongdescriptionList []LongdescriptionItem `xml:"longdescription"`
+ LongDescriptionList []LongDescriptionItem `xml:"longdescription"`
Upstream Upstream `xml:"upstream"`
}
@@ -158,7 +176,7 @@ type Maintainer struct {
Name string `xml:"name"`
}
-type LongdescriptionItem struct {
+type LongDescriptionItem struct {
XMLName xml.Name `xml:"longdescription"`
Content string `xml:",chardata"`
Language string `xml:"lang,attr"`
diff --git a/pkg/portage/repository/use.go b/pkg/portage/repository/use.go
index 9e44328..8ba915a 100644
--- a/pkg/portage/repository/use.go
+++ b/pkg/portage/repository/use.go
@@ -3,7 +3,6 @@
package repository
import (
- "errors"
"soko/pkg/config"
"soko/pkg/database"
"soko/pkg/logger"
@@ -12,26 +11,22 @@ import (
"strings"
)
-var (
- errInvalidLine = errors.New("invalid line")
-)
-
// UpdateUse reads all USE flags descriptions from the given file in
// case the given file contains USE flags descriptions and imports
// each USE flag into the database
func UpdateUse(path string) {
-
splittedLine := strings.Split(path, "\t")
var status, changedFile string
- if len(splittedLine) == 2 {
+ switch len(splittedLine) {
+ case 2:
status = splittedLine[0]
changedFile = splittedLine[1]
- } else if len(splittedLine) == 1 {
+ case 1:
// This happens in case of a full update
status = "A"
changedFile = splittedLine[0]
- } else {
+ default:
// should not happen
return
}
@@ -40,27 +35,34 @@ func UpdateUse(path string) {
rawFlags, _ := utils.ReadLines(config.PortDir() + "/" + changedFile)
+ useFlags := make(map[string]*models.Useflag, len(rawFlags))
for _, rawFlag := range rawFlags {
-
- if strings.TrimSpace(rawFlag) == "" || rawFlag[0:1] == "#" {
+ if strings.TrimSpace(rawFlag) == "" || rawFlag[0] == '#' {
continue
}
-
scope := getScope(changedFile)
-
- var err error
- if scope == "local" || scope == "global" {
- err = createUseflag(rawFlag, scope)
- } else if scope == "use_expand" {
+ switch scope {
+ case "local", "global":
+ if flag := createUseflag(rawFlag, scope); flag != nil {
+ useFlags[flag.Id] = flag
+ }
+ case "use_expand":
file := strings.Split(changedFile, "/")[2]
- err = createUseExpand(rawFlag, file)
+ flag := createUseExpand(rawFlag, file)
+ useFlags[flag.Id] = flag
}
+ }
+ if len(useFlags) > 0 {
+ rows := make([]*models.Useflag, 0, len(useFlags))
+ for _, row := range useFlags {
+ rows = append(rows, row)
+ }
+ res, err := database.DBCon.Model(&rows).OnConflict("(id) DO UPDATE").Insert()
if err != nil {
- logger.Info.Println("Error during updating useflag " + rawFlag)
- logger.Info.Println(err)
- logger.Error.Println("Error during updating useflag " + rawFlag)
- logger.Error.Println(err)
+ logger.Error.Println("Error during updating use flags", err)
+ } else {
+ logger.Info.Println("Updated", res.RowsAffected(), "use flags")
}
}
}
@@ -69,50 +71,42 @@ func UpdateUse(path string) {
// createUseflag parses the description from the file,
// creates a USE flag and imports it into the database
-func createUseflag(rawFlag string, scope string) error {
+func createUseflag(rawFlag string, scope string) *models.Useflag {
pkguse, description, found := strings.Cut(rawFlag, " - ")
if !found {
- return errInvalidLine
+ return nil
}
pkg, use, found := strings.Cut(pkguse, ":")
if found != (scope == "local") {
- return errInvalidLine
+ return nil
} else if !found {
use = pkguse
}
- useflag := &models.Useflag{
+ return &models.Useflag{
Id: pkguse + "-" + scope,
Package: pkg,
Name: use,
Scope: scope,
Description: description,
}
-
- _, err := database.DBCon.Model(useflag).OnConflict("(id) DO UPDATE").Insert()
-
- return err
}
// createUseExpand parses the description from the file,
// creates a USE expand flag and imports it into the database
-func createUseExpand(rawFlag string, file string) error {
- name := strings.ReplaceAll(file, ".desc", "")
- line := strings.Split(rawFlag, " - ")
- id := name + "_" + line[0]
+func createUseExpand(rawFlag string, file string) *models.Useflag {
+ group := strings.TrimSuffix(file, ".desc")
+ unexpanded, description, _ := strings.Cut(rawFlag, " - ")
+ id := group + "_" + unexpanded
- useExpand := &models.Useflag{
+ return &models.Useflag{
Id: id,
- Name: name + "_" + line[0],
+ Name: id,
Scope: "use_expand",
- Description: strings.Join(line[1:], ""),
- UseExpand: name,
+ Description: description,
+ UseExpand: group,
}
-
- _, err := database.DBCon.Model(useExpand).OnConflict("(id) DO UPDATE").Insert()
-
- return err
}
// getScope returns either "local", "global", "use_expand"
diff --git a/pkg/portage/repository/version.go b/pkg/portage/repository/version.go
index 22acbe6..3ee520d 100644
--- a/pkg/portage/repository/version.go
+++ b/pkg/portage/repository/version.go
@@ -19,32 +19,67 @@ func isVersion(path string) bool {
return isVersion
}
-// UpdateVersion updates the version in the database in case
-// the given path points to a package version
-func UpdateVersion(path string) {
-
- line := strings.Split(path, "\t")
+// UpdateVersions updates the versions in the database for each
+// given path that points to a package version
+func UpdateVersions(paths []string) {
+ deleted := map[string]*models.Version{}
+ modified := map[string]*models.Version{}
+
+ for _, path := range paths {
+ line := strings.Split(path, "\t")
+
+ if len(line) != 2 {
+ if len(line) == 1 && isVersion(path) {
+ ver := updateModifiedVersion(path)
+ modified[ver.Id] = ver
+ }
+ continue
+ }
- if len(line) != 2 {
- if len(line) == 1 && isVersion(path) {
- updateModifiedVersion(path)
+ status := line[0]
+ changedFile := line[1]
+
+ if !isVersion(changedFile) {
+ continue
+ } else if status == "D" {
+ ver := updateDeletedVersion(changedFile)
+ deleted[ver.Id] = ver
+ } else if status == "A" || status == "M" {
+ ver := updateModifiedVersion(changedFile)
+ modified[ver.Id] = ver
}
- return
}
- status := line[0]
- changedFile := line[1]
+ if len(deleted) > 0 {
+ rows := make([]*models.Version, 0, len(deleted))
+ for _, row := range deleted {
+ rows = append(rows, row)
+ }
+ res, err := database.DBCon.Model(&rows).Delete()
+ if err != nil {
+ logger.Error.Println("Error during deleting versions", err)
+ } else {
+ logger.Info.Println("Deleted", res.RowsAffected(), "versions")
+ }
+ }
- if isVersion(changedFile) && status == "D" {
- updateDeletedVersion(changedFile)
- } else if isVersion(changedFile) && (status == "A" || status == "M") {
- updateModifiedVersion(changedFile)
+ if len(modified) > 0 {
+ rows := make([]*models.Version, 0, len(modified))
+ for _, row := range modified {
+ rows = append(rows, row)
+ }
+ res, err := database.DBCon.Model(&rows).OnConflict("(id) DO UPDATE").Insert()
+ if err != nil {
+ logger.Error.Println("Error during updating versions", err)
+ } else {
+ logger.Info.Println("Updated", res.RowsAffected(), "versions")
+ }
}
}
// updateDeletedVersion deletes a package version from the database
-func updateDeletedVersion(changedFile string) {
- splitted := strings.Split(strings.ReplaceAll(changedFile, ".ebuild", ""), "/")
+func updateDeletedVersion(changedFile string) *models.Version {
+ splitted := strings.Split(strings.TrimSuffix(changedFile, ".ebuild"), "/")
category := splitted[0]
packagename := splitted[1]
version := strings.ReplaceAll(splitted[2], packagename+"-", "")
@@ -52,19 +87,13 @@ func updateDeletedVersion(changedFile string) {
atom := category + "/" + packagename
id := atom + "-" + version
- versionObject := &models.Version{Id: id}
- _, err := database.DBCon.Model(versionObject).WherePK().Delete()
-
- if err != nil {
- logger.Error.Println("Error during deleting version " + id)
- logger.Error.Println(err)
- }
+ return &models.Version{Id: id}
}
// updateModifiedVersion adds a package version to the database or
// updates it. To do so, it parses the metadata from the md5-cache
-func updateModifiedVersion(changedFile string) {
- splitted := strings.Split(strings.ReplaceAll(changedFile, ".ebuild", ""), "/")
+func updateModifiedVersion(changedFile string) *models.Version {
+ splitted := strings.Split(strings.TrimSuffix(changedFile, ".ebuild"), "/")
category := splitted[0]
packagename := splitted[1]
version := strings.ReplaceAll(splitted[2], packagename+"-", "")
@@ -76,56 +105,47 @@ func updateModifiedVersion(changedFile string) {
slot := "0"
subslot := "0"
- eapi := ""
- keywords := ""
- var useflags []string
- var restricts []string
- var properties []string
- var homepages []string
- license := ""
- description := ""
+ var eapi, keywords, license, description string
+ var useflags, restricts, properties, homepages []string
for _, metadata := range version_metadata {
switch {
case strings.HasPrefix(metadata, "EAPI="):
- eapi = strings.ReplaceAll(metadata, "EAPI=", "")
+ eapi = strings.TrimPrefix(metadata, "EAPI=")
case strings.HasPrefix(metadata, "KEYWORDS="):
- keywords = strings.ReplaceAll(metadata, "KEYWORDS=", "")
+ keywords = strings.TrimPrefix(metadata, "KEYWORDS=")
case strings.HasPrefix(metadata, "IUSE="):
- useflags = strings.Split(strings.ReplaceAll(metadata, "IUSE=", ""), " ")
+ useflags = strings.Split(strings.TrimPrefix(metadata, "IUSE="), " ")
case strings.HasPrefix(metadata, "RESTRICT="):
- restricts = strings.Split(strings.ReplaceAll(strings.ReplaceAll(metadata, "RESTRICT=", ""), "!test? ( test )", ""), " ")
+ restricts = strings.Split(strings.ReplaceAll(strings.TrimPrefix(metadata, "RESTRICT="), "!test? ( test )", ""), " ")
if len(restricts) == 1 && restricts[0] == "" {
restricts = []string{}
}
case strings.HasPrefix(metadata, "PROPERTIES="):
- properties = strings.Split(strings.ReplaceAll(metadata, "PROPERTIES=", ""), " ")
+ properties = strings.Split(strings.TrimPrefix(metadata, "PROPERTIES="), " ")
case strings.HasPrefix(metadata, "HOMEPAGE="):
- homepages = strings.Split(strings.ReplaceAll(metadata, "HOMEPAGE=", ""), " ")
+ homepages = strings.Split(strings.TrimPrefix(metadata, "HOMEPAGE="), " ")
case strings.HasPrefix(metadata, "LICENSE="):
- license = strings.ReplaceAll(metadata, "LICENSE=", "")
+ license = strings.TrimPrefix(metadata, "LICENSE=")
case strings.HasPrefix(metadata, "DESCRIPTION="):
- description = strings.ReplaceAll(metadata, "DESCRIPTION=", "")
+ description = strings.TrimPrefix(metadata, "DESCRIPTION=")
case strings.HasPrefix(metadata, "SLOT="):
- rawslot := strings.ReplaceAll(metadata, "SLOT=", "")
- slot = strings.Split(rawslot, "/")[0]
- if len(strings.Split(rawslot, "/")) > 1 {
- subslot = strings.Split(rawslot, "/")[1]
- }
+ rawSlot := strings.TrimPrefix(metadata, "SLOT=")
+ slot, subslot, _ = strings.Cut(rawSlot, "/")
}
}
- ebuildVersion := &models.Version{
+ return &models.Version{
Id: id,
Category: category,
Package: packagename,
@@ -142,11 +162,4 @@ func updateModifiedVersion(changedFile string) {
License: license,
Description: description,
}
-
- _, err := database.DBCon.Model(ebuildVersion).OnConflict("(id) DO UPDATE").Insert()
-
- if err != nil {
- logger.Error.Println("Error during updating version " + id)
- logger.Error.Println(err)
- }
}
diff --git a/pkg/portage/update.go b/pkg/portage/update.go
index ae38c27..39d8521 100644
--- a/pkg/portage/update.go
+++ b/pkg/portage/update.go
@@ -61,7 +61,9 @@ func updateMetadata() {
latestCommit := utils.GetLatestCommit()
- for _, path := range utils.ChangedFiles(latestCommit, "HEAD") {
+ changed := utils.ChangedFiles(latestCommit, "HEAD")
+ logger.Info.Println("Iterating", len(changed), "changed files")
+ for _, path := range changed {
repository.UpdateUse(path)
repository.UpdateMask(path)
repository.UpdateArch(path)
@@ -82,11 +84,11 @@ func updatePackageData() {
latestCommit := utils.GetLatestCommit()
- for _, path := range utils.ChangedFiles(latestCommit, "HEAD") {
- repository.UpdateVersion(path)
- repository.UpdatePackage(path)
- repository.UpdateCategory(path)
- }
+ changed := utils.ChangedFiles(latestCommit, "HEAD")
+ logger.Info.Println("Iterating", len(changed), "changed files")
+ repository.UpdateVersions(changed)
+ repository.UpdatePackages(changed)
+ repository.UpdateCategories(changed)
}
@@ -149,11 +151,10 @@ func FullUpdate() {
// update the local useflags
repository.UpdateUse("profiles/use.local.desc")
- for _, path := range utils.AllFiles() {
- repository.UpdateVersion(path)
- repository.UpdatePackage(path)
- repository.UpdateCategory(path)
- }
+ allFiles := utils.AllFiles()
+ repository.UpdateVersions(allFiles)
+ repository.UpdatePackages(allFiles)
+ repository.UpdateCategories(allFiles)
// Delete removed entries
logger.Info.Println("Delete removed files from the database")
@@ -169,72 +170,72 @@ func FullUpdate() {
// deleteRemovedVersions removes all versions from the database
// that are present in the database but not in the main tree.
func deleteRemovedVersions() {
- var versions []*models.Version
+ var versions, toDelete []*models.Version
database.DBCon.Model(&versions).Select()
for _, version := range versions {
path := config.PortDir() + "/" + version.Atom + "/" + version.Package + "-" + version.Version + ".ebuild"
if !utils.FileExists(path) {
-
- logger.Error.Println("Found ebuild version in the database that does not exist at:")
- logger.Error.Println(path)
-
- _, err := database.DBCon.Model(version).WherePK().Delete()
-
- if err != nil {
- logger.Error.Println("Error deleting version " + version.Atom + " - " + version.Version)
- logger.Error.Println(err)
- }
+ logger.Error.Println("Found ebuild version in the database that does not exist at:", path)
+ toDelete = append(toDelete, version)
}
+ }
+ if len(toDelete) > 0 {
+ res, err := database.DBCon.Model(&toDelete).Delete()
+ if err != nil {
+ logger.Error.Println("Error deleting versions", err)
+ } else {
+ logger.Info.Println("Deleted", res.RowsAffected(), "versions")
+ }
}
}
// deleteRemovedPackages removes all packages from the database
// that are present in the database but not in the main tree.
func deleteRemovedPackages() {
- var packages []*models.Package
+ var packages, toDelete []*models.Package
database.DBCon.Model(&packages).Select()
- for _, gpackage := range packages {
- path := config.PortDir() + "/" + gpackage.Atom
+ for _, pkg := range packages {
+ path := config.PortDir() + "/" + pkg.Atom
if !utils.FileExists(path) {
-
- logger.Error.Println("Found package in the database that does not exist at:")
- logger.Error.Println(path)
-
- _, err := database.DBCon.Model(gpackage).WherePK().Delete()
-
- if err != nil {
- logger.Error.Println("Error deleting package " + gpackage.Atom)
- logger.Error.Println(err)
- }
+ logger.Error.Println("Found package in the database that does not exist at:", path)
+ toDelete = append(toDelete, pkg)
}
+ }
+ if len(toDelete) > 0 {
+ res, err := database.DBCon.Model(&toDelete).Delete()
+ if err != nil {
+ logger.Error.Println("Error deleting packages", err)
+ } else {
+ logger.Info.Println("Deleted", res.RowsAffected(), "packages")
+ }
}
}
// deleteRemovedCategories removes all categories from the database
// that are present in the database but not in the main tree.
func deleteRemovedCategories() {
- var categories []*models.Category
+ var categories, toDelete []*models.Category
database.DBCon.Model(&categories).Select()
for _, category := range categories {
path := config.PortDir() + "/" + category.Name
if !utils.FileExists(path) {
-
- logger.Error.Println("Found category in the database that does not exist at:")
- logger.Error.Println(path)
-
- _, err := database.DBCon.Model(category).WherePK().Delete()
-
- if err != nil {
- logger.Error.Println("Error deleting category " + category.Name)
- logger.Error.Println(err)
- }
+ logger.Error.Println("Found category in the database that does not exist at:", path)
+ toDelete = append(toDelete, category)
}
+ }
+ if len(toDelete) > 0 {
+ res, err := database.DBCon.Model(&toDelete).Delete()
+ if err != nil {
+ logger.Error.Println("Error deleting categories", err)
+ } else {
+ logger.Info.Println("Deleted", res.RowsAffected(), "categories")
+ }
}
}
@@ -246,15 +247,16 @@ func deleteRemovedCategories() {
// packages' section.
func fixPrecedingCommitsOfPackages() {
var packages []*models.Package
- database.DBCon.Model(&packages).Select()
- for _, gpackage := range packages {
- if gpackage.PrecedingCommits == 0 {
- logger.Error.Println("Preceding Commits of package " + gpackage.Atom + " is null.")
- logger.Error.Println("This should not happen. Preceding Commits will be set to 1")
- gpackage.PrecedingCommits = 1
- database.DBCon.Model(gpackage).WherePK().Update()
- }
+ database.DBCon.Model(&packages).Where("preceding_commits = 0").Select()
+ if len(packages) == 0 {
+ return
+ }
+
+ logger.Error.Println("Found", len(packages), "packages with preceding commits == 0. This should not happen. Fixing...")
+ for _, pkg := range packages {
+ pkg.PrecedingCommits = 1
}
+ database.DBCon.Model(&packages).Update()
}
// GetApplicationData is used to retrieve the
diff --git a/pkg/portage/utils/git.go b/pkg/portage/utils/git.go
index 2f67c01..3e76c34 100644
--- a/pkg/portage/utils/git.go
+++ b/pkg/portage/utils/git.go
@@ -23,8 +23,7 @@ func AllFiles() []string {
cmd.Dir = config.PortDir()
out, err := cmd.CombinedOutput()
if err != nil {
- logger.Error.Println("ERROR: cmd.Run() failed with:")
- logger.Error.Println(err)
+ logger.Error.Println("cmd.Run() failed with:", err)
return allFiles
}
@@ -45,7 +44,7 @@ func ChangedFiles(startCommit string, endCommit string) []string {
cmd.Dir = config.PortDir()
out, err := cmd.CombinedOutput()
if err != nil {
- logger.Error.Println("ERROR: cmd.Run() failed with %s\n", err)
+ logger.Error.Println("cmd.Run() failed with", err)
return changedFiles
}
@@ -56,10 +55,11 @@ func ChangedFiles(startCommit string, endCommit string) []string {
// GetCommits returns the log message of all commits after
// the given startCommit and before the given endCommit. The
// log message:
-// - uses '%Y-%m-%dT%H:%M:%S%z' as date format
-// - doesn't include merges
-// - doesn't include renames
-// - includes the status of the changed files
+// - uses '%Y-%m-%dT%H:%M:%S%z' as date format
+// - doesn't include merges
+// - doesn't include renames
+// - includes the status of the changed files
+//
// Furthermore the commits are in reverse order.
func GetCommits(startCommit string, endCommit string) []string {
var commits []string
@@ -76,7 +76,7 @@ func GetCommits(startCommit string, endCommit string) []string {
cmd.Dir = config.PortDir()
out, err := cmd.CombinedOutput()
if err != nil {
- logger.Error.Println("cmd.Run() failed with %s\n", err)
+ logger.Error.Println("cmd.Run() failed with", err)
return commits
}
@@ -87,14 +87,14 @@ func GetCommits(startCommit string, endCommit string) []string {
// GetLatestCommit retrieves the latest commit in
// the database and returns the hash of the commit
func GetLatestCommit() string {
- latestCommit, _ := GetLatestCommitAndPreceeding()
+ latestCommit, _ := GetLatestCommitAndPreceding()
return latestCommit
}
-// GetLatestCommitAndPreceeding retrieves the latest
+// GetLatestCommitAndPreceding retrieves the latest
// commit in the database. The hash of the latest commit
// as well as the number of preceding commits is returned
-func GetLatestCommitAndPreceeding() (string, int) {
+func GetLatestCommitAndPreceding() (string, int) {
latestCommit := EmptyTree()
PrecedingCommitsOffset := 0