From c62bd4d5644a77d1224f188d092f39e0438062ec Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Thu, 5 Sep 2019 15:51:34 +0200 Subject: Migrate to ES 7.3 and the repository pattern Elasticsearch-persistence is used as the persistence layer for Ruby domain objects in Elasticsearch in this application. So far, the ActiveRecord pattern has been used here. However, this pattern has been deprecated as of version 6 of the gem and was removed in version 7. That's why the application has been migrated to use the repository pattern instead. For further information, please see: https://www.elastic.co/blog/activerecord-to-repository-changing- persistence-patterns-with-the-elasticsearch-rails-gem Note: The old Elasticsearch index won't be compatible with this version anymore. That's why a fresh index should be populated. Signed-off-by: Max Magorsch --- Gemfile | 4 +- Gemfile.lock | 42 ++--- app/controllers/arches_controller.rb | 12 +- app/controllers/categories_controller.rb | 10 +- app/controllers/concerns/package_update_feeds.rb | 8 +- app/controllers/packages_controller.rb | 14 +- app/controllers/useflags_controller.rb | 12 +- app/helpers/application_helper.rb | 3 + app/jobs/category_update_job.rb | 4 +- app/jobs/package_removal_job.rb | 6 +- app/jobs/package_update_job.rb | 2 +- app/jobs/record_change_job.rb | 2 +- app/jobs/useflags_update_job.rb | 16 +- app/models/category.rb | 41 ++++- app/models/change.rb | 53 +++++-- app/models/package.rb | 80 +++++++--- app/models/useflag.rb | 125 ++++++--------- app/models/version.rb | 67 +++++--- app/repositories/base_repository.rb | 88 +++++++++++ app/repositories/category_repository.rb | 21 +++ app/repositories/change_repository.rb | 23 +++ app/repositories/elasticsearch_client.rb | 13 ++ app/repositories/package_repository.rb | 188 +++++++++++++++++++++++ app/repositories/useflag_repository.rb | 99 ++++++++++++ app/repositories/version_repository.rb | 56 +++++++ app/views/arches/keyworded.html.erb | 2 +- app/views/arches/stable.html.erb | 2 +- app/views/feeds/changes.atom.builder | 2 +- app/views/index/_package.html.erb | 2 +- app/views/index/index.html.erb | 4 +- app/views/packages/_metadata.html.erb | 2 +- app/views/packages/_package_header.html.erb | 2 +- app/views/packages/added.html.erb | 2 +- app/views/packages/keyworded.html.erb | 2 +- app/views/packages/search.html.erb | 6 +- app/views/packages/stable.html.erb | 2 +- app/views/packages/updated.html.erb | 2 +- app/views/useflags/_useflag_result_row.html.erb | 6 +- config/initializers/elasticsearch.rb | 7 +- lib/kkuleomi/store.rb | 27 ++-- lib/kkuleomi/store/model.rb | 78 ---------- lib/kkuleomi/store/models/package_import.rb | 22 +-- lib/kkuleomi/store/models/package_search.rb | 161 ------------------- lib/kkuleomi/store/models/version_import.rb | 4 +- 44 files changed, 819 insertions(+), 505 deletions(-) create mode 100644 app/repositories/base_repository.rb create mode 100644 app/repositories/category_repository.rb create mode 100644 app/repositories/change_repository.rb create mode 100644 app/repositories/elasticsearch_client.rb create mode 100644 app/repositories/package_repository.rb create mode 100644 app/repositories/useflag_repository.rb create mode 100644 app/repositories/version_repository.rb delete mode 100644 lib/kkuleomi/store/model.rb delete mode 100644 lib/kkuleomi/store/models/package_search.rb diff --git a/Gemfile b/Gemfile index cdcddd2..fd7147d 100644 --- a/Gemfile +++ b/Gemfile @@ -23,8 +23,8 @@ gem 'jbuilder', '~> 2.0' gem 'sdoc', '~> 1.0', group: :doc # packages stuff -gem 'elasticsearch-rails', '~> 5.0' -gem 'elasticsearch-persistence', '~> 5.0' +gem 'elasticsearch-rails', '~> 7.0.0' +gem 'elasticsearch-persistence', '~> 7.0.0' gem 'nokogiri' gem 'thin' diff --git a/Gemfile.lock b/Gemfile.lock index 78774c0..14ab1e1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -44,42 +44,32 @@ GEM tzinfo (~> 1.1) arel (9.0.0) ast (2.4.0) - axiom-types (0.1.1) - descendants_tracker (~> 0.0.4) - ice_nine (~> 0.11.0) - thread_safe (~> 0.3, >= 0.3.1) bindex (0.8.1) builder (3.2.3) byebug (11.0.1) - coercible (1.0.0) - descendants_tracker (~> 0.0.1) concurrent-ruby (1.1.5) connection_pool (2.2.2) crass (1.0.4) daemons (1.3.1) - descendants_tracker (0.0.4) - thread_safe (~> 0.3, >= 0.3.1) - elasticsearch (5.0.5) - elasticsearch-api (= 5.0.5) - elasticsearch-transport (= 5.0.5) - elasticsearch-api (5.0.5) + elasticsearch (7.3.0) + elasticsearch-api (= 7.3.0) + elasticsearch-transport (= 7.3.0) + elasticsearch-api (7.3.0) multi_json - elasticsearch-model (5.1.0) + elasticsearch-model (7.0.0) activesupport (> 3) - elasticsearch (~> 5) + elasticsearch (> 1) hashie - elasticsearch-persistence (5.1.0) + elasticsearch-persistence (7.0.0) activemodel (> 4) activesupport (> 4) - elasticsearch (~> 5) - elasticsearch-model (~> 5) + elasticsearch (~> 7) + elasticsearch-model (= 7.0.0) hashie - virtus - elasticsearch-rails (5.1.0) - elasticsearch-transport (5.0.5) + elasticsearch-rails (7.0.0) + elasticsearch-transport (7.3.0) faraday multi_json - equalizer (0.0.11) erubi (1.8.0) eventmachine (1.2.7) execjs (2.7.0) @@ -91,7 +81,6 @@ GEM hashie (3.6.0) i18n (1.6.0) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) jaro_winkler (1.5.3) jbuilder (2.9.1) activesupport (>= 4.2.0) @@ -222,11 +211,6 @@ GEM uglifier (4.1.20) execjs (>= 0.3.0, < 3) unicode-display_width (1.6.0) - virtus (1.0.5) - axiom-types (~> 0.1) - coercible (~> 1.0) - descendants_tracker (~> 0.0, >= 0.0.3) - equalizer (~> 0.0, >= 0.0.9) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) @@ -241,8 +225,8 @@ PLATFORMS DEPENDENCIES byebug - elasticsearch-persistence (~> 5.0) - elasticsearch-rails (~> 5.0) + elasticsearch-persistence (~> 7.0.0) + elasticsearch-rails (~> 7.0.0) jbuilder (~> 2.0) jquery-rails (~> 4.3.5) listen diff --git a/app/controllers/arches_controller.rb b/app/controllers/arches_controller.rb index cbbcb65..c72e378 100644 --- a/app/controllers/arches_controller.rb +++ b/app/controllers/arches_controller.rb @@ -43,9 +43,9 @@ class ArchesController < ApplicationController def keyworded_packages(arch) Rails.cache.fetch("keyworded_packages/#{arch}", expires_in: 10.minutes) do - Change.filter_all({ change_type: 'keyword', arches: arch }, - size: 50, - sort: { created_at: { order: 'desc' } }).map do |change| + ChangeRepository.filter_all({ change_type: 'keyword', arches: arch }, + size: 50, + sort: { created_at: { order: 'desc' } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end @@ -53,9 +53,9 @@ class ArchesController < ApplicationController def stabled_packages(arch) Rails.cache.fetch("stabled_packages/#{arch}", expires_in: 10.minutes) do - Change.filter_all({ change_type: 'stable', arches: arch }, - size: 50, - sort: { created_at: { order: 'desc' } }).map do |change| + ChangeRepository.filter_all({ change_type: 'stable', arches: arch }, + size: 50, + sort: { created_at: { order: 'desc' } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rb index 33817aa..a9c9b06 100644 --- a/app/controllers/categories_controller.rb +++ b/app/controllers/categories_controller.rb @@ -3,15 +3,15 @@ class CategoriesController < ApplicationController before_action :set_nav def index - @categories = Category.all_sorted_by(:name, :asc) + @categories = CategoryRepository.all_sorted_by(:id, :asc) end def show @packages = Rails.cache.fetch("category/#{@category.name}/packages", expires_in: 10.minutes) do - Package.find_all_by(:category, - @category.name, - sort: { name_sort: { order: 'asc' } }).map do |pkg| + PackageRepository.find_all_by(:category, + @category.name, + sort: { name_sort: { order: 'asc' } }).map do |pkg| pkg.to_os(:name, :atom, :description) end end @@ -24,7 +24,7 @@ class CategoriesController < ApplicationController private def set_category - @category = Category.find_by(:name, params[:id]) + @category = CategoryRepository.find_by(:name, params[:id]) fail ActionController::RoutingError, 'No such category' unless @category @title = @category.name diff --git a/app/controllers/concerns/package_update_feeds.rb b/app/controllers/concerns/package_update_feeds.rb index 2d20672..28a951b 100644 --- a/app/controllers/concerns/package_update_feeds.rb +++ b/app/controllers/concerns/package_update_feeds.rb @@ -3,7 +3,7 @@ module PackageUpdateFeeds def new_packages Rails.cache.fetch('new_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :created_at) end end @@ -11,7 +11,7 @@ module PackageUpdateFeeds def version_bumps Rails.cache.fetch('version_bumps', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :created_at) end end @@ -19,7 +19,7 @@ module PackageUpdateFeeds def keyworded_packages Rails.cache.fetch('keyworded_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end @@ -27,7 +27,7 @@ module PackageUpdateFeeds def stabled_packages Rails.cache.fetch('stabled_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 64cb289..67cc86f 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -8,24 +8,24 @@ class PackagesController < ApplicationController def search @offset = params[:o].to_i || 0 - @packages = Package.default_search(params[:q], @offset) + @packages = PackageRepository.default_search(params[:q], @offset) redirect_to package_path(@packages.first).gsub('%2F', '/') if @packages.size == 1 end def suggest - @packages = Package.suggest(params[:q]) + @packages = PackageRepository.suggest(params[:q]) end def resolve - @packages = Package.resolve(params[:atom]) + @packages = PackageRepository.resolve(params[:atom]) end def show - @package = Package.find_by(:atom, params[:id]) + @package = PackageRepository.find_by(:atom, params[:id]) fail ActionController::RoutingError, 'No such package' unless @package - fresh_when etag: @package.updated_at, last_modified: @package.updated_at, public: true + fresh_when etag: Time.parse(@package.updated_at), last_modified: Time.parse(@package.updated_at), public: true # Enable this in 2024 (when we have full-color emojis on a Linux desktop) # @title = ' 📦 %s' % @package.atom @@ -34,10 +34,10 @@ class PackagesController < ApplicationController end def changelog - @package = Package.find_by(:atom, params[:id]) + @package = PackageRepository.find_by(:atom, params[:id]) fail ActionController::RoutingError, 'No such package' unless @package - if stale?(etag: @package.updated_at, last_modified: @package.updated_at, public: true) + if stale?(etag: Time.parse(@package.updated_at), last_modified: Time.parse(@package.updated_at), public: true) @changelog = Rails.cache.fetch("changelog/#{@package.atom}") do Portage::Util::History.for(@package.category, @package.name, 5) end diff --git a/app/controllers/useflags_controller.rb b/app/controllers/useflags_controller.rb index 0fa74f4..9802b78 100644 --- a/app/controllers/useflags_controller.rb +++ b/app/controllers/useflags_controller.rb @@ -6,18 +6,18 @@ class UseflagsController < ApplicationController end def show - @useflags = Useflag.get_flags(params[:id]) + @useflags = UseflagRepository.get_flags(params[:id]) if @useflags.empty? || (@useflags[:use_expand].empty? && @useflags[:local].empty? && @useflags[:global].empty?) fail ActionController::RoutingError, 'No such useflag' end - @packages = Package.find_atoms_by_useflag(params[:id]) + @packages = PackageRepository.find_atoms_by_useflag(params[:id]) @title = '%s – %s' % [params[:id], t(:use_flags)] unless @useflags[:use_expand].empty? @useflag = @useflags[:use_expand].first - @use_expand_flags = Useflag.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix) + @use_expand_flags = UseflagRepository.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix) @use_expand_flag_name = @useflag.use_expand_prefix.upcase render template: 'useflags/show_use_expand' @@ -29,16 +29,16 @@ class UseflagsController < ApplicationController def search # TODO: Different search? - @flags = Useflag.suggest(params[:q]) + @flags = UseflagRepository.suggest(params[:q]) end def suggest - @flags = Useflag.suggest(params[:q]) + @flags = UseflagRepository.suggest(params[:q]) end def popular @popular_useflags = Rails.cache.fetch('popular_useflags', expires_in: 24.hours) do - Version.get_popular_useflags(100) + VersionRepository.get_popular_useflags(100) end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 619582c..8405e59 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -37,6 +37,9 @@ module ApplicationHelper end def i18n_date(date, format = '%a, %e %b %Y %H:%M') + + date = Time.parse(date).utc if date.is_a? String + content_tag :span, l(date, format: format), class: 'kk-i18n-date', diff --git a/app/jobs/category_update_job.rb b/app/jobs/category_update_job.rb index 7443099..e764ad8 100644 --- a/app/jobs/category_update_job.rb +++ b/app/jobs/category_update_job.rb @@ -5,8 +5,8 @@ class CategoryUpdateJob < ApplicationJob category_path, options = args model = Portage::Repository::Category.new(category_path) - category = Category.find_by(:name, model.name) || Category.new - idx_packages = Package.find_all_by(:category, model.name) || [] + category = CategoryRepository.find_by(:name, model.name) || Category.new + idx_packages = PackageRepository.find_all_by(:category, model.name) || [] if category.needs_import? model category.import! model diff --git a/app/jobs/package_removal_job.rb b/app/jobs/package_removal_job.rb index 877ed07..e625b96 100644 --- a/app/jobs/package_removal_job.rb +++ b/app/jobs/package_removal_job.rb @@ -4,11 +4,11 @@ class PackageRemovalJob < ApplicationJob def perform(*args) atom, _options = args - package_doc = Package.find_by(:atom, atom) + package_doc = PackageRepository.find_by(:atom, atom) return if package_doc.nil? - package_doc.versions.each(&:delete) - package_doc.delete + package_doc.versions.each { |v| VersionRepository.delete(v) } + PackageRepository.delete(package_doc) Rails.logger.warn { "Package deleted: #{atom}" } # USE flags are cleaned up by the UseflagsUpdateJob diff --git a/app/jobs/package_update_job.rb b/app/jobs/package_update_job.rb index 55e278f..53a352c 100644 --- a/app/jobs/package_update_job.rb +++ b/app/jobs/package_update_job.rb @@ -4,7 +4,7 @@ class PackageUpdateJob < ApplicationJob def perform(*args) path, options = args package_model = Portage::Repository::Package.new(path) - package_doc = Package.find_by(:atom, package_model.to_cp) || Package.new + package_doc = PackageRepository.find_by(:atom, package_model.to_cp) || Package.new if package_doc.needs_import? package_model package_doc.import!(package_model, options) diff --git a/app/jobs/record_change_job.rb b/app/jobs/record_change_job.rb index 0e6a011..ed5dd5e 100644 --- a/app/jobs/record_change_job.rb +++ b/app/jobs/record_change_job.rb @@ -25,6 +25,6 @@ class RecordChangeJob < ApplicationJob c.change_type = 'removal' end - c.save + ChangeRepository.save(c) end end diff --git a/app/jobs/useflags_update_job.rb b/app/jobs/useflags_update_job.rb index 21145c3..5558d47 100644 --- a/app/jobs/useflags_update_job.rb +++ b/app/jobs/useflags_update_job.rb @@ -10,7 +10,7 @@ class UseflagsUpdateJob < ApplicationJob def update_global(repo) model_flags = repo.global_useflags - index_flags = Useflag.global + index_flags = UseflagRepository.global new_flags = model_flags.keys - index_flags.keys del_flags = index_flags.keys - model_flags.keys @@ -21,24 +21,24 @@ class UseflagsUpdateJob < ApplicationJob flag_doc.name = flag flag_doc.description = model_flags[flag] flag_doc.scope = 'global' - flag_doc.save + UseflagRepository.save(flag_doc) end eql_flags.each do |flag| unless index_flags[flag].description == model_flags[flag] index_flags[flag].description = model_flags[flag] - index_flags[flag].save + UseflagRepository.save(index_flags[flag]) end end del_flags.each do |flag| - index_flags[flag].delete + UseflagRepository.delete(index_flags[flag]) end end def update_use_expand(repo) model_flags = repo.use_expand_flags - index_flags = Useflag.use_expand + index_flags = UseflagRepository.use_expand # Calculate keys only once index_flag_keys = index_flags.keys @@ -55,7 +55,7 @@ class UseflagsUpdateJob < ApplicationJob if index_flag_keys.include? _flag unless index_flags[_flag].description == desc index_flags[_flag].description = desc - index_flags[_flag].save + UseflagRepository.save(index_flags[_flag]) end else # New flag @@ -64,14 +64,14 @@ class UseflagsUpdateJob < ApplicationJob flag_doc.description = desc flag_doc.scope = 'use_expand' flag_doc.use_expand_prefix = variable - flag_doc.save + UseflagRepository.save(flag_doc) end end end # Find and process removed flags flag_status.each_pair do |flag, status| - index_flags[flag].delete unless status + UseflagRepository.delete(index_flags[flag]) unless status end end diff --git a/app/models/category.rb b/app/models/category.rb index f629bde..4e361c1 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -1,12 +1,39 @@ class Category - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations - index_name "categories-#{Rails.env}" + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :name, + :description, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @name + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes - attribute :name, String, mapping: { type: 'keyword' } - attribute :description, String, mapping: { type: 'text' } - attribute :metadata_hash, String, mapping: { type: 'text' } # Determines if the document model needs an update from the repository model # @@ -29,7 +56,7 @@ class Category # @param [Portage::Repository::Category] category_model Input category model def import!(category_model) import(category_model) - save + CategoryRepository.save(self) end # Returns the URL parameter for referencing this package (Rails internal stuff) diff --git a/app/models/change.rb b/app/models/change.rb index 6eaf00c..1793da4 100644 --- a/app/models/change.rb +++ b/app/models/change.rb @@ -1,13 +1,48 @@ class Change - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations - index_name "change-#{Rails.env}" + ATTRIBUTES = [:_id, + :created_at, + :updated_at, + :package, + :category, + :change_type, + :version, + :arches, + :commit] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :package, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes + + # Converts the model to an OpenStruct instance + # + # @param [Array] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end - attribute :package, String, mapping: { type: 'keyword' } - attribute :category, String, mapping: { type: 'keyword' } - attribute :change_type, String, mapping: { type: 'keyword' } - attribute :version, String, mapping: { type: 'keyword' } - attribute :arches, String, mapping: { type: 'keyword' } - attribute :commit, Hash, default: {}, mapping: { type: 'object' } end diff --git a/app/models/package.rb b/app/models/package.rb index 7ad3cbe..11ef135 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -1,31 +1,52 @@ class Package - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations include Kkuleomi::Store::Models::PackageImport - include Kkuleomi::Store::Models::PackageSearch - - index_name "packages-#{Rails.env}" - - raw_fields = { - type: 'keyword' - } - - attribute :category, String, mapping: raw_fields - attribute :name, String, mapping: raw_fields - attribute :name_sort, String, mapping: raw_fields - attribute :atom, String, mapping: raw_fields - attribute :description, String, mapping: { type: 'text' } - attribute :longdescription, String, mapping: { type: 'text' } - attribute :homepage, String, default: [], mapping: raw_fields - attribute :license, String, mapping: raw_fields - attribute :licenses, String, default: [], mapping: raw_fields - attribute :herds, String, default: [], mapping: raw_fields - attribute :maintainers, Array, default: [], mapping: { type: 'object' } - attribute :useflags, Hash, default: {}, mapping: { type: 'object' } - attribute :metadata_hash, String, mapping: raw_fields + + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :category, + :name, + :name_sort, + :atom, + :description, + :longdescription, + :homepage, + :license, + :licenses, + :herds, + :maintainers, + :useflags, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @atom + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes def category_model - @category_model ||= Category.find_by(:name, category) + @category_model ||= CategoryRepository.find_by(:name, category) end def to_param @@ -44,7 +65,7 @@ class Package end def versions - @versions ||= Version.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } }) + @versions ||= VersionRepository.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } }) end def latest_version @@ -65,6 +86,15 @@ class Package maintainers.empty? && herds.empty? end + # Converts the model to an OpenStruct instance + # + # @param [Array] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end + private # Splits a license string into single licenses, stripping the permitted logic constructs diff --git a/app/models/useflag.rb b/app/models/useflag.rb index 131a89c..12758cb 100644 --- a/app/models/useflag.rb +++ b/app/models/useflag.rb @@ -1,14 +1,41 @@ class Useflag - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model - - index_name "useflags-#{Rails.env}" + include ActiveModel::Model + include ActiveModel::Validations + + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :name, + :description, + :atom, + :scope, + :use_expand_prefix] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end - attribute :name, String, mapping: { type: 'keyword' } - attribute :description, String, mapping: { type: 'text' } - attribute :atom, String, mapping: { type: 'keyword' } - attribute :scope, String, mapping: { type: 'keyword' } - attribute :use_expand_prefix, String, mapping: { type: 'keyword' } + def attributes + @id = @name + '-' + (@atom || 'global' ) + '-' + @scope + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes def all_fields [:name, :description, :atom, :scope, :use_expand_prefix] @@ -22,78 +49,14 @@ class Useflag name.gsub(use_expand_prefix + '_', '') end - class << self - # Retrieves all flags sorted by their state - def get_flags(name) - result = { local: {}, global: [], use_expand: [] } - - find_all_by(:name, name).each do |flag| - case flag.scope - when 'local' - result[:local][flag.atom] = flag - when 'global' - result[:global] << flag - when 'use_expand' - result[:use_expand] << flag - end - end - - result - end - - def suggest(q) - results = Useflag.search( - size: 20, - query: { match_phrase_prefix: { name: q } } - ) - - processed_results = {} - results.each do |result| - if processed_results.key? result.name - processed_results[result.name] = { - name: result.name, - description: '(multiple definitions)', - scope: 'multi' - } - else - processed_results[result.name] = result - end - end - - processed_results.values.sort { |a, b| a[:name].length <=> b[:name].length } - end - - # Loads the local USE flags for a given package in a name -> model hash - # - # @param [String] atom Package to find flags for - # @return [Hash] - def local_for(atom) - map_by_name find_all_by(:atom, atom) - end - - # Maps the global USE flags in the index by their name - # This is expensive! - # - def global - map_by_name find_all_by(:scope, 'global') - end - - # Maps the USE_EXPAND variables in the index by their name - # - def use_expand - map_by_name find_all_by(:scope, 'use_expand') - end - - private + # Converts the model to a Hash + # + # @param [Array] fields Fields to export into the Hash, or all fields if nil + # @return [Hash] Hash containing the selected fields + def to_hsh(*fields) + fields = all_fields if fields.empty? + Hash[fields.map { |field| [field, send(field)] }] + end - def map_by_name(collection) - map = {} - collection.each do |item| - map[item.name] = item - end - - map - end - end end diff --git a/app/models/version.rb b/app/models/version.rb index 62c72f8..3629b98 100644 --- a/app/models/version.rb +++ b/app/models/version.rb @@ -1,23 +1,52 @@ +require 'date' + class Version - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations include Kkuleomi::Store::Models::VersionImport - index_name "versions-#{Rails.env}" - - attribute :version, String, mapping: { type: 'keyword' } - attribute :package, String, mapping: { type: 'keyword' } - attribute :atom, String, mapping: { type: 'keyword' } - attribute :sort_key, Integer, mapping: { type: 'integer' } - attribute :slot, String, mapping: { type: 'keyword' } - attribute :subslot, String, mapping: { type: 'keyword' } - attribute :eapi, String, mapping: { type: 'keyword' } - attribute :keywords, String, mapping: { type: 'keyword' } - attribute :masks, Array, default: [], mapping: { type: 'object' } - attribute :use, String, default: [], mapping: { type: 'keyword' } - attribute :restrict, String, default: [], mapping: { type: 'keyword' } - attribute :properties, String, default: [], mapping: { type: 'keyword' } - attribute :metadata_hash, String, mapping: { type: 'keyword' } + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :version, + :package, + :atom, + :sort_key, + :slot, + :subslot, + :eapi, + :keywords, + :masks, + :use, + :restrict, + :properties, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :version, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @atom + @created_at ||= DateTime.now + @updated_at = DateTime.now + + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes # Returns the keywording state on a given architecture # @@ -138,14 +167,14 @@ class Version def calc_useflags result = { local: {}, global: {}, use_expand: {} } - local_flag_map = Useflag.local_for(atom.gsub("-#{version}", '')) + local_flag_map = UseflagRepository.local_for(atom.gsub("-#{version}", '')) local_flags = local_flag_map.keys use.sort.each do |flag| if local_flags.include? flag result[:local][flag] = local_flag_map[flag].to_hsh else - useflag = Useflag.find_by(:name, flag) + useflag = UseflagRepository.find_by(:name, flag) # This should not happen, but let's be sure next unless useflag diff --git a/app/repositories/base_repository.rb b/app/repositories/base_repository.rb new file mode 100644 index 0000000..7154691 --- /dev/null +++ b/app/repositories/base_repository.rb @@ -0,0 +1,88 @@ +require 'forwardable' +require 'singleton' + +class BaseRepository + include Elasticsearch::Persistence::Repository + include Elasticsearch::Persistence::Repository::DSL + include Singleton + + client ElasticsearchClient.default + + class << self + extend Forwardable + def_delegators :instance, :find_all_by, :filter_all, :find_by, :find_all_by_parent, :all_sorted_by + def_delegators :instance, :count, :search, :delete, :save, :refresh_index!, :create_index + end + + # Finds instances by exact IDs using the 'term' filter + def find_all_by(field, value, opts = {}) + search({ + size: 10_000, + query: { match: { field => value } } + }.merge(opts)) + end + + # Filter all instances by the given parameters + def filter_all(filters, opts = {}) + filter_args = [] + filters.each_pair { |field, value| filter_args << { term: { field => value } } } + + search({ + query: { + bool: { filter: { bool: { must: filter_args } } } + }, + size: 10_000 + }.merge(opts)) + end + + def find_by(field, value, opts = {}) + find_all_by(field, value, opts).first + end + + def find_all_by_parent(parent, opts = {}) + search(opts.merge( + size: 10_000, + query: { + bool: { + filter: { + has_parent: { + parent_type: parent.class.document_type, + query: { term: { _id: parent.id } } + } + }, + must: { + match_all: {} + } + } + }) + ) + end + + # Returns all (by default 10k) records of this class sorted by a field. + def all_sorted_by(field, order, options = {}) + search({ + size: 10_000, + query: { match_all: {} }, + sort: { field => { order: order } } + }.merge(options)) + end + + # Converts the model to an OpenStruct instance + # + # @param [Array] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end + + # Converts the model to a Hash + # + # @param [Array] fields Fields to export into the Hash, or all fields if nil + # @return [Hash] Hash containing the selected fields + def to_hsh(*fields) + fields = all_fields if fields.empty? + Hash[fields.map { |field| [field, send(field)] }] + end + +end \ No newline at end of file diff --git a/app/repositories/category_repository.rb b/app/repositories/category_repository.rb new file mode 100644 index 0000000..e9cf033 --- /dev/null +++ b/app/repositories/category_repository.rb @@ -0,0 +1,21 @@ +require 'singleton' + +class CategoryRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "categories-#{Rails.env}" + + klass Category + + mapping do + indexes :id, type: 'keyword' + indexes :name, type: 'text' + indexes :description, type: 'text' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + +end diff --git a/app/repositories/change_repository.rb b/app/repositories/change_repository.rb new file mode 100644 index 0000000..e5cc2f2 --- /dev/null +++ b/app/repositories/change_repository.rb @@ -0,0 +1,23 @@ +require 'singleton' + +class ChangeRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "change-#{Rails.env}" + + klass Change + + mapping do + indexes :package, type: 'keyword' + indexes :category, type: 'keyword' + indexes :change_type, type: 'keyword' + indexes :version, type: 'keyword' + indexes :arches, type: 'keyword' + indexes :commit, type: 'object' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + +end diff --git a/app/repositories/elasticsearch_client.rb b/app/repositories/elasticsearch_client.rb new file mode 100644 index 0000000..88de0c8 --- /dev/null +++ b/app/repositories/elasticsearch_client.rb @@ -0,0 +1,13 @@ +class ElasticsearchClient + + def self.default + @default ||= Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' + end + + private + + def initialize(*) + raise "Should not be initialiazed" + end + +end \ No newline at end of file diff --git a/app/repositories/package_repository.rb b/app/repositories/package_repository.rb new file mode 100644 index 0000000..ed77afb --- /dev/null +++ b/app/repositories/package_repository.rb @@ -0,0 +1,188 @@ +require 'forwardable' +require 'singleton' + +class PackageRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :suggest, :resolve, :find_atoms_by_useflag, :default_search_size, :default_search, + :build_query, :match_wildcard, :match_phrase, :match_description, :match_category, :scoring_functions + end + + index_name "packages-#{Rails.env}" + + klass Package + + mapping do + indexes :category, type: 'keyword' + indexes :name, type: 'keyword' + indexes :name_sort, type: 'keyword' + indexes :atom, type: 'keyword' + indexes :description, type: 'text' + indexes :longdescription, type: 'text' + indexes :homepage, type: 'keyword' + indexes :license, type: 'keyword' + indexes :licenses, type: 'keyword' + indexes :herds, type: 'keyword' + indexes :maintainers, type: 'object' + indexes :useflags, type: 'object' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + def suggest(q) + PackageRepository.search( + size: 20, + query: { + wildcard: { + name_sort: { + wildcard: q.downcase + '*' + } + } + } + ) + end + + # Tries to resolve a query atom to one or more packages + def resolve(atom) + [] if atom.nil? || atom.empty? + + PackageRepository.find_all_by(:atom, atom) + PackageRepository.find_all_by(:name, atom) + end + + # Searches the versions index for versions using a certain USE flag. + # Results are aggregated by package atoms. + def find_atoms_by_useflag(useflag) + VersionRepository.search( + size: 0, # collect all packages. + query: { + bool: { + must: { match_all: {} }, + filter: { term: { use: useflag } } + } + }, + aggs: { + group_by_package: { + terms: { + field: 'package', + order: { '_key' => 'asc' }, + # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html + # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see. + # Other limits in this app are also 10k mostly to 'make things fit kinda'. + size: 10000, + } + } + }, + ).response.aggregations['group_by_package'].buckets + end + + def default_search_size + 25 + end + + def default_search(q, offset) + return [] if q.nil? || q.empty? + + part1, part2 = q.split('/', 2) + + if part2.nil? + search(build_query(part1, nil, default_search_size, offset)) + else + search(build_query(part2, part1, default_search_size, offset)) + end + end + + def build_query(q, category, size, offset) + { + size: size, + from: offset, + query: { + function_score: { + query: { bool: bool_query_parts(q, category) }, + functions: scoring_functions + } + } + } + end + + def bool_query_parts(q, category = nil) + q_dwncsd = q.downcase + + query = { + must: [ + match_wildcard(q_dwncsd) + ], + should: [ + match_phrase(q_dwncsd), + match_description(q) + ] + } + + query[:must] << [match_category(category)] if category + + query + end + + def match_wildcard(q) + q = ('*' + q + '*') unless q.include? '*' + q.tr!(' ', '*') + + { + wildcard: { + name_sort: { + wildcard: q, + boost: 4 + } + } + } + end + + def match_phrase(q) + { + match_phrase: { + name: { + query: q, + boost: 5 + } + } + } + end + + def match_description(q) + { + match: { + description: { + query: q, + boost: 0.1 + } + } + } + end + + def match_category(cat) + { + match: { + category: { + query: cat, + boost: 2 + } + } + } + end + + def scoring_functions + [ + { + filter: { + term: { + category: 'virtual' + } + }, + weight: 0.6 + } + ] + end + +end diff --git a/app/repositories/useflag_repository.rb b/app/repositories/useflag_repository.rb new file mode 100644 index 0000000..26328f4 --- /dev/null +++ b/app/repositories/useflag_repository.rb @@ -0,0 +1,99 @@ +require 'singleton' + +class UseflagRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :get_flags, :suggest, :local_for, :global, :use_expand + end + + index_name "useflags-#{Rails.env}" + + klass Useflag + + mapping do + indexes :name, type: 'text' + indexes :description, type: 'text' + indexes :atom, type: 'keyword' + indexes :scope, type: 'keyword' + indexes :use_expand_prefix, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + + # Retrieves all flags sorted by their state + def get_flags(name) + result = { local: {}, global: [], use_expand: [] } + + find_all_by(:name, name).each do |flag| + case flag.scope + when 'local' + result[:local][flag.atom] = flag + when 'global' + result[:global] << flag + when 'use_expand' + result[:use_expand] << flag + end + end + + result + end + + def suggest(q) + results = search( + size: 20, + query: { match_phrase_prefix: { name: q } } + ) + + processed_results = {} + results.each do |result| + if processed_results.key? result.name + processed_results[result.name] = { + name: result.name, + description: '(multiple definitions)', + scope: 'multi' + } + else + processed_results[result.name] = result + end + end + + processed_results.values.sort { |a, b| a.name.length <=> b.name.length } + end + + # Loads the local USE flags for a given package in a name -> model hash + # + # @param [String] atom Package to find flags for + # @return [Hash] + def local_for(atom) + map_by_name find_all_by(:atom, atom) + end + + # Maps the global USE flags in the index by their name + # This is expensive! + # + def global + map_by_name find_all_by(:scope, 'global') + end + + # Maps the USE_EXPAND variables in the index by their name + # + def use_expand + map_by_name find_all_by(:scope, 'use_expand') + end + + private + + def map_by_name(collection) + map = {} + + collection.each do |item| + map[item.name] = item + end + + map + end + +end diff --git a/app/repositories/version_repository.rb b/app/repositories/version_repository.rb new file mode 100644 index 0000000..43168d2 --- /dev/null +++ b/app/repositories/version_repository.rb @@ -0,0 +1,56 @@ +require 'singleton' + +class VersionRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :get_popular_useflags + end + + index_name "versions-#{Rails.env}" + + klass Version + + mapping do + indexes :version, type: 'keyword' + indexes :package, type: 'keyword' + indexes :atom, type: 'keyword' + indexes :sort_key, type: 'integer' + indexes :slot, type: 'keyword' + indexes :subslot, type: 'keyword' + indexes :eapi, type: 'keyword' + indexes :keywords, type: 'keyword' + indexes :masks do + indexes :arches, type: 'keyword' + indexes :atoms, type: 'keyword' + indexes :author, type: 'keyword' + indexes :date, type: 'keyword' + indexes :reason, type: 'text' + end + indexes :use, type: 'keyword' + indexes :restrict, type: 'keyword' + indexes :properties, type: 'keyword' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + # Retrieves the most widely used USE flags by all versions + # Note that packages with many versions are over-represented + def get_popular_useflags(n = 50) + search( + query: { match_all: {} }, + aggs: { + group_by_flag: { + terms: { + field: 'use', + size: n + } + } + }, + size: 0 + ).response.aggregations['group_by_flag'].buckets + end + +end diff --git a/app/views/arches/keyworded.html.erb b/app/views/arches/keyworded.html.erb index b7ae03d..ae1df29 100644 --- a/app/views/arches/keyworded.html.erb +++ b/app/views/arches/keyworded.html.erb @@ -12,7 +12,7 @@ <% cache("keyworded-full-#{@arch}-#{@changes.hash}") do %>
    <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %>
diff --git a/app/views/arches/stable.html.erb b/app/views/arches/stable.html.erb index b1a4548..eb66245 100644 --- a/app/views/arches/stable.html.erb +++ b/app/views/arches/stable.html.erb @@ -12,7 +12,7 @@ <% cache("stable-full-#{@arch}-#{@changes.hash}") do %>
    <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %>
diff --git a/app/views/feeds/changes.atom.builder b/app/views/feeds/changes.atom.builder index 5991f45..a8af3df 100644 --- a/app/views/feeds/changes.atom.builder +++ b/app/views/feeds/changes.atom.builder @@ -10,7 +10,7 @@ atom_feed(id: atom_id(@feed_type, @feed_id, 'feed')) do |feed| @changes.each do |change| atom = cp_to_atom change.category, change.package - package = Package.find_by :atom, atom + package = PackageRepository.find_by :atom, atom if package.nil? logger.warn "Package for change (#{change}) nil!" next diff --git a/app/views/index/_package.html.erb b/app/views/index/_package.html.erb index eeb3109..a364209 100644 --- a/app/views/index/_package.html.erb +++ b/app/views/index/_package.html.erb @@ -1,4 +1,4 @@ -<%- package = Package.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%> +<%- package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%> diff --git a/app/views/index/index.html.erb b/app/views/index/index.html.erb index 890a5f3..af86c9e 100644 --- a/app/views/index/index.html.erb +++ b/app/views/index/index.html.erb @@ -1,5 +1,5 @@
-

Welcome to the Home of <%= number_with_delimiter Package.count %> Gentoo Packages

+

Welcome to the Home of <%= number_with_delimiter PackageRepository.count %> Gentoo Packages

@@ -43,7 +43,7 @@
    <% @version_bumps.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %>
diff --git a/app/views/packages/_metadata.html.erb b/app/views/packages/_metadata.html.erb index 426afd9..5568c08 100644 --- a/app/views/packages/_metadata.html.erb +++ b/app/views/packages/_metadata.html.erb @@ -3,7 +3,7 @@

<%= t :box_metadata %>