# This file has parts from Buildbot and is modifyed by Gentoo Authors. # Buildbot is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright Buildbot Team Members # Origins: buildbot.db.model.py # Modifyed by Gentoo Authors. # Copyright 2024 Gentoo Authors import uuid import alembic import alembic.config import sqlalchemy as sa from twisted.internet import defer from twisted.python import log from twisted.python import util from buildbot.db import base from buildbot.db.migrate_utils import test_unicode from buildbot.db.types.json import JsonObject from buildbot.util import sautils class UpgradeFromBefore0p9Error(Exception): def __init__(self): message = """You are trying to upgrade a buildbot 0.8.x master to buildbot 0.9.x or newer. This is not supported. Please start from a clean database http://docs.buildbot.net/latest/manual/upgrading/0.9-upgrade.html""" # Call the base class constructor with the parameters it needs super().__init__(message) class UpgradeFromBefore3p0Error(Exception): def __init__(self): message = """You are trying to upgrade to Buildbot 3.0 or newer from Buildbot 2.x or older. This is only supported via an intermediate upgrade to newest Buildbot 2.10.x that is available. Please first upgrade to 2.10.x and then try to upgrade to this version. http://docs.buildbot.net/latest/manual/upgrading/3.0-upgrade.html""" super().__init__(message) class Model(base.DBConnectorComponent): property_name_length = 256 property_source_length = 256 hash_length = 40 # # schema # metadata = sa.MetaData() # NOTES # * server_defaults here are included to match those added by the migration # scripts, but they should not be depended on - all code accessing these # tables should supply default values as necessary. The defaults are # required during migration when adding non-nullable columns to existing # tables. # # * dates are stored as unix timestamps (UTC-ish epoch time) # # Tables related to gentoo-ci-cloud # ------------------------- repositorys = sautils.Table( "repositorys", metadata, # unique id per repository sa.Column('uuid', sa.String(36), primary_key=True, default=lambda: str(uuid.uuid4()), ), # repository's name sa.Column('name', sa.String(255), nullable=False), # description of the repository sa.Column('description', sa.Text, nullable=True), sa.Column('url', sa.String(255), nullable=True), sa.Column('branch', sa.String(255), nullable=True), sa.Column('type', sa.Enum('git', 'gitlab'), nullable=False, default='git'), sa.Column('mode', sa.Enum('full', 'incremental'), nullable=False, default='incremental'), sa.Column('method', sa.Enum('clobber', 'fresh', 'clean', 'copy'), nullable=False, default='fresh'), sa.Column('shallow', sa.Integer, nullable=False, default=0), sa.Column('alwaysuselatest', sa.Boolean, default=False), sa.Column('auto', sa.Boolean, default=False), sa.Column('enabled', sa.Boolean, default=False), sa.Column('ebuild', sa.Boolean, default=True), sa.Column('merge', sa.Boolean, default=False), # filenames for SecretInAFile sa.Column('sshprivatekey', sa.String(50), nullable=True), sa.Column('sshhostkey', sa.String(50), nullable=True), ) # Use by GitPoller repositorys_gitpullers = sautils.Table( "repositorys_gitpullers", metadata, # unique id per repository sa.Column('id', sa.Integer, primary_key=True), sa.Column('repository_uuid', sa.String(36), sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), nullable=False), sa.Column('project', sa.String(255), nullable=False, default='gentoo'), sa.Column('url', sa.String(255), nullable=False), sa.Column('branches', sa.String(255), nullable=False, default='all'), sa.Column('poll_interval', sa.Integer, nullable=False, default=600), sa.Column('poll_random_delay_min', sa.Integer, nullable=False, default=600), sa.Column('poll_random_delay_max', sa.Integer, nullable=False, default=600), sa.Column('updated_at', sa.Integer, nullable=True), ) projects = sautils.Table( "projects", metadata, # unique id per project sa.Column('uuid', sa.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), # project's name sa.Column('name', sa.String(255), nullable=False), # project name on gitlab/github sa.Column('git_project_name', sa.String(255), nullable=False), # description of the project sa.Column('description', sa.Text, nullable=True), sa.Column('profile', sa.String(255), nullable=False), sa.Column('profile_repository_uuid', sa.String(36), sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), nullable=False), sa.Column('settings_repository_uuid', sa.String(36), sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), nullable=False), sa.Column('keyword_id', sa.Integer, sa.ForeignKey('keywords.id', ondelete='CASCADE'), nullable=False), sa.Column('image', sa.String(255), nullable=False), sa.Column('status', sa.Enum('stable','unstable','all'), nullable=False), sa.Column('auto', sa.Boolean, default=False), sa.Column('enabled', sa.Boolean, default=False), sa.Column('use_default', sa.Boolean, default=True), sa.Column('created_by', sa.Integer, sa.ForeignKey('users.uid', ondelete='CASCADE'), nullable=False), ) # What repository's use by projects projects_repositorys = sautils.Table( "projects_repositorys", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('repository_uuid', sa.String(36), sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), nullable=False), sa.Column('auto', sa.Boolean, default=False), sa.Column('pkgcheck', sa.Enum('package','full','none'), default='none'), sa.Column('build', sa.Boolean, default=False), sa.Column('test', sa.Boolean, default=False), sa.Column('test_mr', sa.Boolean, default=False), ) # projects etc/portage settings projects_portage = sautils.Table( "projects_portage", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('directorys', sa.Enum('make.profile', 'repos.conf'), nullable=False), sa.Column('value', sa.String(255), nullable=False), ) portages_makeconf = sautils.Table( "portages_makeconf", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('variable', sa.String(255), nullable=False), ) projects_portages_makeconf = sautils.Table( "projects_portages_makeconf", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('makeconf_id', sa.String(255), sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'), nullable=False), sa.Column('value', sa.String(255), nullable=False), ) # projects etc/portage/env settings projects_portages_env = sautils.Table( "projects_portages_env", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('makeconf_id', sa.String(255), sa.ForeignKey('portages_makeconf.id', ondelete='CASCADE'), nullable=False), sa.Column('name', sa.String(255), nullable=False), sa.Column('value', sa.String(255), nullable=False), ) # projects etc/portage/package.* settings projects_portages_package = sautils.Table( "projects_portages_package", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('directory', sa.Enum('use', 'accept_keywords', 'env', 'exclude'), nullable=False), sa.Column('package', sa.String(255), nullable=False), sa.Column('value', sa.String(255), nullable=True), ) projects_emerge_options = sautils.Table( "projects_emerge_options", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('oneshot', sa.Boolean, default=True), sa.Column('depclean', sa.Boolean, default=True), sa.Column('preserved_libs', sa.Boolean, default=True), ) projects_builds = sautils.Table( "projects_builds", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('build_id', sa.Integer), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('version_uuid', sa.String(36), sa.ForeignKey('versions.uuid', ondelete='CASCADE'), nullable=False), sa.Column('buildbot_build_id', sa.Integer), sa.Column('bug_id', sa.Integer, nullable=False, default=0), sa.Column('status', sa.Enum('failed','completed','in-progress','waiting', 'warning'), nullable=False), sa.Column('requested', sa.Boolean, default=False), sa.Column('created_at', sa.Integer, nullable=True), sa.Column('updated_at', sa.Integer, nullable=True), sa.Column('deleted', sa.Boolean, default=False), sa.Column('deleted_at', sa.Integer, nullable=True), ) projects_pattern = sautils.Table( "projects_pattern", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('search', sa.String(50), nullable=False), sa.Column('start', sa.Integer, default=0), sa.Column('end', sa.Integer, default=0), sa.Column('status', sa.Enum('info', 'warning', 'ignore', 'error'), default='info'), sa.Column('type', sa.Enum('info', 'qa', 'compile', 'configure', 'install', 'postinst', 'prepare', 'pretend', 'setup', 'test', 'unpack', 'ignore', 'issues', 'misc', 'elog'), default='info'), sa.Column('search_type', sa.Enum('in', 'startswith', 'endswith', 'search'), default='in'), ) projects_workers = sautils.Table( "projects_workers", metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('project_uuid', sa.String(36), sa.ForeignKey('projects.uuid', ondelete='CASCADE'), nullable=False), sa.Column('worker_uuid', sa.String(36), sa.ForeignKey('workers.uuid', ondelete='CASCADE'), nullable=False), ) keywords = sautils.Table( "keywords", metadata, # unique uuid per keyword sa.Column('id', sa.Integer, primary_key=True), # project's name sa.Column('name', sa.String(255), nullable=False), ) categorys = sautils.Table( "categorys", metadata, sa.Column('uuid', sa.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sa.Column('name', sa.String(255), nullable=False), ) packages = sautils.Table( "packages", metadata, sa.Column('uuid', sa.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sa.Column('name', sa.String(255), nullable=False), sa.Column('category_uuid', sa.String(36), sa.ForeignKey('categorys.uuid', ondelete='CASCADE'), nullable=False), sa.Column('repository_uuid', sa.String(36), sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'), nullable=False), sa.Column('deleted', sa.Boolean, default=False), sa.Column('deleted_at', sa.Integer, nullable=True), ) emails = sautils.Table( "emails", metadata, # unique uuid per keyword sa.Column('id', sa.Integer, primary_key=True), sa.Column('email', sa.String(255), nullable=False), ) packages_emails = sautils.Table( "packages_emails", metadata, # unique uuid per keyword sa.Column('id', sa.Integer, primary_key=True), sa.Column('package_uuid', sa.String(36), sa.ForeignKey('packages.uuid', ondelete='CASCADE')), sa.Column('email_id', sa.Integer, sa.ForeignKey('emails.id', ondelete='CASCADE')), sa.Column('mail_type', sa.Enum('project', 'person', 'unknown',), nullable=False, default='unknown'), sa.Column('proxied', sa.Enum('yes', 'no', 'proxy',), nullable=False, default='no'), ) packages_metadata = sautils.Table( "packages_metadata", metadata, # unique uuid per keyword sa.Column('id', sa.Integer, primary_key=True), sa.Column('package_uuid', sa.String(36), sa.ForeignKey('packages.uuid', ondelete='CASCADE')), sa.Column('sha256', sa.String(255), nullable=False, default='0'), ) versions = sautils.Table( "versions", metadata, sa.Column('uuid', sa.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sa.Column('name', sa.String(255), nullable=False), sa.Column('package_uuid', sa.String(36), sa.ForeignKey('packages.uuid', ondelete='CASCADE'), nullable=False), sa.Column('file_hash', sa.String(255), nullable=False), sa.Column('commit_id', sa.String(255), nullable=False), sa.Column('change_id', sa.Integer, nullable=True, default=0), sa.Column('deleted', sa.Boolean, default=False), sa.Column('deleted_at', sa.Integer, nullable=True), ) versions_keywords = sautils.Table( "versions_keywords", metadata, # unique id per project sa.Column('uuid', sa.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), # project's name sa.Column('keyword_id', sa.Integer, sa.ForeignKey('keywords.id', ondelete='CASCADE')), sa.Column('version_uuid', sa.String(36), sa.ForeignKey('versions.uuid', ondelete='CASCADE')), sa.Column('status', sa.Enum('stable','unstable','negative','all'), nullable=False), ) versions_metadata = sautils.Table( "versions_metadata", metadata, # unique id per project sa.Column('id', sa.Integer, primary_key=True), sa.Column('version_uuid', sa.String(36), sa.ForeignKey('versions.uuid', ondelete='CASCADE')), sa.Column('metadata', sa.Enum('restrict', 'properties', 'iuse', 'required use', 'keyword'), nullable=False), sa.Column('value', sa.String(255), nullable=False), ) workers = sautils.Table( "workers", metadata, # unique id per project sa.Column('uuid', sa.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sa.Column('type', sa.Enum('log','build'), nullable=False), sa.Column('enable', sa.Boolean, default=False), sa.Column('flavor_id', sa.Integer, sa.ForeignKey('workers_flavors.id', ondelete='CASCADE')), sa.Column('image_id', sa.Integer, sa.ForeignKey('workers_images.id', ondelete='CASCADE')), ) workers_flavors = sautils.Table( "workers_flavors", metadata, # unique id per project sa.Column('id', sa.Integer, primary_key=True), sa.Column('name', sa.String(255), nullable=False), sa.Column('ram', sa.Integer), sa.Column('cpu', sa.Integer), ) workers_images = sautils.Table( "workers_images", metadata, # unique id per project sa.Column('id', sa.Integer, primary_key=True), sa.Column('name', sa.String(255), nullable=False), sa.Column('tag', sa.String(255), nullable=False), sa.Column('dockerfile', sa.String(255), nullable=False), sa.Column('bootstrap_tag', sa.String(255), nullable=False), sa.Column('type', sa.Enum('docker',), nullable=False), ) workers_configs = sautils.Table( "workers_configs", metadata, # unique id per project sa.Column('id', sa.Integer, primary_key=True), sa.Column('worker_uuid', sa.String(36), sa.ForeignKey('workers.uuid', ondelete='CASCADE')), sa.Column('type', sa.Enum('cap_add','security_opt'), nullable=False), sa.Column('value', sa.String(255), nullable=False), ) nodes = sautils.Table( "nodes", metadata, # unique id per project sa.Column('uuid', sa.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sa.Column('name', sa.String(255), nullable=False), sa.Column('docker_host_url', sa.String(255), nullable=False), sa.Column('bin_host_url', sa.String(255), nullable=False), sa.Column('enable', sa.Boolean, default=False), ) nodes_workers = sautils.Table( "nodes_workers", metadata, # unique id per project sa.Column('id', sa.Integer, primary_key=True), sa.Column('node_uuid', sa.String(36), sa.ForeignKey('nodes.uuid', ondelete='CASCADE')), sa.Column('worker_uuid', sa.String(36), sa.ForeignKey('workers.uuid', ondelete='CASCADE')), ) # Tables related to users # ----------------------- # This table identifies individual users, and contains buildbot-specific # information about those users. users = sautils.Table( "users", metadata, # unique user id number sa.Column("uid", sa.Integer, primary_key=True), # identifier (nickname) for this user; used for display sa.Column("email", sa.String(255), nullable=False), # username portion of user credentials for authentication sa.Column("bb_username", sa.String(128)), # password portion of user credentials for authentication sa.Column("bb_password", sa.String(128)), ) # Indexes # ------- # MySQL creates indexes for foreign keys, and these appear in the # reflection. This is a list of (table, index) names that should be # expected on this platform implied_indexes = [ ] # Migration support # ----------------- # Buildbot has historically used 3 database migration systems: # - homegrown system that used "version" table to track versions # - SQLAlchemy-migrate that used "migrate_version" table to track versions # - alembic that uses "alembic_version" table to track versions (current) # We need to detect each case and tell the user how to upgrade. config_path = util.sibpath(__file__, "migrations/alembic.ini") def table_exists(self, conn, table): try: r = conn.execute(f"select * from {table} limit 1") r.close() return True except Exception: return False def migrate_get_version(self, conn): r = conn.execute("select version from migrate_version limit 1") version = r.scalar() r.close() return version def alembic_get_scripts(self): alembic_config = alembic.config.Config(self.config_path) return alembic.script.ScriptDirectory.from_config(alembic_config) def alembic_stamp(self, conn, alembic_scripts, revision): context = alembic.runtime.migration.MigrationContext.configure(conn) context.stamp(alembic_scripts, revision) @defer.inlineCallbacks def is_current(self): def thd(conn): if not self.table_exists(conn, 'alembic_version'): return False alembic_scripts = self.alembic_get_scripts() current_script_rev_head = alembic_scripts.get_current_head() context = alembic.runtime.migration.MigrationContext.configure(conn) current_rev = context.get_current_revision() return current_rev == current_script_rev_head ret = yield self.db.pool.do(thd) return ret # returns a Deferred that returns None def create(self): # this is nice and simple, but used only for tests def thd(engine): self.metadata.create_all(bind=engine) return self.db.pool.do_with_engine(thd) @defer.inlineCallbacks def upgrade(self): # the upgrade process must run in a db thread def thd(conn): alembic_scripts = self.alembic_get_scripts() current_script_rev_head = alembic_scripts.get_current_head() #if self.table_exists(conn, 'version'): # raise UpgradeFromBefore0p9Error() if self.table_exists(conn, 'migrate_version'): version = self.migrate_get_version(conn) #if version < 40: # raise UpgradeFromBefore0p9Error() last_sqlalchemy_migrate_version = 0 if version != last_sqlalchemy_migrate_version: raise UpgradeFromBefore3p0Error() self.alembic_stamp(conn, alembic_scripts, alembic_scripts.get_base()) conn.execute('drop table migrate_version') if not self.table_exists(conn, 'alembic_version'): log.msg("Initializing empty database") # Do some tests first test_unicode(conn) Model.metadata.create_all(conn) self.alembic_stamp(conn, alembic_scripts, current_script_rev_head) return context = alembic.runtime.migration.MigrationContext.configure(conn) current_rev = context.get_current_revision() if current_rev == current_script_rev_head: log.msg('Upgrading database: the current database schema is already the newest') return log.msg('Upgrading database') with sautils.withoutSqliteForeignKeys(conn): with context.begin_transaction(): context.run_migrations() log.msg('Upgrading database: done') yield self.db.pool.do(thd)