Add support for workes, nodes, images and flavors in db

Signed-off-by: Magnus Granberg <zorry@gentoo.org>

5 files changed, 298 insertions, 58 deletions

diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index 82552fa..edcd0c1 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -1,11 +1,24 @@
-# Copyright 2021 Gentoo Authors
+# Copyright 2022 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from twisted.internet import defer
 
 from buildbot.plugins import util
 from buildbot_gentoo_ci.config import buildfactorys
-from buildbot_gentoo_ci.config.workers import gentoo_ci_workers
+
+def getWorkersLocal(worker_type, workers):
+        worker_list = []
+        for worker in workers:
+            if worker['type'] == worker_type and worker['enable'] is True:
+                worker_list.append(worker['name'])
+        return worker_list
+
+def getWorkersDocker(worker_type, workers):
+        worker_list = []
+        for worker in workers:
+            if worker['type'] == worker_type and worker['enable'] is True:
+                worker_list.append(worker['uuid'])
+        return worker_list
 
 @defer.inlineCallbacks
 def CanWorkerBuildProject(builder, wfb, request):
@@ -16,8 +29,9 @@ def CanWorkerBuildProject(builder, wfb, request):
     print(wfb)
     for worker in project_workers:
         if wfb.worker.workername == worker['worker_uuid']:
+            print('Find a worker: YES')
             return True
-    print('no worker')
+    print('Find a worker: NO')
     return False
 
 # Use same worker as update_cpv_data was done by so we have same git commit
@@ -25,15 +39,16 @@ def CanWorkerUpdateV(builder, wfb, request):
     print(request.properties['cp_worker'])
     print(wfb)
     if wfb.worker.workername == request.properties['cp_worker']:
+        print('Find a worker: YES')
         return True
+    print('Find a worker: NO')
     return False
 
-def gentoo_builders(worker_data):
+def gentoo_builders(worker_data_local, worker_data_docker):
     b = []
-    g_ci_w = gentoo_ci_workers(worker_data)
     b.append(util.BuilderConfig(
         name='update_db_check',
-        workername=g_ci_w.getWorkersUuid('local')[0],
+        workername=getWorkersLocal('local', worker_data_local)[0],
         workerbuilddir='builds',
         collapseRequests=False,
         factory=buildfactorys.update_db_check()
@@ -41,7 +56,7 @@ def gentoo_builders(worker_data):
     )
     b.append(util.BuilderConfig(
         name='update_repo_check',
-        workername=g_ci_w.getWorkersUuid('local')[1],
+        workername=getWorkersLocal('local', worker_data_local)[1],
         workerbuilddir='builds',
         collapseRequests=True,
         factory=buildfactorys.update_repo_check()
@@ -52,7 +67,7 @@ def gentoo_builders(worker_data):
     # (builders.UpdateRepos step)
     b.append(util.BuilderConfig(
         name='update_cpv_data',
-        workernames=g_ci_w.getWorkersUuid('log')[0],
+        workernames=getWorkersDocker('log', worker_data_docker)[0],
         workerbuilddir='builds',
         collapseRequests=False,
         factory=buildfactorys.update_db_cpv()
@@ -61,7 +76,7 @@ def gentoo_builders(worker_data):
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='update_v_data',
-        workername=g_ci_w.getWorkersUuid('log')[0],
+        workername=getWorkersDocker('log', worker_data_docker)[0],
         workerbuilddir='builds',
         collapseRequests=False,
         canStartBuild=CanWorkerUpdateV,
@@ -71,7 +86,7 @@ def gentoo_builders(worker_data):
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='build_request_data',
-        workernames=g_ci_w.getWorkersUuid('local'),
+        workernames=getWorkersLocal('local', worker_data_local),
         collapseRequests=False,
         factory=buildfactorys.build_request_check()
         )
@@ -79,7 +94,7 @@ def gentoo_builders(worker_data):
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='run_build_request',
-        workernames=g_ci_w.getWorkersUuid('docker'),
+        workernames=getWorkersDocker('build', worker_data_docker),
         canStartBuild=CanWorkerBuildProject,
         collapseRequests=False,
         factory=buildfactorys.run_build_request()
@@ -88,7 +103,7 @@ def gentoo_builders(worker_data):
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='parse_build_log',
-        workernames=g_ci_w.getWorkersUuid('log')[1:],
+        workernames=getWorkersDocker('log', worker_data_docker)[1:],
         collapseRequests=False,
         factory=buildfactorys.parse_build_log()
         )
@@ -96,7 +111,7 @@ def gentoo_builders(worker_data):
     # For node workers
     b.append(util.BuilderConfig(
         name='run_build_stage4_request',
-        workernames=g_ci_w.getWorkersUuid('node'),
+        workernames=getWorkersLocal('local', worker_data_local),
         #FIXME: support more the one node
         #canStartBuild=CanWorkerBuildProject,
         collapseRequests=False,
diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py
index ea05f89..3dcef1a 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -1,31 +1,9 @@
 # Copyright 2022 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
-from buildbot.plugins import worker, util
-
-class gentoo_ci_workers():
-    def __init__(self, worker_data, **kwargs):
-        self.worker_data = worker_data
-
-    def getWorkersUuid(self, worker_type):
-        worker_list = []
-        for worker in self.worker_data:
-            if worker['type'] == worker_type and worker['enable'] is True:
-                worker_list.append(worker['uuid'])
-        print(worker_list)
-        return worker_list
+from twisted.internet import defer
 
-    def getWorkersAllData(self, worker_type):
-        worker_list = []
-        for worker in self.worker_data:
-            if worker['type'] == worker_type and worker['enable'] is True:
-                worker_list.append(worker)
-        print(worker_list)
-        return worker_list
-
-@util.renderer
-def build_docker_images(props):
-    return 'bb-worker-' + props.getProperty('project_uuid') + ':latest'
+from buildbot.plugins import worker, util
 
 @util.renderer
 def log_docker_images(props):
@@ -55,14 +33,32 @@ def docker_volumes_repositorys(props):
     volumes_list.append(':'.join([src_dir, dest_dir]))
     return volumes_list
 
-def gentoo_workers(worker_data):
-    w = []
-    g_ci_w = gentoo_ci_workers(worker_data)
+@util.renderer
+@defer.inlineCallbacks
+def getDockerHost(props, docker_worker):
+    print(docker_worker)
+    gentooci = props.master.namedServices['services'].namedServices['gentooci']
+    worker_data = yield gentooci.db.workers.getWorkerByUuid(docker_worker['uuid'])
+    node = yield gentooci.db.workers.getNodeByWorkerUuid(worker_data['uuid'])
+    node_data = yield gentooci.db.workers.getNodeByUuid(node['node_uuid'])
+    print(worker_data)
+    print(node_data)
+    return node_data['host_url']
+
+@util.renderer
+@defer.inlineCallbacks
+def GetBuildDockerImage(props, docker_worker):
+    gentooci = props.master.namedServices['services'].namedServices['gentooci']
+    worker_data = yield gentooci.db.workers.getWorkerByUuid(docker_worker['uuid'])
+    image_data = yield gentooci.db.workers.getImageById(worker_data['image_id'])
+    print(image_data)
+    return image_data['tag']
 
-    for local_worker in g_ci_w.getWorkersUuid('local'):
-        w.append(worker.LocalWorker(local_worker))
-    for node_worker in g_ci_w.getWorkersAllData('node'):
-        w.append(worker.Worker(node_worker['uuid'], node_worker['password']))
+def gentoo_workers(worker_data_local, worker_data_docker):
+    w = []
+    for local_worker in worker_data_local:
+        if local_worker['enable']:
+            w.append(worker.LocalWorker(local_worker['name']))
     # docker workers
     docker_hostconfig = {}
     # For use of sandbox stuff
@@ -71,24 +67,23 @@ def gentoo_workers(worker_data):
     # libseccomp overhead
     # https://github.com/seccomp/libseccomp/issues/153
     docker_hostconfig['security_opt'] = ['seccomp=unconfined']
-    for build_worker in g_ci_w.getWorkersAllData('docker'):
-        #FIXME: set settings in master.cfg
-        if build_worker['type'] == 'docker':
-            w.append(worker.DockerLatentWorker(build_worker['uuid'],
-                            build_worker['password'],
-                            docker_host='tcp://192.168.1.12:2375',
-                            image=build_docker_images,
+    print(worker_data_docker)
+    for docker_worker in worker_data_docker:
+        #FIXME: get settings from db
+        if docker_worker['type'] == 'build':
+            w.append(worker.DockerLatentWorker(docker_worker['uuid'],
+                            None,
+                            docker_host=getDockerHost.withArgs(docker_worker),
+                            image=GetBuildDockerImage.withArgs(docker_worker),
                             volumes=docker_volumes,
                             hostconfig=docker_hostconfig,
                             followStartupLogs=True,
                             masterFQDN='192.168.1.5',
                             build_wait_timeout=3600
                             ))
-    for log_worker in g_ci_w.getWorkersAllData('log'):
-        #FIXME: set settings in master.cfg
-        if log_worker['type'] == 'log':
-            w.append(worker.DockerLatentWorker(log_worker['uuid'],
-                            log_worker['password'],
+        if docker_worker['type'] == 'log':
+            w.append(worker.DockerLatentWorker(docker_worker['uuid'],
+                            None,
                             docker_host='tcp://192.168.1.12:2375',
                             image=log_docker_images,
                             volumes=docker_volumes_repositorys,
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index 7256cab..7665f84 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -38,6 +38,7 @@ from buildbot_gentoo_ci.db import versions
 from buildbot_gentoo_ci.db import keywords
 from buildbot_gentoo_ci.db import portages
 from buildbot_gentoo_ci.db import builds
+from buildbot_gentoo_ci.db import workers
 
 upgrade_message = textwrap.dedent("""\
 
@@ -85,6 +86,7 @@ class DBConnector(service.ReconfigurableServiceMixin,
         self.keywords = keywords.KeywordsConnectorComponent(self)
         self.portages = portages.PortagesConnectorComponent(self)
         self.builds = builds.BuildsConnectorComponent(self)
+        self.workers = workers.WorkersConnectorComponent(self)
 
     @defer.inlineCallbacks
     def setup(self, config, check_version=True, verbose=True):
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 3f1dc0a..87d7dbc 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -329,8 +329,61 @@ class Model(base.DBConnectorComponent):
         # unique id per project
         sa.Column('uuid', sa.String(36), primary_key=True,
                   default=lambda: str(uuid.uuid4())),
-        sa.Column('type', sa.Enum('local','default','latent'), nullable=False),
-        sa.Column('enabled', sa.Boolean, default=False),
+        sa.Column('type', sa.Enum('log','build'), nullable=False),
+        sa.Column('enable', sa.Boolean, default=False),
+        sa.Column('flavor_id', sa.Integer,
+                  sa.ForeignKey('workers_flavors.id', ondelete='CASCADE')),
+        sa.Column('image_id', sa.Integer,
+                  sa.ForeignKey('workers_images.id', ondelete='CASCADE')),
+    )
+
+    workers_flavors = sautils.Table(
+        "workers_flavors", metadata,
+        # unique id per project
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('name', sa.String(255), nullable=False),
+        sa.Column('ram', sa.Integer),
+        sa.Column('cpu', sa.Integer),
+    )
+
+    workers_images = sautils.Table(
+        "workers_images", metadata,
+        # unique id per project
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('name', sa.String(255), nullable=False),
+        sa.Column('tag', sa.String(255), nullable=False),
+        sa.Column('dockerfile', sa.String(255), nullable=False),
+        sa.Column('bootstrap_tag', sa.String(255), nullable=False),
+    )
+
+    workers_configs = sautils.Table(
+        "workers_configs", metadata,
+        # unique id per project
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('worker_uuid', sa.String(36),
+                  sa.ForeignKey('workers.uuid', ondelete='CASCADE')),
+        sa.Column('type', sa.Enum('cap_add','security_opt'), nullable=False),
+        sa.Column('value', sa.String(255), nullable=False),
+    )
+
+    nodes = sautils.Table(
+        "nodes", metadata,
+        # unique id per project
+        sa.Column('uuid', sa.String(36), primary_key=True,
+                  default=lambda: str(uuid.uuid4())),
+        sa.Column('name', sa.String(255), nullable=False),
+        sa.Column('host_url', sa.String(255), nullable=False),
+        sa.Column('enable', sa.Boolean, default=False),
+    )
+
+    nodes_workers = sautils.Table(
+        "nodes_workers", metadata,
+        # unique id per project
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('node_uuid', sa.String(36),
+                  sa.ForeignKey('nodes.uuid', ondelete='CASCADE')),
+        sa.Column('worker_uuid', sa.String(36),
+                  sa.ForeignKey('workers.uuid', ondelete='CASCADE')),
     )
 
     # Tables related to users
diff --git a/buildbot_gentoo_ci/db/workers.py b/buildbot_gentoo_ci/db/workers.py
new file mode 100644
index 0000000..afc7815
--- /dev/null
+++ b/buildbot_gentoo_ci/db/workers.py
@@ -0,0 +1,175 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors. 
+# Buildbot is free software: you can redistribute it and/or modify it 
+# under the terms of the GNU General Public License as published by the 
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
+# Copyright 2021 Gentoo Authors
+
+import sqlalchemy as sa
+
+from twisted.internet import defer
+
+from buildbot.db import base
+
+class WorkersConnectorComponent(base.DBConnectorComponent):
+
+    @defer.inlineCallbacks
+    def getWorkerByUuid(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.workers
+            q = tbl.select()
+            q = q.where(tbl.c.uuid == uuid)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getWorkersByTypeAndEnable(self, type):
+        def thd(conn):
+            tbl = self.db.model.workers
+            q = tbl.select()
+            q = q.where(tbl.c.type == type)
+            q = q.where(tbl.c.enable == true)
+            return [self._row2dict(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getWorkersByNodeUuid(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.nodes_workers
+            q = tbl.select()
+            q = q.where(tbl.c.node_uuid == uuid)
+            return [self._row2dict_node_worker(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getFlavorById(self, id):
+        def thd(conn):
+            tbl = self.db.model.workers_flavors
+            q = tbl.select()
+            q = q.where(tbl.c.id == id)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict_flavor(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getImageById(self, id):
+        def thd(conn):
+            tbl = self.db.model.workers_images
+            q = tbl.select()
+            q = q.where(tbl.c.id == id)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict_image(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getWorkerConfigByWorkerUuid(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.worker_configs
+            q = tbl.select()
+            q = q.where(tbl.c.project_uuid == uuid)
+            return [self._row2dict_node_config(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
+
+    @defer.inlineCallbacks
+    def getNodeByUuid(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.nodes
+            q = tbl.select()
+            q = q.where(tbl.c.uuid == uuid)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict_node(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getNodeByWorkerUuid(self, uuid):
+        def thd(conn):
+            tbl = self.db.model.nodes_workers
+            q = tbl.select()
+            q = q.where(tbl.c.worker_uuid == uuid)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict_node_worker(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
+    def _row2dict(self, conn, row):
+        return dict(
+            uuid=row.uuid,
+            type=row.type,
+            enable=row.enable,
+            flavor_id=row.flavor_id,
+            image_id=row.image_id
+            )
+    def _row2dict_flavor(self, conn, row):
+        return dict(
+            id=row.id,
+            name=row.name,
+            ram=row.ram,
+            cpu=row.cpu
+            )
+    def _row2dict_image(self, conn, row):
+        return dict(
+            id=row.id,
+            name=row.name,
+            tag=row.tag,
+            dockerfile=row.dockerfile,
+            bootstrap_image_tag=row.bootstrap_image_tag
+            )
+    def _row2dict_worker_config(self, conn, row):
+        return dict(
+            id=row.id,
+            worker_uuid=row.worker_uuid,
+            type=row.type,
+            value=row.value
+            )
+    def _row2dict_node(self, conn, row):
+        return dict(
+            uuid=row.uuid,
+            name=row.name,
+            host_url=row.host_url,
+            enable=row.enable
+            )
+    def _row2dict_node_worker(self, conn, row):
+        return dict(
+            id=row.id,
+            node_uuid=row.node_uuid,
+            worker_uuid=row.worker_uuid
+            )