Package coprs :: Package logic :: Module builds_logic
[hide private]
[frames] | no frames]

Source Code for Module coprs.logic.builds_logic

   1  import tempfile 
   2  import shutil 
   3  import json 
   4  import os 
   5  import pprint 
   6  import time 
   7  import requests 
   8   
   9  from sqlalchemy.sql import text 
  10  from sqlalchemy.sql.expression import not_ 
  11  from sqlalchemy.orm import joinedload 
  12  from sqlalchemy import or_ 
  13  from sqlalchemy import and_ 
  14  from sqlalchemy import func, desc 
  15  from sqlalchemy.sql import false,true 
  16  from werkzeug.utils import secure_filename 
  17  from sqlalchemy import bindparam, Integer, String 
  18  from sqlalchemy.exc import IntegrityError 
  19   
  20  from copr_common.enums import FailTypeEnum, StatusEnum 
  21  from coprs import app 
  22  from coprs import db 
  23  from coprs import models 
  24  from coprs import helpers 
  25  from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT 
  26  from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, \ 
  27                               UnrepeatableBuildException, RequestCannotBeExecuted, DuplicateException 
  28   
  29  from coprs.logic import coprs_logic 
  30  from coprs.logic import users_logic 
  31  from coprs.logic.actions_logic import ActionsLogic 
  32  from coprs.models import BuildChroot 
  33  from .coprs_logic import MockChrootsLogic 
  34  from coprs.logic.packages_logic import PackagesLogic 
  35   
  36  log = app.logger 
37 38 39 -class BuildsLogic(object):
40 @classmethod
41 - def get(cls, build_id):
42 return models.Build.query.filter(models.Build.id == build_id)
43 44 @classmethod
45 - def get_build_tasks(cls, status, background=None):
46 """ Returns tasks with given status. If background is specified then 47 returns normal jobs (false) or background jobs (true) 48 """ 49 result = models.BuildChroot.query.join(models.Build)\ 50 .filter(models.BuildChroot.status == status)\ 51 .order_by(models.Build.id.asc()) 52 if background is not None: 53 result = result.filter(models.Build.is_background == (true() if background else false())) 54 return result
55 56 @classmethod
57 - def get_srpm_build_tasks(cls, status, background=None):
58 """ Returns srpm build tasks with given status. If background is 59 specified then returns normal jobs (false) or background jobs (true) 60 """ 61 result = models.Build.query\ 62 .filter(models.Build.source_status == status)\ 63 .order_by(models.Build.id.asc()) 64 if background is not None: 65 result = result.filter(models.Build.is_background == (true() if background else false())) 66 return result
67 68 @classmethod
69 - def get_recent_tasks(cls, user=None, limit=100, period_days=2):
70 query_args = ( 71 models.BuildChroot.build_id, 72 func.max(models.BuildChroot.ended_on).label('max_ended_on'), 73 models.Build.submitted_on, 74 ) 75 group_by_args = ( 76 models.BuildChroot.build_id, 77 models.Build.submitted_on, 78 ) 79 80 81 if user: 82 query_args += (models.Build.user_id,) 83 group_by_args += (models.Build.user_id,) 84 85 subquery = (db.session.query(*query_args) 86 .join(models.Build) 87 .group_by(*group_by_args) 88 .having(func.count() == func.count(models.BuildChroot.ended_on)) 89 .having(models.Build.submitted_on > time.time() - 3600*24*period_days) 90 ) 91 if user: 92 subquery = subquery.having(models.Build.user_id == user.id) 93 94 subquery = subquery.order_by(desc('max_ended_on')).limit(limit).subquery() 95 96 query = models.Build.query.join(subquery, subquery.c.build_id == models.Build.id) 97 return list(query.all())
98 99 @classmethod
100 - def get_running_tasks_by_time(cls, start, end):
101 result = models.BuildChroot.query\ 102 .filter(models.BuildChroot.ended_on > start)\ 103 .filter(models.BuildChroot.started_on < end)\ 104 .order_by(models.BuildChroot.started_on.asc()) 105 106 return result
107 108 @classmethod
109 - def get_chroot_histogram(cls, start, end):
110 chroots = [] 111 chroot_query = BuildChroot.query\ 112 .filter(models.BuildChroot.started_on < end)\ 113 .filter(models.BuildChroot.ended_on > start)\ 114 .with_entities(BuildChroot.mock_chroot_id, 115 func.count(BuildChroot.mock_chroot_id))\ 116 .group_by(BuildChroot.mock_chroot_id)\ 117 .order_by(BuildChroot.mock_chroot_id) 118 119 for chroot in chroot_query: 120 chroots.append([chroot[0], chroot[1]]) 121 122 mock_chroots = coprs_logic.MockChrootsLogic.get_multiple() 123 for mock_chroot in mock_chroots: 124 for l in chroots: 125 if l[0] == mock_chroot.id: 126 l[0] = mock_chroot.name 127 128 return chroots
129 130 @classmethod
131 - def get_pending_jobs_bucket(cls, start, end):
132 query = text(""" 133 SELECT COUNT(*) as result 134 FROM build_chroot JOIN build on build.id = build_chroot.build_id 135 WHERE 136 build.submitted_on < :end 137 AND ( 138 build_chroot.started_on > :start 139 OR (build_chroot.started_on is NULL AND build_chroot.status = :status) 140 -- for currently pending builds we need to filter on status=pending because there might be 141 -- failed builds that have started_on=NULL 142 ) 143 AND NOT build.canceled 144 """) 145 146 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("pending")) 147 return res.first().result
148 149 @classmethod
150 - def get_running_jobs_bucket(cls, start, end):
151 query = text(""" 152 SELECT COUNT(*) as result 153 FROM build_chroot 154 WHERE 155 started_on < :end 156 AND (ended_on > :start OR (ended_on is NULL AND status = :status)) 157 -- for currently running builds we need to filter on status=running because there might be failed 158 -- builds that have ended_on=NULL 159 """) 160 161 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("running")) 162 return res.first().result
163 164 @classmethod
165 - def get_cached_graph_data(cls, params):
166 data = { 167 "pending": [], 168 "running": [], 169 } 170 result = models.BuildsStatistics.query\ 171 .filter(models.BuildsStatistics.stat_type == params["type"])\ 172 .filter(models.BuildsStatistics.time >= params["start"])\ 173 .filter(models.BuildsStatistics.time <= params["end"])\ 174 .order_by(models.BuildsStatistics.time) 175 176 for row in result: 177 data["pending"].append(row.pending) 178 data["running"].append(row.running) 179 180 return data
181 182 @classmethod
183 - def get_task_graph_data(cls, type):
184 data = [["pending"], ["running"], ["avg running"], ["time"]] 185 params = cls.get_graph_parameters(type) 186 cached_data = cls.get_cached_graph_data(params) 187 data[0].extend(cached_data["pending"]) 188 data[1].extend(cached_data["running"]) 189 190 for i in range(len(data[0]) - 1, params["steps"]): 191 step_start = params["start"] + i * params["step"] 192 step_end = step_start + params["step"] 193 pending = cls.get_pending_jobs_bucket(step_start, step_end) 194 running = cls.get_running_jobs_bucket(step_start, step_end) 195 data[0].append(pending) 196 data[1].append(running) 197 cls.cache_graph_data(type, time=step_start, pending=pending, running=running) 198 199 running_total = 0 200 for i in range(1, params["steps"] + 1): 201 running_total += data[1][i] 202 203 data[2].extend([running_total * 1.0 / params["steps"]] * (len(data[0]) - 1)) 204 205 for i in range(params["start"], params["end"], params["step"]): 206 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i))) 207 208 return data
209 210 @classmethod
211 - def get_small_graph_data(cls, type):
212 data = [[""]] 213 params = cls.get_graph_parameters(type) 214 cached_data = cls.get_cached_graph_data(params) 215 data[0].extend(cached_data["running"]) 216 217 for i in range(len(data[0]) - 1, params["steps"]): 218 step_start = params["start"] + i * params["step"] 219 step_end = step_start + params["step"] 220 running = cls.get_running_jobs_bucket(step_start, step_end) 221 data[0].append(running) 222 cls.cache_graph_data(type, time=step_start, running=running) 223 224 return data
225 226 @classmethod
227 - def cache_graph_data(cls, type, time, pending=0, running=0):
228 result = models.BuildsStatistics.query\ 229 .filter(models.BuildsStatistics.stat_type == type)\ 230 .filter(models.BuildsStatistics.time == time).first() 231 if result: 232 return 233 234 try: 235 cached_data = models.BuildsStatistics( 236 time = time, 237 stat_type = type, 238 running = running, 239 pending = pending 240 ) 241 db.session.add(cached_data) 242 db.session.commit() 243 except IntegrityError: # other process already calculated the graph data and cached it 244 db.session.rollback()
245 246 @classmethod
247 - def get_graph_parameters(cls, type):
248 if type is "10min": 249 # 24 hours with 10 minute intervals 250 step = 600 251 steps = 144 252 elif type is "30min": 253 # 24 hours with 30 minute intervals 254 step = 1800 255 steps = 48 256 elif type is "24h": 257 # 90 days with 24 hour intervals 258 step = 86400 259 steps = 90 260 261 end = int(time.time()) 262 end = end - (end % step) # align graph interval to a multiple of step 263 start = end - (steps * step) 264 265 return { 266 "type": type, 267 "step": step, 268 "steps": steps, 269 "start": start, 270 "end": end, 271 }
272 273 @classmethod
274 - def get_build_importing_queue(cls, background=None):
275 """ 276 Returns Builds which are waiting to be uploaded to dist git 277 """ 278 query = (models.Build.query 279 .filter(models.Build.canceled == false()) 280 .filter(models.Build.source_status == StatusEnum("importing")) 281 .order_by(models.Build.id.asc())) 282 if background is not None: 283 query = query.filter(models.Build.is_background == (true() if background else false())) 284 return query
285 286 @classmethod
287 - def get_pending_srpm_build_tasks(cls, background=None):
288 query = (models.Build.query 289 .filter(models.Build.canceled == false()) 290 .filter(models.Build.source_status == StatusEnum("pending")) 291 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 292 if background is not None: 293 query = query.filter(models.Build.is_background == (true() if background else false())) 294 return query
295 296 @classmethod
297 - def get_pending_build_tasks(cls, background=None):
298 query = (models.BuildChroot.query 299 .outerjoin(models.Build) 300 .outerjoin(models.CoprDir) 301 .outerjoin(models.Package, models.Package.id == models.Build.package_id) 302 .options(joinedload('build').joinedload('copr_dir'), 303 joinedload('build').joinedload('package')) 304 .filter(models.Build.canceled == false()) 305 .filter(or_( 306 models.BuildChroot.status == StatusEnum("pending"), 307 and_( 308 models.BuildChroot.status == StatusEnum("running"), 309 models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), 310 models.BuildChroot.ended_on.is_(None) 311 ) 312 )) 313 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 314 if background is not None: 315 query = query.filter(models.Build.is_background == (true() if background else false())) 316 return query
317 318 @classmethod
319 - def get_build_task(cls, task_id):
320 try: 321 build_id, chroot_name = task_id.split("-", 1) 322 except ValueError: 323 raise MalformedArgumentException("Invalid task_id {}".format(task_id)) 324 325 build_chroot = BuildChrootsLogic.get_by_build_id_and_name(build_id, chroot_name) 326 return build_chroot.join(models.Build).first()
327 328 @classmethod
329 - def get_srpm_build_task(cls, build_id):
330 return BuildsLogic.get_by_id(build_id).first()
331 332 @classmethod
333 - def get_multiple(cls):
334 return models.Build.query.order_by(models.Build.id.desc())
335 336 @classmethod
337 - def get_multiple_by_copr(cls, copr):
338 """ Get collection of builds in copr sorted by build_id descending 339 """ 340 return cls.get_multiple().filter(models.Build.copr == copr)
341 342 @classmethod
343 - def get_multiple_by_user(cls, user):
344 """ Get collection of builds in copr sorted by build_id descending 345 form the copr belonging to `user` 346 """ 347 return cls.get_multiple().join(models.Build.copr).filter( 348 models.Copr.user == user)
349 350 @classmethod
351 - def init_db(cls):
352 if db.engine.url.drivername == "sqlite": 353 return 354 355 status_to_order = """ 356 CREATE OR REPLACE FUNCTION status_to_order (x integer) 357 RETURNS integer AS $$ BEGIN 358 RETURN CASE WHEN x = 3 THEN 1 359 WHEN x = 6 THEN 2 360 WHEN x = 7 THEN 3 361 WHEN x = 4 THEN 4 362 WHEN x = 0 THEN 5 363 WHEN x = 1 THEN 6 364 WHEN x = 5 THEN 7 365 WHEN x = 2 THEN 8 366 WHEN x = 8 THEN 9 367 WHEN x = 9 THEN 10 368 ELSE x 369 END; END; 370 $$ LANGUAGE plpgsql; 371 """ 372 373 order_to_status = """ 374 CREATE OR REPLACE FUNCTION order_to_status (x integer) 375 RETURNS integer AS $$ BEGIN 376 RETURN CASE WHEN x = 1 THEN 3 377 WHEN x = 2 THEN 6 378 WHEN x = 3 THEN 7 379 WHEN x = 4 THEN 4 380 WHEN x = 5 THEN 0 381 WHEN x = 6 THEN 1 382 WHEN x = 7 THEN 5 383 WHEN x = 8 THEN 2 384 WHEN x = 9 THEN 8 385 WHEN x = 10 THEN 9 386 ELSE x 387 END; END; 388 $$ LANGUAGE plpgsql; 389 """ 390 391 db.engine.connect() 392 db.engine.execute(status_to_order) 393 db.engine.execute(order_to_status)
394 395 @classmethod
396 - def get_copr_builds_list(cls, copr, dirname=''):
397 query_select = """ 398 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on, 399 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status, 400 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name, build.copr_id 401 FROM build 402 LEFT OUTER JOIN package 403 ON build.package_id = package.id 404 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses 405 ON statuses.build_id=build.id 406 LEFT OUTER JOIN copr 407 ON copr.id = build.copr_id 408 LEFT OUTER JOIN copr_dir 409 ON build.copr_dir_id = copr_dir.id 410 LEFT OUTER JOIN "user" 411 ON copr.user_id = "user".id 412 LEFT OUTER JOIN "group" 413 ON copr.group_id = "group".id 414 WHERE build.copr_id = :copr_id 415 AND (:dirname = '' OR :dirname = copr_dir.name) 416 GROUP BY 417 build.id 418 ORDER BY 419 build.id DESC; 420 """ 421 422 if db.engine.url.drivername == "sqlite": 423 def sqlite_status_to_order(x): 424 if x == 3: 425 return 1 426 elif x == 6: 427 return 2 428 elif x == 7: 429 return 3 430 elif x == 4: 431 return 4 432 elif x == 0: 433 return 5 434 elif x == 1: 435 return 6 436 elif x == 5: 437 return 7 438 elif x == 2: 439 return 8 440 elif x == 8: 441 return 9 442 elif x == 9: 443 return 10 444 return 1000
445 446 def sqlite_order_to_status(x): 447 if x == 1: 448 return 3 449 elif x == 2: 450 return 6 451 elif x == 3: 452 return 7 453 elif x == 4: 454 return 4 455 elif x == 5: 456 return 0 457 elif x == 6: 458 return 1 459 elif x == 7: 460 return 5 461 elif x == 8: 462 return 2 463 elif x == 9: 464 return 8 465 elif x == 10: 466 return 9 467 return 1000
468 469 conn = db.engine.connect() 470 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order) 471 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status) 472 statement = text(query_select) 473 statement.bindparams(bindparam("copr_id", Integer)) 474 statement.bindparams(bindparam("dirname", String)) 475 result = conn.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 476 else: 477 statement = text(query_select) 478 statement.bindparams(bindparam("copr_id", Integer)) 479 statement.bindparams(bindparam("dirname", String)) 480 result = db.engine.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 481 482 return result 483 484 @classmethod
485 - def join_group(cls, query):
486 return query.join(models.Copr).outerjoin(models.Group)
487 488 @classmethod
489 - def get_multiple_by_name(cls, username, coprname):
490 query = cls.get_multiple() 491 return (query.join(models.Build.copr) 492 .options(db.contains_eager(models.Build.copr)) 493 .join(models.Copr.user) 494 .filter(models.Copr.name == coprname) 495 .filter(models.User.username == username))
496 497 @classmethod
498 - def get_by_ids(cls, ids):
499 return models.Build.query.filter(models.Build.id.in_(ids))
500 501 @classmethod
502 - def get_by_id(cls, build_id):
503 return models.Build.query.filter(models.Build.id == build_id)
504 505 @classmethod
506 - def create_new_from_other_build(cls, user, copr, source_build, 507 chroot_names=None, **build_options):
508 skip_import = False 509 git_hashes = {} 510 511 if source_build.source_type == helpers.BuildSourceEnum('upload'): 512 if source_build.repeatable: 513 skip_import = True 514 for chroot in source_build.build_chroots: 515 git_hashes[chroot.name] = chroot.git_hash 516 else: 517 raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.") 518 519 build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, 520 pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, 521 srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options) 522 build.package_id = source_build.package_id 523 build.pkg_version = source_build.pkg_version 524 return build
525 526 @classmethod
527 - def create_new_from_url(cls, user, copr, url, chroot_names=None, 528 copr_dirname=None, **build_options):
529 """ 530 :type user: models.User 531 :type copr: models.Copr 532 533 :type chroot_names: List[str] 534 535 :rtype: models.Build 536 """ 537 source_type = helpers.BuildSourceEnum("link") 538 source_json = json.dumps({"url": url}) 539 srpm_url = None if url.endswith('.spec') else url 540 return cls.create_new(user, copr, source_type, source_json, chroot_names, 541 pkgs=url, srpm_url=srpm_url, copr_dirname=copr_dirname, **build_options)
542 543 @classmethod
544 - def create_new_from_scm(cls, user, copr, scm_type, clone_url, 545 committish='', subdirectory='', spec='', srpm_build_method='rpkg', 546 chroot_names=None, copr_dirname=None, **build_options):
547 """ 548 :type user: models.User 549 :type copr: models.Copr 550 551 :type chroot_names: List[str] 552 553 :rtype: models.Build 554 """ 555 source_type = helpers.BuildSourceEnum("scm") 556 source_json = json.dumps({"type": scm_type, 557 "clone_url": clone_url, 558 "committish": committish, 559 "subdirectory": subdirectory, 560 "spec": spec, 561 "srpm_build_method": srpm_build_method}) 562 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
563 564 @classmethod
565 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, spec_template, 566 python_versions, chroot_names=None, copr_dirname=None, **build_options):
567 """ 568 :type user: models.User 569 :type copr: models.Copr 570 :type package_name: str 571 :type version: str 572 :type python_versions: List[str] 573 574 :type chroot_names: List[str] 575 576 :rtype: models.Build 577 """ 578 source_type = helpers.BuildSourceEnum("pypi") 579 source_json = json.dumps({"pypi_package_name": pypi_package_name, 580 "pypi_package_version": pypi_package_version, 581 "spec_template": spec_template, 582 "python_versions": python_versions}) 583 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
584 585 @classmethod
586 - def create_new_from_rubygems(cls, user, copr, gem_name, chroot_names=None, 587 copr_dirname=None, **build_options):
588 """ 589 :type user: models.User 590 :type copr: models.Copr 591 :type gem_name: str 592 :type chroot_names: List[str] 593 :rtype: models.Build 594 """ 595 source_type = helpers.BuildSourceEnum("rubygems") 596 source_json = json.dumps({"gem_name": gem_name}) 597 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
598 599 @classmethod
600 - def create_new_from_custom(cls, user, copr, script, script_chroot=None, script_builddeps=None, 601 script_resultdir=None, chroot_names=None, copr_dirname=None, **kwargs):
602 """ 603 :type user: models.User 604 :type copr: models.Copr 605 :type script: str 606 :type script_chroot: str 607 :type script_builddeps: str 608 :type script_resultdir: str 609 :type chroot_names: List[str] 610 :rtype: models.Build 611 """ 612 source_type = helpers.BuildSourceEnum("custom") 613 source_dict = { 614 'script': script, 615 'chroot': script_chroot, 616 'builddeps': script_builddeps, 617 'resultdir': script_resultdir, 618 } 619 620 return cls.create_new(user, copr, source_type, json.dumps(source_dict), 621 chroot_names, copr_dirname=copr_dirname, **kwargs)
622 623 @classmethod
624 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, 625 chroot_names=None, copr_dirname=None, **build_options):
626 """ 627 :type user: models.User 628 :type copr: models.Copr 629 :param f_uploader(file_path): function which stores data at the given `file_path` 630 :return: 631 """ 632 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"]) 633 tmp_name = os.path.basename(tmp) 634 filename = secure_filename(orig_filename) 635 file_path = os.path.join(tmp, filename) 636 f_uploader(file_path) 637 638 # make the pkg public 639 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format( 640 baseurl=app.config["PUBLIC_COPR_BASE_URL"], 641 tmp_dir=tmp_name, 642 filename=filename) 643 644 # create json describing the build source 645 source_type = helpers.BuildSourceEnum("upload") 646 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name}) 647 srpm_url = None if pkg_url.endswith('.spec') else pkg_url 648 649 try: 650 build = cls.create_new(user, copr, source_type, source_json, 651 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, 652 copr_dirname=copr_dirname, **build_options) 653 except Exception: 654 shutil.rmtree(tmp) # todo: maybe we should delete in some cleanup procedure? 655 raise 656 657 return build
658 659 @classmethod
660 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="", 661 git_hashes=None, skip_import=False, background=False, batch=None, 662 srpm_url=None, copr_dirname=None, **build_options):
663 """ 664 :type user: models.User 665 :type copr: models.Copr 666 :type chroot_names: List[str] 667 :type source_type: int value from helpers.BuildSourceEnum 668 :type source_json: str in json format 669 :type pkgs: str 670 :type git_hashes: dict 671 :type skip_import: bool 672 :type background: bool 673 :type batch: models.Batch 674 :rtype: models.Build 675 """ 676 chroots = None 677 if chroot_names: 678 chroots = [] 679 for chroot in copr.active_chroots: 680 if chroot.name in chroot_names: 681 chroots.append(chroot) 682 683 build = cls.add( 684 user=user, 685 pkgs=pkgs, 686 copr=copr, 687 chroots=chroots, 688 source_type=source_type, 689 source_json=source_json, 690 enable_net=build_options.get("enable_net", copr.build_enable_net), 691 background=background, 692 git_hashes=git_hashes, 693 skip_import=skip_import, 694 batch=batch, 695 srpm_url=srpm_url, 696 copr_dirname=copr_dirname, 697 ) 698 699 if user.proven: 700 if "timeout" in build_options: 701 build.timeout = build_options["timeout"] 702 703 return build
704 705 @classmethod
706 - def add(cls, user, pkgs, copr, source_type=None, source_json=None, 707 repos=None, chroots=None, timeout=None, enable_net=True, 708 git_hashes=None, skip_import=False, background=False, batch=None, 709 srpm_url=None, copr_dirname=None):
710 711 if chroots is None: 712 chroots = [] 713 714 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action( 715 copr, "Can't build while there is an operation in progress: {action}") 716 users_logic.UsersLogic.raise_if_cant_build_in_copr( 717 user, copr, 718 "You don't have permissions to build in this copr.") 719 720 if not repos: 721 repos = copr.repos 722 723 # todo: eliminate pkgs and this check 724 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs): 725 raise MalformedArgumentException("Trying to create a build using src_pkg " 726 "with bad characters. Forgot to split?") 727 728 # just temporary to keep compatibility 729 if not source_type or not source_json: 730 source_type = helpers.BuildSourceEnum("link") 731 source_json = json.dumps({"url":pkgs}) 732 733 if skip_import and srpm_url: 734 chroot_status = StatusEnum("pending") 735 source_status = StatusEnum("succeeded") 736 else: 737 chroot_status = StatusEnum("waiting") 738 source_status = StatusEnum("pending") 739 740 copr_dir = None 741 if copr_dirname: 742 if not copr_dirname.startswith(copr.name+':') and copr_dirname != copr.name: 743 raise MalformedArgumentException("Copr dirname not starting with copr name.") 744 copr_dir = coprs_logic.CoprDirsLogic.get_or_create(copr, copr_dirname) 745 746 build = models.Build( 747 user=user, 748 pkgs=pkgs, 749 copr=copr, 750 repos=repos, 751 source_type=source_type, 752 source_json=source_json, 753 source_status=source_status, 754 submitted_on=int(time.time()), 755 enable_net=bool(enable_net), 756 is_background=bool(background), 757 batch=batch, 758 srpm_url=srpm_url, 759 copr_dir=copr_dir, 760 ) 761 762 if timeout: 763 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT 764 765 db.session.add(build) 766 767 for chroot in chroots: 768 # Chroots were explicitly set per-build. 769 git_hash = None 770 if git_hashes: 771 git_hash = git_hashes.get(chroot.name) 772 buildchroot = models.BuildChroot( 773 build=build, 774 status=chroot_status, 775 mock_chroot=chroot, 776 git_hash=git_hash, 777 ) 778 db.session.add(buildchroot) 779 780 return build
781 782 @classmethod
783 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None, 784 scm_object_type=None, scm_object_id=None, 785 scm_object_url=None, submitted_by=None):
786 787 source_dict = package.source_json_dict 788 source_dict.update(source_dict_update) 789 source_json = json.dumps(source_dict) 790 791 if not copr_dir: 792 copr_dir = package.copr.main_dir 793 794 build = models.Build( 795 user=None, 796 pkgs=None, 797 package=package, 798 copr=package.copr, 799 repos=package.copr.repos, 800 source_status=StatusEnum("pending"), 801 source_type=package.source_type, 802 source_json=source_json, 803 submitted_on=int(time.time()), 804 enable_net=package.copr.build_enable_net, 805 timeout=DEFAULT_BUILD_TIMEOUT, 806 copr_dir=copr_dir, 807 update_callback=update_callback, 808 scm_object_type=scm_object_type, 809 scm_object_id=scm_object_id, 810 scm_object_url=scm_object_url, 811 submitted_by=submitted_by, 812 ) 813 db.session.add(build) 814 815 status = StatusEnum("waiting") 816 for chroot in package.chroots: 817 buildchroot = models.BuildChroot( 818 build=build, 819 status=status, 820 mock_chroot=chroot, 821 git_hash=None 822 ) 823 db.session.add(buildchroot) 824 825 cls.process_update_callback(build) 826 return build
827 828 829 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")} 830 831 @classmethod
832 - def get_buildchroots_by_build_id_and_branch(cls, build_id, branch):
833 """ 834 Returns a list of BuildChroots identified by build_id and dist-git 835 branch name. 836 """ 837 return ( 838 models.BuildChroot.query 839 .join(models.MockChroot) 840 .filter(models.BuildChroot.build_id==build_id) 841 .filter(models.MockChroot.distgit_branch_name==branch) 842 ).all()
843 844 845 @classmethod
846 - def delete_local_source(cls, build):
847 """ 848 Deletes the locally stored data for build purposes. This is typically 849 uploaded srpm file, uploaded spec file or webhook POST content. 850 """ 851 # is it hosted on the copr frontend? 852 data = json.loads(build.source_json) 853 if 'tmp' in data: 854 tmp = data["tmp"] 855 storage_path = app.config["STORAGE_DIR"] 856 try: 857 shutil.rmtree(os.path.join(storage_path, tmp)) 858 except: 859 pass
860 861 862 @classmethod
863 - def update_state_from_dict(cls, build, upd_dict):
864 """ 865 :param build: 866 :param upd_dict: 867 example: 868 { 869 "builds":[ 870 { 871 "id": 1, 872 "copr_id": 2, 873 "started_on": 1390866440 874 }, 875 { 876 "id": 2, 877 "copr_id": 1, 878 "status": 0, 879 "chroot": "fedora-18-x86_64", 880 "result_dir": "baz", 881 "ended_on": 1390866440 882 }] 883 } 884 """ 885 log.info("Updating build {} by: {}".format(build.id, upd_dict)) 886 887 # create the package if it doesn't exist 888 pkg_name = upd_dict.get('pkg_name', None) 889 if pkg_name: 890 if not PackagesLogic.get(build.copr_dir.id, pkg_name).first(): 891 try: 892 package = PackagesLogic.add( 893 build.copr.user, build.copr_dir, 894 pkg_name, build.source_type, build.source_json) 895 db.session.add(package) 896 db.session.commit() 897 except (IntegrityError, DuplicateException) as e: 898 app.logger.exception(e) 899 db.session.rollback() 900 return 901 build.package = PackagesLogic.get(build.copr_dir.id, pkg_name).first() 902 903 for attr in ["built_packages", "srpm_url", "pkg_version"]: 904 value = upd_dict.get(attr, None) 905 if value: 906 setattr(build, attr, value) 907 908 # update source build status 909 if str(upd_dict.get("task_id")) == str(build.task_id): 910 build.result_dir = upd_dict.get("result_dir", "") 911 912 new_status = upd_dict.get("status") 913 if new_status == StatusEnum("succeeded"): 914 new_status = StatusEnum("importing") 915 chroot_status=StatusEnum("waiting") 916 if not build.build_chroots: 917 # create the BuildChroots from Package setting, if not 918 # already set explicitly for concrete build 919 for chroot in build.package.chroots: 920 buildchroot = models.BuildChroot( 921 build=build, 922 status=chroot_status, 923 mock_chroot=chroot, 924 git_hash=None, 925 ) 926 db.session.add(buildchroot) 927 else: 928 for buildchroot in build.build_chroots: 929 buildchroot.status = chroot_status 930 db.session.add(buildchroot) 931 932 build.source_status = new_status 933 if new_status == StatusEnum("failed") or \ 934 new_status == StatusEnum("skipped"): 935 for ch in build.build_chroots: 936 ch.status = new_status 937 ch.ended_on = upd_dict.get("ended_on") or time.time() 938 ch.started_on = upd_dict.get("started_on", ch.ended_on) 939 db.session.add(ch) 940 941 if new_status == StatusEnum("failed"): 942 build.fail_type = FailTypeEnum("srpm_build_error") 943 944 cls.process_update_callback(build) 945 db.session.add(build) 946 return 947 948 if "chroot" in upd_dict: 949 # update respective chroot status 950 for build_chroot in build.build_chroots: 951 if build_chroot.name == upd_dict["chroot"]: 952 build_chroot.result_dir = upd_dict.get("result_dir", "") 953 954 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states: 955 build_chroot.status = upd_dict["status"] 956 957 if upd_dict.get("status") in BuildsLogic.terminal_states: 958 build_chroot.ended_on = upd_dict.get("ended_on") or time.time() 959 960 if upd_dict.get("status") == StatusEnum("starting"): 961 build_chroot.started_on = upd_dict.get("started_on") or time.time() 962 963 db.session.add(build_chroot) 964 965 # If the last package of a module was successfully built, 966 # then send an action to create module repodata on backend 967 if (build.module 968 and upd_dict.get("status") == StatusEnum("succeeded") 969 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): 970 ActionsLogic.send_build_module(build.copr, build.module) 971 972 cls.process_update_callback(build) 973 db.session.add(build)
974 975 @classmethod
976 - def process_update_callback(cls, build):
977 parsed_git_url = helpers.get_parsed_git_url(build.copr.scm_repo_url) 978 if not parsed_git_url: 979 return 980 981 if build.update_callback == 'pagure_flag_pull_request': 982 api_url = 'https://{0}/api/0/{1}/pull-request/{2}/flag'.format( 983 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 984 return cls.pagure_flag(build, api_url) 985 986 elif build.update_callback == 'pagure_flag_commit': 987 api_url = 'https://{0}/api/0/{1}/c/{2}/flag'.format( 988 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 989 return cls.pagure_flag(build, api_url)
990 991 @classmethod
992 - def pagure_flag(cls, build, api_url):
993 headers = { 994 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key')) 995 } 996 997 if build.srpm_url: 998 progress = 50 999 else: 1000 progress = 10 1001 1002 state_table = { 1003 'failed': ('failure', 0), 1004 'succeeded': ('success', 100), 1005 'canceled': ('canceled', 0), 1006 'running': ('pending', progress), 1007 'pending': ('pending', progress), 1008 'skipped': ('error', 0), 1009 'starting': ('pending', progress), 1010 'importing': ('pending', progress), 1011 'forked': ('error', 0), 1012 'waiting': ('pending', progress), 1013 'unknown': ('error', 0), 1014 } 1015 1016 build_url = os.path.join( 1017 app.config['PUBLIC_COPR_BASE_URL'], 1018 'coprs', build.copr.full_name.replace('@', 'g/'), 1019 'build', str(build.id) 1020 ) 1021 1022 data = { 1023 'username': 'Copr build', 1024 'comment': '#{}'.format(build.id), 1025 'url': build_url, 1026 'status': state_table[build.state][0], 1027 'percent': state_table[build.state][1], 1028 'uid': str(build.id), 1029 } 1030 1031 log.debug('Sending data to Pagure API: %s', pprint.pformat(data)) 1032 response = requests.post(api_url, data=data, headers=headers) 1033 log.debug('Pagure API response: %s', response.text)
1034 1035 @classmethod
1036 - def cancel_build(cls, user, build):
1037 if not user.can_build_in(build.copr): 1038 raise InsufficientRightsException( 1039 "You are not allowed to cancel this build.") 1040 if not build.cancelable: 1041 if build.status == StatusEnum("starting"): 1042 # this is not intuitive, that's why we provide more specific message 1043 err_msg = "Cannot cancel build {} in state 'starting'".format(build.id) 1044 else: 1045 err_msg = "Cannot cancel build {}".format(build.id) 1046 raise RequestCannotBeExecuted(err_msg) 1047 1048 if build.status == StatusEnum("running"): # otherwise the build is just in frontend 1049 ActionsLogic.send_cancel_build(build) 1050 1051 build.canceled = True 1052 cls.process_update_callback(build) 1053 1054 for chroot in build.build_chroots: 1055 chroot.status = 2 # canceled 1056 if chroot.ended_on is not None: 1057 chroot.ended_on = time.time()
1058 1059 @classmethod
1060 - def check_build_to_delete(cls, user, build):
1061 """ 1062 :type user: models.User 1063 :type build: models.Build 1064 """ 1065 if not user.can_edit(build.copr) or build.persistent: 1066 raise InsufficientRightsException( 1067 "You are not allowed to delete build `{}`.".format(build.id)) 1068 1069 if not build.finished: 1070 raise ActionInProgressException( 1071 "You can not delete build `{}` which is not finished.".format(build.id), 1072 "Unfinished build")
1073 1074 @classmethod
1075 - def delete_build(cls, user, build, send_delete_action=True):
1076 """ 1077 :type user: models.User 1078 :type build: models.Build 1079 """ 1080 cls.check_build_to_delete(user, build) 1081 1082 if send_delete_action: 1083 ActionsLogic.send_delete_build(build) 1084 1085 db.session.delete(build)
1086 1087 @classmethod
1088 - def delete_multiple_builds(cls, user, builds):
1089 """ 1090 :type user: models.User 1091 :type builds: list of models.Build 1092 """ 1093 to_delete = [] 1094 for build in builds: 1095 cls.check_build_to_delete(user, build) 1096 to_delete.append(build) 1097 1098 if to_delete: 1099 ActionsLogic.send_delete_multiple_builds(to_delete) 1100 1101 for build in to_delete: 1102 for build_chroot in build.build_chroots: 1103 db.session.delete(build_chroot) 1104 1105 db.session.delete(build)
1106 1107 @classmethod
1108 - def mark_as_failed(cls, build_id):
1109 """ 1110 Marks build as failed on all its non-finished chroots 1111 """ 1112 build = cls.get(build_id).one() 1113 chroots = filter(lambda x: x.status != StatusEnum("succeeded"), build.build_chroots) 1114 for chroot in chroots: 1115 chroot.status = StatusEnum("failed") 1116 if build.source_status != StatusEnum("succeeded"): 1117 build.source_status = StatusEnum("failed") 1118 cls.process_update_callback(build) 1119 return build
1120 1121 @classmethod
1122 - def last_modified(cls, copr):
1123 """ Get build datetime (as epoch) of last successful build 1124 1125 :arg copr: object of copr 1126 """ 1127 builds = cls.get_multiple_by_copr(copr) 1128 1129 last_build = ( 1130 builds.join(models.BuildChroot) 1131 .filter((models.BuildChroot.status == StatusEnum("succeeded")) 1132 | (models.BuildChroot.status == StatusEnum("skipped"))) 1133 .filter(models.BuildChroot.ended_on.isnot(None)) 1134 .order_by(models.BuildChroot.ended_on.desc()) 1135 ).first() 1136 if last_build: 1137 return last_build.ended_on 1138 else: 1139 return None
1140 1141 @classmethod
1142 - def filter_is_finished(cls, query, is_finished):
1143 # todo: check that ended_on is set correctly for all cases 1144 # e.g.: failed dist-git import, cancellation 1145 if is_finished: 1146 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.isnot(None)) 1147 else: 1148 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.is_(None))
1149 1150 @classmethod
1151 - def filter_by_group_name(cls, query, group_name):
1152 return query.filter(models.Group.name == group_name)
1153 1154 @classmethod
1155 - def filter_by_package_name(cls, query, package_name):
1156 return query.join(models.Package).filter(models.Package.name == package_name)
1157 1158 @classmethod
1159 - def clean_old_builds(cls):
1160 dirs = ( 1161 db.session.query( 1162 models.CoprDir.id, 1163 models.Package.id, 1164 models.Package.max_builds) 1165 .join(models.Build, models.Build.copr_dir_id==models.CoprDir.id) 1166 .join(models.Package) 1167 .filter(models.Package.max_builds > 0) 1168 .group_by( 1169 models.CoprDir.id, 1170 models.Package.max_builds, 1171 models.Package.id) 1172 .having(func.count(models.Build.id) > models.Package.max_builds) 1173 ) 1174 1175 for dir_id, package_id, limit in dirs.all(): 1176 delete_builds = ( 1177 models.Build.query.filter( 1178 models.Build.copr_dir_id==dir_id, 1179 models.Build.package_id==package_id) 1180 .order_by(desc(models.Build.id)) 1181 .offset(limit) 1182 .all() 1183 ) 1184 1185 for build in delete_builds: 1186 try: 1187 cls.delete_build(build.copr.user, build) 1188 except ActionInProgressException: 1189 # postpone this one to next day run 1190 log.error("Build(id={}) delete failed, unfinished action.".format(build.id))
1191 1192 @classmethod
1193 - def delete_orphaned_builds(cls):
1194 builds_to_delete = models.Build.query\ 1195 .join(models.Copr, models.Build.copr_id == models.Copr.id)\ 1196 .filter(models.Copr.deleted == True) 1197 1198 counter = 0 1199 for build in builds_to_delete: 1200 cls.delete_build(build.copr.user, build) 1201 counter += 1 1202 if counter >= 100: 1203 db.session.commit() 1204 counter = 0 1205 1206 if counter > 0: 1207 db.session.commit()
1208
1209 1210 -class BuildChrootsLogic(object):
1211 @classmethod
1212 - def get_by_build_id_and_name(cls, build_id, name):
1213 mc = MockChrootsLogic.get_from_name(name).one() 1214 1215 return ( 1216 BuildChroot.query 1217 .filter(BuildChroot.build_id == build_id) 1218 .filter(BuildChroot.mock_chroot_id == mc.id) 1219 )
1220 1221 @classmethod
1222 - def get_multiply(cls):
1223 query = ( 1224 models.BuildChroot.query 1225 .join(models.BuildChroot.build) 1226 .join(models.BuildChroot.mock_chroot) 1227 .join(models.Build.copr) 1228 .join(models.Copr.user) 1229 .outerjoin(models.Group) 1230 ) 1231 return query
1232 1233 @classmethod
1234 - def filter_by_build_id(cls, query, build_id):
1235 return query.filter(models.Build.id == build_id)
1236 1237 @classmethod
1238 - def filter_by_project_id(cls, query, project_id):
1239 return query.filter(models.Copr.id == project_id)
1240 1241 @classmethod
1242 - def filter_by_project_user_name(cls, query, username):
1243 return query.filter(models.User.username == username)
1244 1245 @classmethod
1246 - def filter_by_state(cls, query, state):
1247 return query.filter(models.BuildChroot.status == StatusEnum(state))
1248 1249 @classmethod
1250 - def filter_by_group_name(cls, query, group_name):
1251 return query.filter(models.Group.name == group_name)
1252
1253 1254 -class BuildsMonitorLogic(object):
1255 @classmethod
1256 - def get_monitor_data(cls, copr):
1257 query = """ 1258 SELECT 1259 package.id as package_id, 1260 package.name AS package_name, 1261 build.id AS build_id, 1262 build_chroot.status AS build_chroot_status, 1263 build.pkg_version AS build_pkg_version, 1264 mock_chroot.id AS mock_chroot_id, 1265 mock_chroot.os_release AS mock_chroot_os_release, 1266 mock_chroot.os_version AS mock_chroot_os_version, 1267 mock_chroot.arch AS mock_chroot_arch 1268 FROM package 1269 JOIN (SELECT 1270 MAX(build.id) AS max_build_id_for_chroot, 1271 build.package_id AS package_id, 1272 build_chroot.mock_chroot_id AS mock_chroot_id 1273 FROM build 1274 JOIN build_chroot 1275 ON build.id = build_chroot.build_id 1276 WHERE build.copr_id = {copr_id} 1277 AND build_chroot.status != 2 1278 GROUP BY build.package_id, 1279 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot 1280 ON package.id = max_build_ids_for_a_chroot.package_id 1281 JOIN build 1282 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1283 JOIN build_chroot 1284 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id 1285 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1286 JOIN mock_chroot 1287 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id 1288 JOIN copr_dir ON build.copr_dir_id=copr_dir.id WHERE copr_dir.main IS TRUE 1289 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC 1290 """.format(copr_id=copr.id) 1291 rows = db.session.execute(query) 1292 return rows
1293