Package run :: Module check_for_anitya_version_updates
[hide private]
[frames] | no frames]

Source Code for Module run.check_for_anitya_version_updates

  1  #!/usr/bin/python3 
  2   
  3  import subprocess 
  4  import argparse 
  5  import sys 
  6  import os 
  7  import json 
  8  import re 
  9  import logging 
 10   
 11  sys.path.append( 
 12      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 13  ) 
 14   
 15  from coprs import db, app, helpers 
 16  from coprs.logic.builds_logic import BuildsLogic 
 17  from coprs.logic.coprs_logic import CoprsLogic 
 18   
 19  logging.basicConfig( 
 20      filename="{0}/check_for_anitya_version_updates.log".format(app.config.get("LOG_DIR")), 
 21      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 22      level=logging.DEBUG) 
 23  log = logging.getLogger(__name__) 
 24  log.addHandler(logging.StreamHandler(sys.stdout)) 
 25   
 26  parser = argparse.ArgumentParser(description='Fetch package version updates by using datagrepper log of anitya emitted messages and issue rebuilds of the respective COPR packages for each such update. Requires httpie package.') 
 27   
 28  parser.add_argument('--backend', action='store', default='pypi', choices=['pypi', 'rubygems'], 
 29                     help='only check for updates from backend BACKEND, default pypi') 
 30  parser.add_argument('--delta', action='store', type=int, metavar='SECONDS', default=86400, 
 31                     help='ignore updates older than SECONDS, default 86400') 
 32  parser.add_argument('-v', '--version', action='version', version='1.0', 
 33                     help='print program version and exit') 
 34   
 35  args = parser.parse_args() 
 36   
 37   
38 -def run_cmd(cmd):
39 """ 40 Run given command in a subprocess 41 """ 42 log.info('Executing: '+' '.join(cmd)) 43 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 44 (stdout, stderr) = process.communicate() 45 if process.returncode != 0: 46 log.error(stderr) 47 sys.exit(1) 48 return stdout
49
50 -def to_json(data_bytes):
51 try: 52 data = data_bytes.decode("utf-8") 53 data_json = json.loads(data) 54 except Exception as e: 55 log.info(data) 56 log.exception(str(e)) 57 return data_json
58
59 -def get_updates_messages():
60 cmd_binary = 'curl' 61 url_template = 'https://apps.fedoraproject.org/datagrepper/raw?category=anitya&delta={delta}&topic=org.release-monitoring.prod.anitya.project.version.update&rows_per_page=64&order=asc&page={page}' 62 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=1)] 63 result_json = to_json(run_cmd(get_updates_cmd)) 64 messages = result_json['raw_messages'] 65 pages = result_json['pages'] 66 67 for p in range(2, pages+1): 68 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=p)] 69 result_json = to_json(run_cmd(get_updates_cmd)) 70 messages += result_json['raw_messages'] 71 72 return messages
73
74 -def get_updated_packages(updates_messages):
75 updated_packages = {} 76 for message in updates_messages: 77 update = message['msg'] 78 project = update['project'] 79 if args.backend.lower() != project['backend'].lower(): 80 continue 81 updated_packages[project['name'].lower()] = project['version'] 82 return updated_packages
83
84 -def get_copr_package_info_rows(updated_packages):
85 pkg_name_pattern = '(' + '|'.join(updated_packages.keys()) + ')' 86 source_type = helpers.BuildSourceEnum(args.backend.lower()) 87 if db.engine.url.drivername == "sqlite": 88 placeholder = '?' 89 true = '1' 90 else: 91 placeholder = '%s' 92 true = 'true' 93 rows = db.engine.execute( 94 """ 95 SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id 96 FROM package 97 LEFT OUTER JOIN build ON build.package_id = package.id 98 WHERE package.source_type = {placeholder} AND 99 package.source_json ~* '{pkg_name_pattern}' AND 100 package.webhook_rebuild = {true} AND 101 (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id)); 102 """.format(placeholder=placeholder, pkg_name_pattern=pkg_name_pattern, true=true), source_type 103 ) 104 return rows
105 106
107 -class RubyGemsPackage(object):
108 - def __init__(self, source_json):
109 self.name = source_json['gem_name'].lower()
110
111 - def build(self, copr, new_update_version):
112 return BuildsLogic.create_new_from_rubygems(copr.user, copr, self.name, chroot_names=None)
113 114
115 -class PyPIPackage(object):
116 - def __init__(self, source_json):
117 self.name = source_json['pypi_package_name'].lower() 118 self.python_versions = source_json['python_versions'] 119 self.spec_template = source_json['spec_template']
120
121 - def build(self, copr, new_updated_version):
122 return BuildsLogic.create_new_from_pypi(copr.user, copr, self.name, new_updated_version, 123 self.spec_template, self.python_versions, 124 chroot_names=None)
125 126
127 -def package_from_source(backend, source_json):
128 try: 129 return { 130 'pypi': PyPIPackage, 131 'rubygems': RubyGemsPackage, 132 }[backend](source_json) 133 except KeyError: 134 raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend))
135 136
137 -def main():
138 updated_packages = get_updated_packages(get_updates_messages()) 139 log.info('Updated packages according to datagrepper: {0}'.format(updated_packages)) 140 141 for row in get_copr_package_info_rows(updated_packages): 142 source_json = json.loads(row.source_json) 143 package = package_from_source(args.backend.lower(), source_json) 144 145 latest_build_version = row.pkg_version 146 log.info('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(package.name, row.package_id, row.copr_id)) 147 if package.name in updated_packages: 148 new_updated_version = updated_packages[package.name] 149 log.debug('name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(package.name, latest_build_version, new_updated_version)) 150 151 # if the last build's package version is "different" from new remote package version, rebuild 152 if not latest_build_version or not re.match(new_updated_version, latest_build_version): 153 try: 154 copr = CoprsLogic.get_by_id(row.copr_id)[0] 155 except Exception as e: 156 log.exception(e) 157 continue 158 159 log.info('Launching {} build for package of source name: {}, package_id: {}, copr_id: {}, user_id: {}' 160 .format(args.backend.lower(), package.name, row.package_id, copr.id, copr.user.id)) 161 build = package.build(copr, new_updated_version) 162 db.session.commit() 163 log.info('Launched build id {0}'.format(build.id))
164 165 if __name__ == '__main__': 166 try: 167 main() 168 except Exception as e: 169 log.exception(str(e)) 170