#!/usr/bin/python3.2 import argparse, os, portage, shutil, string from bugz.cli import PrettyBugz from bugz.bugzilla import BugzillaProxy from http.cookiejar import LWPCookieJar from gentoopm.portagepm.atom import PortageAtom, InvalidAtomStringError WORKDIR='/var/lib/gat' CVSROOT='/usr/portage' class ATJob: states = 'new', 'prepared' stored = 'url', 'bugid', 'packages', 'state', 'summary' def __init__(self, tag, workdir=WORKDIR, create=False): if not tag: raise ValueError("tag must not be empty") self.workdir=os.path.join(workdir, str(tag)) if not os.path.exists(self.workdir): if not create: raise ValueError("workdir %s does not exist" % self.workdir) os.makedirs(self.workdir) self.fn = dict(map(lambda item: (item, os.path.join(self.workdir, item)), self.stored)) self.tag = tag for k in self.stored: self.__dict__[k] = None self.read() def read(self): for k in self.stored: d = None try: f = open(self.fn.get(k), 'r') d = f.read() f.close() except IOError: pass if d and d.count('\n'): d = d.split('\n')[:-1] self.__dict__[k] = d self.packages = self.packages and \ list(map(lambda s: s.strip(), self.packages)) def write(self): for k in self.stored: d = self.__dict__.get(k, None) if type(d) == type([]): d = '\n'.join(d) + '\n' fn = self.fn.get(k) if not d: if os.path.exists(fn): os.unlink(fn) else: f = open(fn, 'w') f.write(d) f.close() def __str__(self): return '[%s][%s] %s\n' % (self.state, self.tag, self.summary) + \ '\t%s\n' % self.url + \ ''.join(map(lambda s: '\t%s\n' % s, self.packages or [])) def listjobs(workdir=WORKDIR): if not os.path.exists(workdir): return [] else: return sorted(filter(lambda s: s not in ['.git'], os.listdir(workdir))) def default_input(s, default=None): return input('%s [%s]: ' % (s, default)) or default def create(tag, workdir): job = ATJob(tag, workdir=workdir, create=True) if job.state: raise RuntimeError('job %s already exists in state %s' % (tag, job.state)) job.bugid = default_input('bug id', job.bugid or job.tag) if job.bugid.isdigit(): bp = BugzillaProxy('https://bugs.gentoo.org/xmlrpc.cgi') bug, = bp.Bug.get({'ids': int(job.bugid)})['bugs'] job.summary = default_input('summary', bug['summary']) job.url = default_input('url', job.url or 'https://bugs.gentoo.org/%s' % job.bugid) job.packages = [] for fragment in bug['summary'].split(): try: job.packages.append(PortageAtom(fragment)) except ValueError: pass except InvalidAtomStringError: pass job.packages = list(map(str, (filter(lambda a: a.complete, job.packages)))) else: job.summary = default_input('summary', None) job.url = default_input('url', job.url or 'https://bugs.gentoo.org/%s' % job.bugid) job.state = 'created' job.write() print(job) while input('edit packages list?'): run_command('%s %s' % (os.getenv('EDITOR', 'vim'), job.fn['packages'])) job.read() print(job) os.symlink(os.path.join(workdir, tag, 'packages'), os.path.join('/etc/portage/package.keywords', tag)) def create_all(tags, workdir, **args): if not tags: tags = [input('tag:'), ] if not tags[0]: raise RuntimeError('tag name must be given') for tag in tags: create(tag, workdir) def prepare(job): if job.state != 'created': raise RuntimeError('job already exists in state %s' % job.state) while run_command('emerge -pvt --autounmask --autounmask-keep-masks %s' % ' '.join(job.packages), die=False): input('Edit /etc/portage/package.keywords/%s and press enter' % job.tag) job.read() job.state = 'prepared' job.write() def build(job, rerun): if job.state not in ('prepared', 'failed'): raise RuntimeError('job already exists in state %s' % job.state) if run_command('emerge -vk1 --onlydeps %s %s' % (rerun and ' ' or '--noreplace', ' '.join(job.packages)), die=False): job.state = 'failed' elif run_command('emerge -v1 %s %s' % (rerun and ' ' or '--noreplace', ' '.join(job.packages)), die=False): job.state = 'failed' else: job.state = 'build' job.write() def exec_all(tags, workdir, execf, filterf, armor=True, one=False, **args): tags = tags or ATJob.listjobs(workdir=workdir) jobs = map(lambda tag: ATJob(tag, workdir=workdir), tags) for job in filter(filterf, jobs): armor and print(job) armor and print("\033k%s %s\033\\" % (job.state, job.tag)) execf(job) armor and print(job) armor and print("\033k%s %s\033\\" % (job.state, job.tag)) if one: break armor and print("\033k%s\033\\" % os.path.split(os.getenv('SHELL'))[1]) info_all = lambda tags, workdir, all, **args: exec_all(tags, workdir, lambda j: print(j), lambda j: all or not j.state == 'done', armor=False, **args) prepare_all = lambda tags, workdir, **args: exec_all(tags, workdir, prepare, lambda j: j.state in ['created'], **args) def build_all(tags, workdir, failed, rebuild, **args): filterf = failed and (lambda j: j.state in ['prepared', 'failed']) \ or (lambda j: j.state in ['prepared']) exec_all(tags, workdir, lambda j: build(j, rebuild), filterf) def run_command(cmd, die=True, cwd=os.getcwd()): print('>>> %s in %s' % (cmd, cwd)) cwd_ = os.getcwd() os.chdir(cwd) ret = os.system(cmd) if ret: if ret & 255: raise RuntimeError('killed by signal %i: %s' (ret & 255, cmd)) if ret // 256 and die: raise RuntimeError('returned %i: %s' % (ret // 256, cmd)) print('>>> done (%i): %s\n' % (ret // 256, cmd)) os.chdir(cwd_) return ret def commit(job, cvsroot=CVSROOT): if job.state != 'build': raise RuntimeError('job already exists in state %s' % job.state) error = list(filter(lambda p: p[0] != '=', job.packages)) if error: raise ValueError('package(s) do not start with =: %s' % ' '.join(error)) pn = lambda s: portage.versions.pkgsplit(s[1:])[0] p = lambda s: portage.versions.catsplit(s[1:])[1] e = lambda s: os.path.join(pn(s), p(s) + '.ebuild') run_command('cvs update %s' % ' '.join(map(pn, job.packages)), cwd=cvsroot) run_command('cvs diff %s' % ' '.join(map(pn, job.packages)), cwd=cvsroot) run_command('ekeyword ppc %s' % ' '.join(map(e, job.packages)), cwd=cvsroot) updates = [] for s in job.packages: if not run_command('cvs diff %s' % e(s), die=False, cwd=cvsroot) == 256: print('no changes in %s' % s) continue updates.append(s) run_command('repoman manifest', cwd=os.path.join(cvsroot, pn(s))) for s in updates: run_command('repoman commit --ask --echangelog=y -m "ppc stable (bug %s)."' % job.bugid, cwd=os.path.join(cvsroot, pn(s))) jar = LWPCookieJar(os.path.join(os.environ['HOME'], '.bugz_cookie')) jar.load() bp = BugzillaProxy('https://bugs.gentoo.org/xmlrpc.cgi', cookiejar=jar) bug, = bp.Bug.get({'ids': int(job.bugid)})['bugs'] arch_addr = map(lambda alias: alias + '@gentoo.org', portage.archlist) arch_cc = set(arch_addr) & set(bug['cc']) if 'ppc@gentoo.org' in arch_cc: mod = {'ids': int(job.bugid)} mod['comment'] = {'body': 'ppc stable.'} mod['cc'] = {'remove': ['ppc@gentoo.org']} if len(arch_cc) == 1 and not 'Security' in bug['product']: mod['status'] = 'RESOLVED' mod['resolution'] = 'FIXED' mod['comment']['body'] += ' last arch, closing.' print('updating bugzilla \n\t%s', '\n\t'.join(map(lambda a: ': '.join(map(str, a)), sorted(mod.items())))) bp.Bug.update(mod) job.state = 'done' job.write() commit_all = lambda tags, workdir, cvsroot=CVSROOT, **args: exec_all(tags, workdir, commit, lambda j: j.state in ['build'], **args) def pull(workdir=WORKDIR, **args): run_command('git pull origin master', cwd=workdir) def diff(workdir=WORKDIR, **args): run_command('git diff', cwd=workdir) def push(workdir=WORKDIR, **args): run_command('git add .', cwd=workdir) run_command('git commit -m "automated commit"', cwd=workdir, die=False) run_command('git push origin master', cwd=workdir) def set_state(state, tags, workdir=WORKDIR, **args): for tag in tags: job = ATJob(tag, workdir=workdir) print('%s: %s -> %s' % (tag, job.state, state)) job.state = state job.write() if __name__ == '__main__': parser = argparse.ArgumentParser(description='arch testing', formatter_class=argparse.ArgumentDefaultsHelpFormatter) subparsers = parser.add_subparsers(help='actions') def adddefaults(parser, tags=True): parser.add_argument('--workdir', default=WORKDIR, help='topdir for all operations') parser.add_argument('--one', action='store_true', default=False, help='exit after first step') parser.add_argument('tags', nargs='*', default=None, help='job name') subparser = subparsers.add_parser('info', help='print job informations', formatter_class=argparse.ArgumentDefaultsHelpFormatter) adddefaults(subparser) subparser.add_argument('--all', action='store_true', default=False, help='show all states') subparser.set_defaults(func=info_all) subparser = subparsers.add_parser('create', help='create job', formatter_class=argparse.ArgumentDefaultsHelpFormatter) adddefaults(subparser) subparser.set_defaults(func=create_all) subparser = subparsers.add_parser('prepare', help='prepare', formatter_class=argparse.ArgumentDefaultsHelpFormatter) adddefaults(subparser) subparser.set_defaults(func=prepare_all) subparser = subparsers.add_parser('build', help='run emerge build process', formatter_class=argparse.ArgumentDefaultsHelpFormatter) adddefaults(subparser) subparser.add_argument('--failed', action='store_true', default=False, help='retry failed jobs') subparser.add_argument('--rebuild', action='store_true', default=False, help='replace installed versions') subparser.set_defaults(func=build_all) subparser = subparsers.add_parser('commit', help='do cvs commit and bugzie', formatter_class=argparse.ArgumentDefaultsHelpFormatter) adddefaults(subparser, tags=False) subparser.set_defaults(func=commit_all) subparser = subparsers.add_parser('pull', help='pull remote data', formatter_class=argparse.ArgumentDefaultsHelpFormatter) adddefaults(subparser, tags=False) subparser.set_defaults(func=pull) subparser = subparsers.add_parser('diff', help='show differences', formatter_class=argparse.ArgumentDefaultsHelpFormatter) adddefaults(subparser, tags=False) subparser.set_defaults(func=diff) subparser = subparsers.add_parser('push', help='push remote data', formatter_class=argparse.ArgumentDefaultsHelpFormatter) adddefaults(subparser, tags=False) subparser.set_defaults(func=push) subparser = subparsers.add_parser('set_state', help='set state', formatter_class=argparse.ArgumentDefaultsHelpFormatter) subparser.add_argument('state', help='created|prepared|build|failed|done') adddefaults(subparser) subparser.set_defaults(func=set_state) args = parser.parse_args() args.workdir = os.path.expanduser(args.workdir) args.func(**dict(args._get_kwargs()))