[Python] Quickbuild module - Release mate v1.10 #3

- 이전 build와 CL 차이 확인 추가
- 외부 xml file 사용하지 않도록 수정
- Release mate라고 이름도 붙임

==============================
import time
import re

import requests
#from xml.etree.ElementTree import ElementTree as ETR
import xml.etree.ElementTree as ETR
from xml.etree.ElementTree import Element as ET
from xml.etree.ElementTree import parse

VERSIONINFO = 'Release Mate v1.10'

BUILD_XML = 'build_data.xml'
BUILDINFOXML = 'buildinfo.xml'

QB_URL = 'https://android.qb.sec.samsung.net/rest/'
QB_TOKENRQ = 'https://android.qb.sec.samsung.net/signin?1-1.IFormSubmitListener-form'
QB_CONFPATH = 'ids?configuration_path='
QB_BUILDRQ = 'build_requests'
QB_BUILDS  = 'builds/'

class QB:
    def __init__(self):
        self.__qb_cookies = dict()

    def qblogin(self, user, password, reflash=False):
        if len(self.__qb_cookies) != 0:
            return self.session, self.__qb_cookies

        token_url = QB_TOKENRQ
        token_data = {'userName': user, 'password': password}
        self.session = requests.session()
        self.session.post(token_url, token_data)
        self.__qb_cookies = requests.utils.dict_from_cookiejar(self.session.cookies)

        if len(self.__qb_cookies) < 3:
            print('Quickbuild login failed ...\n')

        time.sleep(1)

        return self.session, self.__qb_cookies

    def qb_getconfid(self, path):
        if len(self.__qb_cookies) == 0:
            print('quickbuild has not been logged ...')
            return 0

        txt = requests.get(QB_URL + QB_CONFPATH + path, cookies=self.__qb_cookies, verify=True).text
        time.sleep(1)

        return txt

    def qb_getbuild(self, bid:str):
        if len(self.__qb_cookies) == 0:
            print('quickbuild has not been logged ...')
            return 0

        txt = requests.get(QB_URL + QB_BUILDS + bid, cookies=self.__qb_cookies, verify=True).text
# TO REMOVE XML FILE
#        with open(BUILDINFOXML, 'wt', encoding='utf-8') as f:
#            n = f.write(txt)
#        return n
        return txt

    def qb_getclinfo(self, xml):
        base = ''
        partials = ''

# TO REMOVE XML FILE
#        with open(BUILDINFOXML, 'rt', encoding='utf-8') as f:
        if xml != '':
#            tree = parse(BUILDINFOXML)
#            root = tree.getroot()
            root = ETR.fromstring(xml)
            sav = root.find('secretAwareVariableValues')
            entlist = sav.getchildren()
            for ent in entlist:
                l = ent.getchildren()
                if l[0].text == 'CL_SYNC':
                    base = l[1].getchildren()[0].text
                if l[0].text == 'CL_PARTIAL' :
                    sublist = l[1].getchildren()
                    if len(sublist) != 0:
                        partials = l[1].getchildren()[0].text
            if partials is None:
                partials = ''
        return base, partials

    def __qb_entry(self, vname, vval):
        ent = ET('entry')
        name = ET('string')
        name.text = vname
        ent.append(name)
        val = ET('string')
        val.text = vval
        ent.append(val)
        return ent

    def qb_data(self, confid, base, partials, usermode=True, csc=None):
        if type(confid) != str or type(base) != str or (partials != None and type(partials) != str):
            raise TypeError

        breq = ET('com.pmease.quickbuild.BuildRequest')

        cid = ET('configurationId')
        cid.text = confid
        breq.append(cid)

        variables = ET('variables')

        if usermode==True:
            variables.append(self.__qb_entry('DEFAULT_BUILD_MODE', 'USER_BUILD_DEFAULT_OPTIONS'))
        else:
            variables.append(self.__qb_entry('DEFAULT_BUILD_MODE', 'ENG_BUILD_DEFAULT_OPTIONS'))
        variables.append(self.__qb_entry('CL_SYNC', base))
        variables.append(self.__qb_entry('CL_PARTIAL', partials))
        variables.append(self.__qb_entry('CL_SHELVE', ''))
        if csc!=None:
            variables.append(self.__qb_entry('BUILD_CSC', csc))

        breq.append(variables)
# TO REMOVE XML FILE
#        ETR(breq).write(BUILD_XML, encoding='utf-8', method='xml', xml_declaration=True)
        return ETR.tostring(breq)

    def qb_build(self, confid, base, partials, usermode=True):
        qb_header = {'Content-Type': 'text/xml', 'charset': 'utf-8'}
# TO REMOVE XML FILE
#        self.qb_data(confid, base, partials, usermode)
#        rtn = requests.post(QB_URL + QB_BUILDRQ, data=open(BUILD_XML, 'rt', encoding='utf-8').read().replace('\n', ''), cookies=self.__qb_cookies, headers = qb_header)
        xml = self.qb_data(confid, base, partials, usermode)
        rtn = requests.post(QB_URL + QB_BUILDRQ, data=xml, cookies=self.__qb_cookies, headers = qb_header)

        time.sleep(1)

        return rtn

CLLIST = 'cl_list.txt'
CHKCL = 'checkcl.txt'
CLFORM = re.compile(r'\d+')
ENC = 'utf-8'
ENC2 = 'euc-kr'
ENC3 = 'cp949'

#################################################################################

class CL:
    qb = None

    def reset(self):
        self.refmode = 0        # 0 : use base and partial, 1 : user build-id
        self.kormode = 0
        self.depot = ''
        self.rbid = self.cbid = self.pbid = ''
        self.e = self.k = self.a = self.s = self.m = self.orgstr = self.errmsg = self.basecl = self.qb_base = self.qb_partials = ''
        self.pbase = self.ppartials = ''
        self.pmode = -1         # prev build 저장방식
        self.eurclset = set()
        self.korclset = set()
        self.addclset = set()
        self.subclset = set()
        self.mustset  = set()
        self.diff1 = []
        self.diff2 = []
        self.resultlist = []
        self.mustlist = []
        self.underlist = []
        self.newlist = []
        self.config = []
        self.confid = dict()

    def setfile(self, fn):
        self.filename = fn

    def __init__(self, qbinst=None, fn=None):
        self.qb = qbinst if qbinst != None else None
        self.setfile(CLLIST if fn==None else fn)
        self.reset()

    def setqb(qbinst):
        self.qb = qbinst

    def str2set(self, cls):
        CL = re.compile(r'\w+')
        clset = set()
        #
        for match in CL.finditer(cls):
            cl = match.group()
            clset.add(cl)
        return clset

    def set2str(self, clset):
        clstr = ''
        for cl in clset:
            clstr += str(cl)
            clstr += ','
        return clstr[:-1]

    def set2str2(self, clset, chkset):
        clstr = ''
        for cl in clset:
            clstr += str(cl)
            if cl in chkset:
                clstr += '*'
            clstr += ','
        return clstr[:-1]

    def __match_comment(self, line):
        temp = re.findall('^\s*#(.*)', line)
        if temp != []:
            return True
        return False

    def __matchcls(self, base, matchstr, line):
        match = False
        temp = re.findall('^\s*'+matchstr+'\s*=', line)
        if len(temp) > 0:
            match = True
        temp = re.findall('^\s*'+matchstr+'\s*=\s*([\d,\s]*)#{0,1}', line)
        if temp != []:
            if base != '':
                base += ','
            base += temp[0].replace(' ','').replace('\n','')
        return match, base

    def __getcls(self):
        self.reset()
        with open(self.filename, mode='rt', encoding='utf-8') as f:
            #
            ln = 0
            while(1):
                line = f.readline()
                ln += 1

                self.orgstr += line
                if line == '' or line[:13] == '# end of data':
                    break

                temp = []
                #
                temp = re.findall('\s*', line)
                if temp[0] == line:
                    continue

                temp = re.findall('^depot\s*=\s*([\w/\.]+)', line)
                if temp != []:
                    self.depot = temp[0]
                    continue

                temp = re.findall('^qb\s*=\s*([\w/]+)', line)
                if temp != []:
                    self.config.append(temp[0])
                    continue

                if self.pmode == -1:
                    temp = re.findall('^prev_buildid\s*=\s*([\d]+)', line)                 
                    if temp != []:
                        self.pbid = temp[0]
                        try:
                            xml = self.qb.qb_getbuild(self.pbid)
                            self.pbase, self.ppartials = self.qb.qb_getclinfo(xml)
                        except:
                            print('Error occured !!! : prev - self.pbase = %s, self.ppartials = %s' % (self.pbase, self.ppartials))

                        self.pmode = 1
                        continue

                    temp = re.findall('^prev_base\s*=\s*([\d]+)', line)
                    if temp != []:
                        self.pbase = temp[0]
                        self.pmode = 0
                        continue

                temp = re.findall('^ref_buildid\s*=\s*([\d]+)', line)
                if temp != []:
                    self.rbid = temp[0]
                    try:
                        xml = self.qb.qb_getbuild(self.rbid)
                        self.basecl, self.e = self.qb.qb_getclinfo(xml)
                    except:
                        print('Error occured !!! : ref - self.basecl = %s, self.e = %s' % (self.basecl, self.e))

                    self.refmode = 1
                    continue

                temp = re.findall('^cur_buildid', line)
                if temp != []:
                    temp = re.findall('^cur_buildid\s*=\s*([\d]+)', line)
                    if temp != []:
                        self.cbid = temp[0]
                        try:
                            xml = self.qb.qb_getbuild(self.cbid)
                            tempbase, self.k = self.qb.qb_getclinfo(xml)
                        except:
                            print('Error occured !!! : cur - tempbase = %s, self.k = %s' % (tempbase, self.k))

                        if tempbase != self.basecl:
                            print("Warning !!! ... current base is not the same one as ref base's ")

                    self.kormode = 1
                    continue

                if(self.refmode == 0):
                    temp = re.findall('^base\s*=\s*([\d]+)', line)
                    if temp != []:
                        self.basecl = temp[0]
                        continue
                #
                match = mt = False

                if self.pmode == 0:
                    mt, self.ppartials = self.__matchcls(self.e, 'prev_partials', line)
                    match |= mt           
                if self.refmode == 0:
                    mt, self.e = self.__matchcls(self.e, 'ref_partials', line)
                    match |= mt
                if self.kormode == 0:
                    mt, self.k = self.__matchcls(self.k, 'cur_partials', line)
                    match |= mt
                mt, self.a = self.__matchcls(self.a, 'add_partials', line)
                match |= mt
                mt, self.s = self.__matchcls(self.s, 'sub_partials', line)
                match |= mt
                mt, self.m = self.__matchcls(self.m, 'must_partials', line)
                match |= mt
                match |= self.__match_comment(line)
                if match == False :
                    print('Error in line #%d' % ln)
                    self.errmsg += 'Error in line #'
                    self.errmsg += str(ln)
                    self.errmsg += '\n'

                self.eurclset = self.str2set(self.e)
                self.korclset = self.str2set(self.k)
                self.addclset = self.str2set(self.a)
                self.subclset = self.str2set(self.s)
                self.mustset  = self.str2set(self.m)


    def __putcls(self):
        result_str = '------------------------------------------\n' +         \
        self.errmsg                                                      + '\n'    \
        '1. REF - CUR : \n' +                 \
         str([int(x) for x in self.diff1]).replace(', ',',')[1:-1] + '\n\n' +   \
        '2. CUR - REF : (Please check whether to add or not!) \n' + \
         str([int(x) for x in self.diff2]).replace(', ',',')[1:-1] + '\n\n' +   \
        '3. New CL list = REF + ADD CL - SUB CL (' + str(len(self.newcllist)) + ' CLs)\n\n' + 'Base : ' + str(self.basecl) + '\nPartial : \n' +   \
        str([int(x) for x in self.newcllist]).replace(', ',',')[1:-1] + '\n\n' +   \
        '***** CLs below the Base *****\n' +                               \
        str([int(x) for x in self.underlist]).replace(', ',',')[1:-1] + '\n'     
        #
        with open(self.filename, mode='wt+', encoding='utf-8') as f:
            f.write(self.orgstr)
            f.write(result_str)
        return result_str

    def update(self):
        print('##### process CL list "%s" ... #####' % self.filename)
     
        self.reset()
        self.__getcls()
     
        self.diff1 = list(self.eurclset - self.korclset) # - subclset
        self.diff2 = list(self.korclset - self.eurclset) # - addclset
        self.diff1.sort()
        self.diff2.sort()

        # self.korclset = self.korclset.union(eurclset)
        # self.korclset = self.korclset.union(addclset)
        self.korclset = self.eurclset | self.addclset   # korclset에 넣어둔 CL때문에 잘못 나오는 것이 있어 로직 변경

        self.resultlist = list(self.korclset - self.subclset)
        self.mustlist = list(self.mustset)

        self.underlist = [ x for x in self.resultlist if int(x) < int(self.basecl) ]
        self.newcllist = [ x for x in self.resultlist if int(x) > int(self.basecl) ]

        self.underlist.sort()
        self.newcllist.sort()
        self.mustlist.sort()

        for cl in self.mustlist:
            if cl not in self.newcllist:
                self.newcllist.append(cl)
     
        resultstr = self.__putcls()
        print('##### Completed #####\n')

        # prepare variables to build
        self.qb_base = str(self.basecl)
        self.qb_partials = str([int(x) for x in self.newcllist]).replace(', ',',')[1:-1]

        return resultstr

##################################################################

from P4 import P4

CHKCL = 'checkcl.txt'
CLFORM = re.compile(r'\d+')
ENC = 'utf-8'
ENC2 = 'euc-kr'
ENC3 = 'cp949'

def p4con(user, password):
    p4 = P4()
    p4.user = user
    p4.password = password
    p4.port = '165.213.202.46:1716'
    p4.client = 'KRSW_AND_CHINST_TRELTE_SERIES7'

    depot = '//PROD_PEACE/STAR_STAR2_CROWN/FLUMEN/...'

    p4.connect()

    return p4

def p4discon(p4):
    p4.disconnect()

# p4_run_changes()

def readfile():
    cllist = []
    with open(CHKCL, mode='r', encoding=ENC) as f:
        orgfile = ''
        while(1):
            line = f.readline()
            orgfile += line
            if line == '' or line[:13] == '# end of data':
                break
            cls = re.findall(CLFORM, line)
            for cl in cls:
                cllist.append(int(cl))
    return cllist, orgfile

def getchginfo(p4, cl):
    # i = p4.fetch_change(cl)
    i = p4.run('describe', cl)

    info = ''
    info += '=======================================================\n'
    info += '- CL ' + str(i[0].get('change')) + ', DATE ' + i[0].get('time') + ', User : ' + i[0].get('user') + '\n'
    info += '-------------------------------------\n'
    info += '- Description : \n'
    info += i[0].get('desc')
    info += '-------------------------------------\n'
    files = i[0].get('depotFile')
    if files :
        info += '- Files : ' + str(len(files)) + ' file(s)\n'
        for f in files:
            info += f
            info += '\n'
    info += '\n'
    return info

def getinfo(p4, cllist):
    info = ''
    for cl in cllist:
        info += getchginfo(p4, cl)
    return info

def writefile(orgstr, newstr):
    with open(CHKCL, mode='w', encoding=ENC) as f:
        f.write(orgstr)
        f.write(newstr)

def viewcls(user, password):
    cllist, orgstr = readfile()

    p4 = p4con(user, password)
    info = getinfo(p4, cllist)
    writefile(orgstr, info)

    p4discon(p4)

def viewcl(user, password, cl):
    p4 = p4con(user, password)
    info = getchginfo(p4, cl)
    p4discon(p4)

    return info

def yesno(str):
    while True:
        resp = input(str)
        resp = re.findall(r'(\w)', resp)
        if len(resp) == 0:
            continue
        resp = [x.lower() for x in resp]
        break

    print('')
    if resp[0] != 'y':
        return 0
    return 1

##################################################################

class Release:
    def __init__(self):
        self.qb = QB()
        self.cl = CL(self.qb, CLLIST)
 
    def update(self):
        return self.cl.update()

    def build(self, target='all'):
        if len(self.cl.config) == 0:
            print('Threre is no configurations to build ...\n')
            return
        print('Builds ... (base : %s, partials : %s)' % (self.cl.qb_base, self.cl.qb_partials))
        for x in self.cl.config:
            print('   Triggering build for %s ... ' % x)       
            cid = self.cl.confid.get(x, self.qb.qb_getconfid(x))
            if target=='user':
                self.qb.qb_build(cid, self.cl.qb_base, self.cl.qb_partials, True)
            if target=='eng' :
                self.qb.qb_build(cid, self.cl.qb_base, self.cl.qb_partials, False)
            if target=='all' :
                self.qb.qb_build(cid, self.cl.qb_base, self.cl.qb_partials, True)
                time.sleep(2)
                self.qb.qb_build(cid, self.cl.qb_base, self.cl.qb_partials, False)
            print('   Finished ...\n')       
            time.sleep(2)

    def build_check(self):
        print('Build check ... (base : %s, partials : %s)' % (self.cl.qb_base, self.cl.qb_partials))
        print('Build configurations are ...')
        for x in self.cl.config:
            print('- %s' % x)       
        print('\n')

    def getNewCL(self, user, password):
        if self.cl.pmode == -1:
            return []

        p4 = p4con(user, password)

        relcls = []
        if self.cl.pbase == self.cl.qb_base:
            # partial 추가된 것만 뽑기
            relcls = sorted(list(set(self.cl.qb_partials)-set(self.cl.ppartials)))
        else:
            # base 사이에 추가된 것 뽑기
            added = p4.run('changes', '-ssubmitted', self.cl.depot+'@'+self.cl.pbase+','+self.cl.qb_base)
            for x in added:
                cl = x.get('change')
                # 이전 partial에 포함된 것은 빼기
                if cl in self.cl.ppartials:
                    continue
                relcls.append(cl)       
            # 이번 partial에 추가된 것은 더하기
            addcls = self.cl.qb_partials.split(',')
            if len(addcls) == 1 and addcls[0] != '':
                relcls += addcls
            # 중복된 것 삭제
            relcls = list(set(relcls))
            relcls.sort(key=str)

        p4discon(p4)

        return relcls

    def REL2FLUMEN(self, user, password, relcls):
        resultlist = []
        resultstr = ''
        chg = r'\*+\sCherry-picking\s([\d]+)'
        flm = r'/FLUMEN/'

        p4 = p4con(user, password)
        for relcl in relcls:
            p4cl = p4.run('describe', relcl)
            if len(p4cl) == 0:
                errstr = '   @........ -> @???????? : cannot find' + relcl
                resultstr = resultstr + errstr + '\n'
                print(errstr)
                continue
            fcl = re.findall(chg, p4cl[0].get('desc'))
            if len(fcl) == 0:
                errstr = '   @........ -> @' + relcl + ' : cannot process description for ' + relcl
                resultstr = resultstr + errstr + '\n'
                print(errstr)
                continue
            p4fcl = p4.run('describe', fcl[0])
            if len(p4fcl) == 0:
                errstr = '   @???????? -> @' + relcl + ' : Cannot find cherry picked cl ' + fcl[0] + ' for org cl ' + relcl
                resultstr = resultstr + errstr + '\n'
                print(errstr)
                continue
         
            depotfiles = p4fcl[0].get('depotFile')
            chkflm = False
            for x in depotfiles:
                if len(re.findall(flm, x)) > 0:
                    chkflm = True

            if chkflm != True:
                errstr = '   @' + fcl[0] + ' -> @' + relcl + ' : Found a cherry picked cl! But not FLUMEN'
                resultstr = resultstr + errstr + '\n'
                print(errstr)
                continue

            errstr = '   @' + fcl[0] + ' -> @' + relcl
            resultstr = resultstr + errstr + '\n'
            print(errstr)
         
            resultlist.append((fcl[0], relcl))

        resultdict = dict(resultlist)

        p4discon(p4) 

        return (resultstr, resultdict)

    def checkNewCL(self, user, password):
        relcls = self.getNewCL(user, password)

        if len(relcls) < 0:
            print('There is nothing new or failed to obtain new cls due to unknown errors.\n')
            return

        resultlist = []
        if yesno('Do you wanna convert REL CLs into FLUMEN CLs ? : ') == 0:
            print('Newly added CLs are ...')
            for x in relcls:
                print('@%s' % x)
                resultlist.append((x, x))
            resultdict = dict(resultlist)
            return resultdict

        resultstr, resultdict = self.REL2FLUMEN(user, password, relcls)
        # CSV update하는 코드도 추가 예정

##################################################################

# release = Release()

# update = lambda : release.u()
# build = lambda : release.b()

##################################################################

class ReleaseUI:
    user = ''
    password = ''

    def __qblogin(self):
        if self.user == '' or self.password == '':
           print('You should enter login information agian.\nMenu login finished.')

        print('Quickbuild log-in ...\n')
        self.release.qb.qblogin(self.user, self.password)

    def doLogin(self):
        if self.user == '':
            self.user = input('userName cannot be null.\nPlease enter userName : ')
        if self.password == '':
            self.password = input('password cannot be null.\nPlease enter password : ')

        self.__qblogin()

    def doReLogin(self):
        self.user = input('Please enter userName : ')
        self.password = input('Please tner password : ')
        self.doLogin()

    def __printVer(self):
        print(VERSIONINFO + '\n' + '-'*30 + '\n')

    def __init__(self):
        self.__printVer()
     
        self.release = Release()
        self.doReLogin()

    def doUpdate(self):
        print(self.release.update())

    def doBuild(self, target='all'):
        if target == 'all' or target == 'user' or target == 'eng':
            self.release.build(target)
        else:
            print('Cannot recognize build option : %s' % target)

    def doCheckBuild(self, target='all'):
        self.release.build_check()

    def doCheckCL(self):
        viewcls(self.user, self.password)
        print('CL info has been updated ...\n')

    def doCheckNewCL(self):
        self.release.checkNewCL(self.user, self.password)

    def doHelp(self):
        self.__printVer()
        print('Please be sure to place "cl_list.txt" at the same folder with this Program !!! \n')
        print('commands')
        print('- help or ? : this screen')
        print('- update : update cl_list.txt')
        print('- login  : log on Quickbuild server')
        print('- relogin  : log on Quickbuild server with new user and password')
        print('- build  : build with respect to the CLs on cl_list.txt')
        print('           build or build all - build user and eng binary')
        print('           build user         - build user binary')
        print('           build eng          - build eng binary')
        print('           to use this, you should login on QB server')
        print('- newbuild : same as what "build" does.')
        print('             But "update" will be performed before build is processed')
        print('- check build : display parameters when build command is performed')
        print('        newcl : check newly added cl w.r.t the previous build ')
        print('                to use this, ')
        print('                prev_buildid or prev_base and prev_partials should be provided')
        print('                and log on P4 server \( assume that P4 user info is the same as QB \) ')
        print('- exit   : Exit from menu')
        print('')

    def __call__(self):
        while True:
            instr = input('Enter command ? ')
            command = re.findall(r'([\w]+|\?)', instr)
            cmd = [x.lower() for x in command]
            if len(cmd) == 0:
                continue

            if cmd[0] == 'update':
                self.doUpdate()
            elif cmd[0] == 'login':
                self.doLogin()
            elif cmd[0] == 'relogin':
                self.doReLogin()
            elif cmd[0] == 'build' :
                if len(cmd) > 1:
                    self.doBuild(target=cmd[1])
                else:
                    self.doBuild()
            elif cmd[0] == 'newbuild' :
                self.doUpdate()
                if yesno('=> Wanna build ? (Y/N) : ') != 0:
                    continue

                if len(cmd) > 1:
                    self.doBuild(target=cmd[1])
                else:
                    self.doBuild()
            elif cmd[0] == 'check' :
                if len(cmd) > 1:
                    if cmd[1] == 'cl':
                        self.doCheckCL()
                    elif cmd[1] == 'build':
                        self.doCheckBuild()
                    elif cmd[1] == 'newcl':
                        self.doCheckNewCL()
            elif cmd[0] == 'help' or cmd[0] == '?':
                self.doHelp()
            elif cmd[0] == 'exit':
                break;
            else:
                print('Invalid command : %s' % command[0])

if __name__ == '__main__':
    try:
        with open(CLLIST, 'rt') as f:
            pass
        R = ReleaseUI()
        R()
    except:
        print('Please be sure to place "cl_list.txt" at the same folder with this Program !!! \n')