source: soft/build_system/build_system/repsys/branches/V1_6_X/RepSys/log.py @ 1

Last change on this file since 1 was 1, checked in by fasma, 12 years ago

Initial Import from Mandriva's soft revision 224062 and package revision 45733

File size: 16.0 KB
Line 
1#!/usr/bin/python
2from RepSys import Error, config, RepSysTree
3from RepSys.svn import SVN
4from RepSys.util import execcmd
5
6try:
7    from Cheetah.Template import Template
8except ImportError:
9    raise Error, "repsys requires the package python-cheetah"
10
11import sys
12import os
13import re
14import time
15import locale
16import glob
17import tempfile
18import shutil
19
20
21locale.setlocale(locale.LC_ALL, "C")
22
23default_template = """
24#for $rel in $releases_by_author
25* $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release
26 ##
27 #if not $rel.released
28  (not released yet)
29 #end if
30 #for $rev in $rel.release_revisions
31  #for $line in $rev.lines
32  $line
33  #end for
34 #end for
35
36 #for $author in $rel.authors
37  + $author.name <$author.email>
38  #for $rev in $author.revisions
39    #for $line in $rev.lines
40    $line
41    #end for
42  #end for
43
44 #end for
45#end for
46"""
47
48def getrelease(pkgdirurl, rev=None, macros=[], exported=None):
49    """Tries to obtain the version-release of the package for a
50    yet-not-markrelease revision of the package.
51
52    Is here where things should be changed if "automatic release increasing"
53    will be used.
54    """
55    from RepSys.rpmutil import rpm_macros_defs
56    svn = SVN(baseurl=pkgdirurl)
57    pkgcurrenturl = os.path.join(pkgdirurl, "current")
58    specurl = os.path.join(pkgcurrenturl, "SPECS")
59    if exported is None:
60        tmpdir = tempfile.mktemp()
61        svn.export(specurl, tmpdir, rev=rev)
62    else:
63        tmpdir = os.path.join(exported, "SPECS")
64    try:
65        found = glob.glob(os.path.join(tmpdir, "*.spec"))
66        if not found:
67            raise Error, "no .spec file found inside %s" % specurl
68        specpath = found[0]
69        options = rpm_macros_defs(macros)
70        command = (("rpm -q --qf '%%{EPOCH}:%%{VERSION}-%%{RELEASE}\n' "
71                   "--specfile %s %s 2>/dev/null") % 
72                   (specpath, options))
73        status, output = execcmd(command)
74        if status != 0:
75            raise Error, "Error in command %s: %s" % (command, output)
76        releases = output.split()
77        try:
78            epoch, vr = releases[0].split(":", 1)
79            version, release = vr.split("-", 1)
80        except ValueError:
81            raise Error, "Invalid command output: %s: %s" % \
82                    (command, output)
83        #XXX check if this is the right way:
84        if epoch == "(none)":
85            ev = version
86        else:
87            ev = epoch + ":" + version
88        return ev, release
89    finally:
90        if exported is None and os.path.isdir(tmpdir):
91            shutil.rmtree(tmpdir)
92           
93class _Revision:
94    lines = []
95    date = None
96    raw_date = None
97    revision = None
98    author_name = None
99    author_email = None
100
101    def __init__(self, **kwargs):
102        self.__dict__.update(kwargs)
103
104    def __repr__(self):
105        lines = repr(self.lines)[:30] + "...]" 
106        line = "<_Revision %d author=%r date=%r lines=%s>" % \
107                    (self.revision, self.author, self.date, lines)
108        return line
109
110
111class _Release(_Revision):
112    version = None
113    release = None
114    revisions = []
115    release_revisions = []
116    authors = []
117    visible = False
118
119    def __init__(self, **kwargs):
120        self.revisions = []
121        _Revision.__init__(self, **kwargs)
122
123    def __repr__(self):
124        line = "<_Release v=%s r=%s revs=%r>" % \
125                    (self.version, self.release, self.revisions)
126        return line
127
128unescaped_macro_pat = re.compile(r"([^%])%([^%])")
129
130def escape_macros(text):
131    escaped = unescaped_macro_pat.sub("\\1%%\\2", text)
132    return escaped
133
134def format_lines(lines):
135    first = 1
136    entrylines = []
137    perexpr = re.compile(r"([^%])%([^%])")
138    for line in lines:
139        if line:
140            line = escape_macros(line)
141            if first:
142                first = 0
143                line = line.lstrip()
144                if line[0] != "-":
145                    nextline = "- " + line
146                else:
147                    nextline = line
148            elif line[0] != " " and line[0] != "-":
149                nextline = "  " + line
150            else:
151                nextline = line
152            if nextline not in entrylines:
153                entrylines.append(nextline)
154    return entrylines
155
156
157class _Author:
158    name = None
159    email = None
160    revisions = None
161
162
163def group_releases_by_author(releases):
164    allauthors = []
165    grouped = []
166    for release in releases:
167        authors = {}
168        latest = None
169        for revision in release.revisions:
170            authors.setdefault(revision.author, []).append(revision)
171
172        # all the mess below is to sort by author and by revision number
173        decorated = []
174        for authorname, revs in authors.iteritems():
175            author = _Author()
176            author.name = revs[0].author_name
177            author.email = revs[0].author_email
178            revdeco = [(r.revision, r) for r in revs]
179            revdeco.sort(reverse=1)
180            author.revisions = [t[1] for t in revdeco]
181            revlatest = author.revisions[0]
182            # keep the latest revision even for silented authors (below)
183            if latest is None or revlatest.revision > latest.revision:
184                latest = revlatest
185            count = sum(len(rev.lines) for rev in author.revisions)
186            if count == 0:
187                # skipping author with only silented lines
188                continue
189            decorated.append((revdeco[0][0], author))
190
191        if not decorated:
192            # skipping release with only authors with silented lines
193            continue
194
195        decorated.sort(reverse=1)
196        release.authors = [t[1] for t in decorated]
197        # the difference between a released and a not released _Release is
198        # the way the release numbers is obtained. So, when this is a
199        # released, we already have it, but if we don't, we should get de
200        # version/release string using getrelease and then get the first
201        first, release.authors = release.authors[0], release.authors[1:]
202        release.author_name = first.name
203        release.author_email = first.email
204        release.release_revisions = first.revisions
205
206        #release.date = first.revisions[0].date
207        release.date = latest.date
208        release.raw_date = latest.raw_date
209        #release.revision = first.revisions[0].revision
210        release.revision = latest.revision
211
212        grouped.append(release)
213
214    return grouped
215
216
217def group_revisions_by_author(currentlog):
218    revisions = []
219    last_author = None
220    for entry in currentlog:
221        revision = _Revision()
222        revision.lines = format_lines(entry.lines)
223        revision.raw_date = entry.date
224        revision.date = parse_raw_date(entry.date)
225        revision.revision = entry.revision
226        if entry.author == last_author:
227            revisions[-1].revisions.append(revision)
228        else:
229            author = _Author()
230            author.name, author.email = get_author_name(entry.author)
231            author.revisions = [revision]
232            revisions.append(author)
233        last_author = entry.author
234    return revisions
235
236
237emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>")
238
239def get_author_name(author):
240    found = emailpat.match(config.get("users", author, author))
241    name = ((found and found.group("name")) or author)
242    email = ((found and found.group("email")) or author)
243    return name, email
244
245def parse_raw_date(rawdate):
246    return time.strftime("%a %b %d %Y", rawdate)
247
248def filter_log_lines(lines):
249    # lines in commit messages containing SILENT at any position will be
250    # skipped; commits with their log messages beggining with SILENT in the
251    # first positionj of the first line will have all lines ignored.
252    ignstr = config.get("log", "ignore-string", "SILENT")
253    if len(lines) and lines[0].startswith(ignstr):
254        return []
255    filtered = [line for line in lines if ignstr not in line]
256    return filtered
257
258
259def make_release(author=None, revision=None, date=None, lines=None,
260        entries=[], released=True, version=None, release=None):
261    rel = _Release()
262    rel.author = author
263    if author:
264        rel.author_name, rel.author_email = get_author_name(author)
265    rel.revision = revision
266    rel.version = version
267    rel.release = release
268    rel.date = (date and parse_raw_date(date)) or None
269    rel.lines = lines
270    rel.released = released
271    rel.visible = False
272    for entry in entries:
273        lines = filter_log_lines(entry.lines)
274        if lines:
275            rel.visible = True
276        revision = _Revision()
277        revision.revision = entry.revision
278        revision.lines = format_lines(lines)
279        revision.date = parse_raw_date(entry.date)
280        revision.raw_date = entry.date
281        revision.author = entry.author
282        (revision.author_name, revision.author_email) = \
283                get_author_name(entry.author)
284        rel.revisions.append(revision)
285    return rel
286
287
288def dump_file(releases, currentlog=None, template=None):
289    templpath = template or config.get("template", "path", None)
290    params = {}
291    if templpath is None or not os.path.exists(templpath):
292        params["source"] = default_template
293        sys.stderr.write("warning: %s not found. using built-in template.\n"%
294                templpath)
295    else:
296        params["file"] = templpath
297    releases_author = group_releases_by_author(releases)
298    revisions_author = group_revisions_by_author(currentlog)
299    params["searchList"] = [{"releases_by_author" : releases_author,
300                             "releases" : releases,
301                             "revisions_by_author": revisions_author}]
302    t = Template(**params)
303    return t.respond()
304
305
306class InvalidEntryError(Exception):
307    pass
308
309def parse_repsys_entry(revlog):
310    # parse entries in the format:
311    # %repsys <operation>
312    # key: value
313    # ..
314    # <newline>
315    # <comments>
316    #
317    if len(revlog.lines) == 0 or not revlog.lines[0].startswith("%repsys"):
318        raise InvalidEntryError
319    try:       
320        data = {"operation" : revlog.lines[0].split()[1]}
321    except IndexError:
322        raise InvalidEntryError
323    for line in revlog.lines[1:]:
324        if not line:
325            break
326        try:
327            key, value = line.split(":", 1)
328        except ValueError:
329            raise InvalidEntryError
330        data[key.strip().lower()] = value.strip() # ???
331    return data
332       
333
334def get_revision_offset():
335    try:
336        revoffset = config.getint("log", "revision-offset", 0)
337    except (ValueError, TypeError):
338        raise Error, ("Invalid revision-offset number in configuration "
339                      "file(s).")
340    return revoffset or 0
341
342oldmsgpat = re.compile(
343        r"Copying release (?P<rel>[^\s]+) to (?P<dir>[^\s]+) directory\.")
344
345def parse_markrelease_log(relentry):
346    if not ((relentry.lines and oldmsgpat.match(relentry.lines[0]) \
347            or parse_repsys_entry(relentry))):
348        raise InvalidEntryError
349    from_rev = None
350    path = None
351    for changed in relentry.changed:
352        if changed["action"] == "A" and changed["from_rev"]:
353            from_rev = changed["from_rev"]
354            path = changed["path"]
355            break
356    else:
357        raise InvalidEntryError
358    # get the version and release from the names in the path, do not relay
359    # on log messages
360    version, release = path.rsplit(os.path.sep, 3)[-2:]
361    return version, release, from_rev
362
363
364def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
365        template=None, macros=[], exported=None):
366    concat = config.get("log", "concat", "").split()
367    revoffset = get_revision_offset()
368    svn = SVN(baseurl=pkgdirurl)
369    pkgreleasesurl = os.path.join(pkgdirurl, "releases")
370    pkgcurrenturl = os.path.join(pkgdirurl, "current")
371    releaseslog = svn.log(pkgreleasesurl, noerror=1)
372    currentlog = svn.log(pkgcurrenturl, limit=size, start=rev,
373            end=revoffset)
374
375    # sort releases by copyfrom-revision, so that markreleases for same
376    # revisions won't look empty
377    releasesdata = []
378    if releaseslog:
379        for relentry in releaseslog[::-1]:
380            try:
381                (version, release, relrevision) = \
382                        parse_markrelease_log(relentry)
383            except InvalidEntryError:
384                continue
385            releasesdata.append((relrevision, -relentry.revision, relentry, 
386                version, release))
387        releasesdata.sort()
388
389    # collect valid releases using the versions provided by the changes and
390    # the packages
391    prevrevision = 0
392    releases = []
393    for (relrevision, dummy, relentry, version, release) in releasesdata:
394        if prevrevision == relrevision: 
395            # ignore older markrelease of the same revision, since they
396            # will have no history
397            continue
398        entries = [entry for entry in currentlog
399                    if relrevision >= entry.revision and
400                      (prevrevision < entry.revision)]
401        if not entries:
402            #XXX probably a forced release, without commits in current/,
403            # check if this is the right behavior
404            sys.stderr.write("warning: skipping (possible) release "
405                    "%s-%s@%s, no commits since previous markrelease (r%r)\n" %
406                    (version, release, relrevision, prevrevision))
407            continue
408
409        release = make_release(author=relentry.author,
410                revision=relentry.revision, date=relentry.date,
411                lines=relentry.lines, entries=entries,
412                version=version, release=release)
413        releases.append(release)
414        prevrevision = relrevision
415           
416    # look for commits that have been not submited (released) yet
417    # this is done by getting all log entries newer (revision larger)
418    # than releaseslog[0] (in the case it exists)
419    if releaseslog:
420        latest_revision = releaseslog[0].revision
421    else:
422        latest_revision = 0
423    notsubmitted = [entry for entry in currentlog
424                    if entry.revision > latest_revision]
425    if notsubmitted:
426        # if they are not submitted yet, what we have to do is to add
427        # a release/version number from getrelease()
428        version, release = getrelease(pkgdirurl, macros=macros,
429                exported=exported)
430        toprelease = make_release(entries=notsubmitted, released=False,
431                        version=version, release=release)
432        releases.append(toprelease)
433
434    data = dump_file(releases[::-1], currentlog=currentlog, template=template)
435    return data
436
437
438
439def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
440        submit=False, template=None, macros=[], exported=None):
441    newlines = []
442    found = 0
443   
444    # Strip old changelogs
445    for line in open(specfile):
446        if line.startswith("%changelog"):
447            found = 1
448        elif not found:
449            newlines.append(line)
450        elif line.startswith("%"):
451            found = 0
452            newlines.append(line)
453
454    # Create new changelog
455    newlines.append("\n\n%changelog\n")
456    newlines.append(svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
457        template=template, macros=macros, exported=exported))
458
459    # Merge old changelog, if available
460    oldurl = config.get("log", "oldurl")
461    if oldurl:
462        svn = SVN(baseurl=pkgdirurl)
463        tmpdir = tempfile.mktemp()
464        try:
465            pkgname = RepSysTree.pkgname(pkgdirurl)
466            pkgoldurl = os.path.join(oldurl, pkgname)
467            try:
468                # we're using HEAD here because fixes in misc/ (oldurl) may
469                # be newer than packages' last changed revision.
470                svn.export(pkgoldurl, tmpdir)
471            except Error:
472                pass
473            else:
474                logfile = os.path.join(tmpdir, "log")
475                if os.path.isfile(logfile):
476                    file = open(logfile)
477                    newlines.append("\n")
478                    log = file.read()
479                    log = escape_macros(log)
480                    newlines.append(log)
481                    file.close()
482        finally:
483            if os.path.isdir(tmpdir):
484                shutil.rmtree(tmpdir)
485
486    # Write new specfile
487    file = open(specfile, "w")
488    file.write("".join(newlines))
489    file.close()
490
491
492if __name__ == "__main__":
493    l = svn2rpm(sys.argv[1])
494    print l
495
496# vim:et:ts=4:sw=4
Note: See TracBrowser for help on using the repository browser.