source: soft/build_system/build_system/repsys/tags/V1_6_17_1/RepSys/log.py @ 1

Last change on this file since 1 was 1, checked in by fasma, 12 years ago

Initial Import from Mandriva's soft revision 224062 and package revision 45733

File size: 15.6 KB
Line 
1#!/usr/bin/python
2from RepSys import Error, config, RepSysTree
3from RepSys.svn import SVN
4from RepSys.util import execcmd
5
6from Cheetah.Template import Template
7
8import sys
9import os
10import re
11import time
12import locale
13import glob
14import tempfile
15import shutil
16
17
18locale.setlocale(locale.LC_ALL, "C")
19
20default_template = """
21#for $rel in $releases_by_author
22* $rel.date $rel.author_name <$rel.author_email> $rel.version-$rel.release
23 ##
24 #if not $rel.released
25  (not released yet)
26 #end if
27 #for $rev in $rel.release_revisions
28  #for $line in $rev.lines
29  $line
30  #end for
31 #end for
32
33 #for $author in $rel.authors
34  + $author.name <$author.email>
35  #for $rev in $author.revisions
36    #for $line in $rev.lines
37    $line
38    #end for
39  #end for
40
41 #end for
42#end for
43"""
44
45def getrelease(pkgdirurl, rev=None, macros=[]):
46    """Tries to obtain the version-release of the package for a
47    yet-not-markrelease revision of the package.
48
49    Is here where things should be changed if "automatic release increasing"
50    will be used.
51    """
52    from RepSys.rpmutil import rpm_macros_defs
53    svn = SVN(baseurl=pkgdirurl)
54    tmpdir = tempfile.mktemp()
55    try:
56        pkgname = RepSysTree.pkgname(pkgdirurl)
57        pkgcurrenturl = os.path.join(pkgdirurl, "current")
58        specurl = os.path.join(pkgcurrenturl, "SPECS")
59        if svn.ls(specurl, noerror=1):
60            svn.export(specurl, tmpdir, rev=rev)
61            found = glob.glob(os.path.join(tmpdir, "*.spec"))
62            if found:
63                specpath = found[0]
64                options = rpm_macros_defs(macros)
65                command = (("rpm -q --qf '%%{VERSION}-%%{RELEASE}\n' "
66                           "--specfile %s %s 2>/dev/null") % 
67                           (specpath, options))
68                status, output = execcmd(command)
69                if status != 0:
70                    raise Error, "Error in command %s: %s" % (command, output)
71                releases = output.split()
72                try:
73                    version, release = releases[0].split("-", 1)
74                except ValueError:
75                    raise Error, "Invalid command output: %s: %s" % \
76                            (command, output)
77                return version, release
78    finally:
79        if os.path.isdir(tmpdir):
80            shutil.rmtree(tmpdir)
81           
82class _Revision:
83    lines = []
84    date = None
85    raw_date = None
86    revision = None
87    author_name = None
88    author_email = None
89
90    def __init__(self, **kwargs):
91        self.__dict__.update(kwargs)
92
93    def __repr__(self):
94        lines = repr(self.lines)[:30] + "...]" 
95        line = "<_Revision %d author=%r date=%r lines=%s>" % \
96                    (self.revision, self.author, self.date, lines)
97        return line
98
99
100class _Release(_Revision):
101    version = None
102    release = None
103    revisions = []
104    release_revisions = []
105    authors = []
106    visible = False
107
108    def __init__(self, **kwargs):
109        self.revisions = []
110        _Revision.__init__(self, **kwargs)
111
112    def __repr__(self):
113        line = "<_Release v=%s r=%s revs=%r>" % \
114                    (self.version, self.release, self.revisions)
115        return line
116
117unescaped_macro_pat = re.compile(r"([^%])%([^%])")
118
119def escape_macros(text):
120    escaped = unescaped_macro_pat.sub("\\1%%\\2", text)
121    return escaped
122
123def format_lines(lines):
124    first = 1
125    entrylines = []
126    perexpr = re.compile(r"([^%])%([^%])")
127    for line in lines:
128        if line:
129            line = escape_macros(line)
130            if first:
131                first = 0
132                line = line.lstrip()
133                if line[0] != "-":
134                    nextline = "- " + line
135                else:
136                    nextline = line
137            elif line[0] != " " and line[0] != "-":
138                nextline = "  " + line
139            else:
140                nextline = line
141            if nextline not in entrylines:
142                entrylines.append(nextline)
143    return entrylines
144
145
146class _Author:
147    name = None
148    email = None
149    revisions = None
150
151
152def group_releases_by_author(releases):
153    allauthors = []
154    grouped = []
155    for release in releases:
156        authors = {}
157        latest = None
158        for revision in release.revisions:
159            authors.setdefault(revision.author, []).append(revision)
160
161        # all the mess below is to sort by author and by revision number
162        decorated = []
163        for authorname, revs in authors.iteritems():
164            author = _Author()
165            author.name = revs[0].author_name
166            author.email = revs[0].author_email
167            revdeco = [(r.revision, r) for r in revs]
168            revdeco.sort(reverse=1)
169            author.revisions = [t[1] for t in revdeco]
170            revlatest = author.revisions[0]
171            # keep the latest revision even for silented authors (below)
172            if latest is None or revlatest.revision > latest.revision:
173                latest = revlatest
174            count = sum(len(rev.lines) for rev in author.revisions)
175            if count == 0:
176                # skipping author with only silented lines
177                continue
178            decorated.append((revdeco[0][0], author))
179
180        if not decorated:
181            # skipping release with only authors with silented lines
182            continue
183
184        decorated.sort(reverse=1)
185        release.authors = [t[1] for t in decorated]
186        # the difference between a released and a not released _Release is
187        # the way the release numbers is obtained. So, when this is a
188        # released, we already have it, but if we don't, we should get de
189        # version/release string using getrelease and then get the first
190        first, release.authors = release.authors[0], release.authors[1:]
191        release.author_name = first.name
192        release.author_email = first.email
193        release.release_revisions = first.revisions
194
195        #release.date = first.revisions[0].date
196        release.date = latest.date
197        release.raw_date = latest.raw_date
198        #release.revision = first.revisions[0].revision
199        release.revision = latest.revision
200
201        grouped.append(release)
202
203    return grouped
204
205
206def group_revisions_by_author(currentlog):
207    revisions = []
208    last_author = None
209    for entry in currentlog:
210        revision = _Revision()
211        revision.lines = format_lines(entry.lines)
212        revision.raw_date = entry.date
213        revision.date = parse_raw_date(entry.date)
214        revision.revision = entry.revision
215        if entry.author == last_author:
216            revisions[-1].revisions.append(revision)
217        else:
218            author = _Author()
219            author.name, author.email = get_author_name(entry.author)
220            author.revisions = [revision]
221            revisions.append(author)
222        last_author = entry.author
223    return revisions
224
225
226emailpat = re.compile("(?P<name>.*?)\s*<(?P<email>.*?)>")
227
228def get_author_name(author):
229    found = emailpat.match(config.get("users", author, author))
230    name = ((found and found.group("name")) or author)
231    email = ((found and found.group("email")) or author)
232    return name, email
233
234def parse_raw_date(rawdate):
235    return time.strftime("%a %b %d %Y", rawdate)
236
237def filter_log_lines(lines):
238    # lines in commit messages containing SILENT at any position will be
239    # skipped; commits with their log messages beggining with SILENT in the
240    # first positionj of the first line will have all lines ignored.
241    ignstr = config.get("log", "ignore-string", "SILENT")
242    if len(lines) and lines[0].startswith(ignstr):
243        return []
244    filtered = [line for line in lines if ignstr not in line]
245    return filtered
246
247
248def make_release(author=None, revision=None, date=None, lines=None,
249        entries=[], released=True, version=None, release=None):
250    rel = _Release()
251    rel.author = author
252    if author:
253        rel.author_name, rel.author_email = get_author_name(author)
254    rel.revision = revision
255    rel.version = version
256    rel.release = release
257    rel.date = (date and parse_raw_date(date)) or None
258    rel.lines = lines
259    rel.released = released
260    rel.visible = False
261    for entry in entries:
262        lines = filter_log_lines(entry.lines)
263        if lines:
264            rel.visible = True
265        revision = _Revision()
266        revision.revision = entry.revision
267        revision.lines = format_lines(lines)
268        revision.date = parse_raw_date(entry.date)
269        revision.raw_date = entry.date
270        revision.author = entry.author
271        (revision.author_name, revision.author_email) = \
272                get_author_name(entry.author)
273        rel.revisions.append(revision)
274    return rel
275
276
277def dump_file(releases, currentlog=None, template=None):
278    templpath = template or config.get("template", "path", None)
279    params = {}
280    if templpath is None or not os.path.exists(templpath):
281        params["source"] = default_template
282        sys.stderr.write("warning: %s not found. using built-in template.\n"%
283                templpath)
284    else:
285        params["file"] = templpath
286    releases_author = group_releases_by_author(releases)
287    revisions_author = group_revisions_by_author(currentlog)
288    params["searchList"] = [{"releases_by_author" : releases_author,
289                             "releases" : releases,
290                             "revisions_by_author": revisions_author}]
291    t = Template(**params)
292    return repr(t)
293
294
295class InvalidEntryError(Exception):
296    pass
297
298def parse_repsys_entry(revlog):
299    # parse entries in the format:
300    # %repsys <operation>
301    # key: value
302    # ..
303    # <newline>
304    # <comments>
305    #
306    if len(revlog.lines) == 0 or not revlog.lines[0].startswith("%repsys"):
307        raise InvalidEntryError
308    try:       
309        data = {"operation" : revlog.lines[0].split()[1]}
310    except IndexError:
311        raise InvalidEntryError
312    for line in revlog.lines[1:]:
313        if not line:
314            break
315        try:
316            key, value = line.split(":", 1)
317        except ValueError:
318            raise InvalidEntryError
319        data[key.strip().lower()] = value.strip() # ???
320    return data
321       
322
323def get_revision_offset():
324    try:
325        revoffset = config.getint("log", "revision-offset", 0)
326    except (ValueError, TypeError):
327        raise Error, ("Invalid revision-offset number in configuration "
328                      "file(s).")
329    return revoffset or 0
330
331oldmsgpat = re.compile(
332        r"Copying release (?P<rel>[^\s]+) to (?P<dir>[^\s]+) directory\.")
333
334def parse_markrelease_log(relentry):
335    if not ((relentry.lines and oldmsgpat.match(relentry.lines[0]) \
336            or parse_repsys_entry(relentry))):
337        raise InvalidEntryError
338    from_rev = None
339    path = None
340    for changed in relentry.changed:
341        if changed["action"] == "A" and changed["from_rev"]:
342            from_rev = changed["from_rev"]
343            path = changed["path"]
344            break
345    else:
346        raise InvalidEntryError
347    # get the version and release from the names in the path, do not relay
348    # on log messages
349    version, release = path.rsplit(os.path.sep, 3)[-2:]
350    return version, release, from_rev
351
352
353def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
354        template=None, macros=[]):
355    concat = config.get("log", "concat", "").split()
356    revoffset = get_revision_offset()
357    svn = SVN(baseurl=pkgdirurl)
358    pkgreleasesurl = os.path.join(pkgdirurl, "releases")
359    pkgcurrenturl = os.path.join(pkgdirurl, "current")
360    releaseslog = svn.log(pkgreleasesurl, noerror=1)
361    currentlog = svn.log(pkgcurrenturl, limit=size, start=rev,
362            end=revoffset)
363
364    # sort releases by copyfrom-revision, so that markreleases for same
365    # revisions won't look empty
366    releasesdata = []
367    if releaseslog:
368        for relentry in releaseslog[::-1]:
369            try:
370                (version, release, relrevision) = \
371                        parse_markrelease_log(relentry)
372            except InvalidEntryError:
373                continue
374            releasesdata.append((relrevision, -relentry.revision, relentry, 
375                version, release))
376        releasesdata.sort()
377
378    # collect valid releases using the versions provided by the changes and
379    # the packages
380    prevrevision = 0
381    releases = []
382    for (relrevision, dummy, relentry, version, release) in releasesdata:
383        if prevrevision == relrevision: 
384            # ignore older markrelease of the same revision, since they
385            # will have no history
386            continue
387        entries = [entry for entry in currentlog
388                    if relrevision >= entry.revision and
389                      (prevrevision < entry.revision)]
390        if not entries:
391            #XXX probably a forced release, without commits in current/,
392            # check if this is the right behavior
393            sys.stderr.write("warning: skipping (possible) release "
394                    "%s-%s@%s, no commits since previous markrelease (r%r)\n" %
395                    (version, release, relrevision, prevrevision))
396            continue
397
398        release = make_release(author=relentry.author,
399                revision=relentry.revision, date=relentry.date,
400                lines=relentry.lines, entries=entries,
401                version=version, release=release)
402        releases.append(release)
403        prevrevision = relrevision
404           
405    # look for commits that have been not submited (released) yet
406    # this is done by getting all log entries newer (revision larger)
407    # than releaseslog[0] (in the case it exists)
408    if releaseslog:
409        latest_revision = releaseslog[0].revision
410    else:
411        latest_revision = 0
412    notsubmitted = [entry for entry in currentlog
413                    if entry.revision > latest_revision]
414    if notsubmitted:
415        # if they are not submitted yet, what we have to do is to add
416        # a release/version number from getrelease()
417        version, release = getrelease(pkgdirurl, macros=macros)
418        toprelease = make_release(entries=notsubmitted, released=False,
419                        version=version, release=release)
420        releases.append(toprelease)
421
422    data = dump_file(releases[::-1], currentlog=currentlog, template=template)
423    return data
424
425
426
427def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None,
428        submit=False, template=None, macros=[]):
429    newlines = []
430    found = 0
431   
432    # Strip old changelogs
433    for line in open(specfile):
434        if line.startswith("%changelog"):
435            found = 1
436        elif not found:
437            newlines.append(line)
438        elif line.startswith("%"):
439            found = 0
440            newlines.append(line)
441
442    # Create new changelog
443    newlines.append("\n\n%changelog\n")
444    newlines.append(svn2rpm(pkgdirurl, rev=rev, size=size, submit=submit,
445        template=template, macros=macros))
446
447    # Merge old changelog, if available
448    oldurl = config.get("log", "oldurl")
449    if oldurl:
450        svn = SVN(baseurl=pkgdirurl)
451        tmpdir = tempfile.mktemp()
452        try:
453            pkgname = RepSysTree.pkgname(pkgdirurl)
454            pkgoldurl = os.path.join(oldurl, pkgname)
455            if svn.ls(pkgoldurl, noerror=1):
456                # we're using HEAD here because fixes in misc/ (oldurl) may
457                # be newer than packages' last changed revision.
458                svn.export(pkgoldurl, tmpdir)
459                logfile = os.path.join(tmpdir, "log")
460                if os.path.isfile(logfile):
461                    file = open(logfile)
462                    newlines.append("\n")
463                    log = file.read()
464                    log = escape_macros(log)
465                    newlines.append(log)
466                    file.close()
467        finally:
468            if os.path.isdir(tmpdir):
469                shutil.rmtree(tmpdir)
470
471    # Write new specfile
472    file = open(specfile, "w")
473    file.write("".join(newlines))
474    file.close()
475
476
477if __name__ == "__main__":
478    l = svn2rpm(sys.argv[1])
479    print l
480
481# vim:et:ts=4:sw=4
Note: See TracBrowser for help on using the repository browser.