[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20230209-39-gf5a68e0

Jon Turney jturney@sourceware.org
Sun Aug 20 17:44:48 GMT 2023




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f5a68e0073ccfa5e7b339d77d77376b134961f99

commit f5a68e0073ccfa5e7b339d77d77376b134961f99
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Wed Aug 16 13:51:37 2023 +0100

    Also allow announce message to be determined by cygport

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0e2738cbfd177fcb4d0d99f330ea3469e3e052d4

commit 0e2738cbfd177fcb4d0d99f330ea3469e3e052d4
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sun Aug 13 15:20:44 2023 +0100

    Try to add relevant changelog excerpt to announce message
    
    Look for a relevant section of changelog in README, between '----'
    delimiters, starting with one also containing the version, to add to the
    generated announce message.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=ac9d311b31da6e64f470a52982e62af825c97c91

commit ac9d311b31da6e64f470a52982e62af825c97c91
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sun Aug 13 14:19:32 2023 +0100

    Deploys can now automatically generate an announce email
    
    This is controlled by the 'announce' token.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=3d778791f91c1440fb6a5a7418d9549071ee7814

commit 3d778791f91c1440fb6a5a7418d9549071ee7814
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Jul 8 13:23:11 2023 +0100

    Factor out email sending to utils
    
    Smooth out some issues when --email isn't specified, making args.email
    an empty list rather than None, which can't be iterated.
    
    When the To: address is 'debug', dump email to log, rather than stdout.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=f2764f165b23b2e9ae03849c84fb7ed310ed12b7

commit f2764f165b23b2e9ae03849c84fb7ed310ed12b7
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Tue Aug 15 15:13:17 2023 +0100

    Explicitly use count keyword argument to re.sub()
    
    This avoids flake B034: sub should pass `count` and `flags` as keyword
    arguments to avoid confusion due to unintuitive argument positions.


Diff:
---
 calm/buffering_smtp_handler.py |  44 ++++-------------
 calm/calm.py                   | 105 +++++++++++++++++++++++++++++++++++++++--
 calm/untest.py                 |   2 +-
 calm/utils.py                  |  38 +++++++++++++++
 calm/version.py                |   2 +-
 test/test_calm.py              |   8 ++--
 test/test_entrypoints.py       |   4 +-
 7 files changed, 157 insertions(+), 46 deletions(-)

diff --git a/calm/buffering_smtp_handler.py b/calm/buffering_smtp_handler.py
index f6d5f34..d0a8119 100644
--- a/calm/buffering_smtp_handler.py
+++ b/calm/buffering_smtp_handler.py
@@ -21,13 +21,11 @@
 #
 
 
-import email.message
-import email.utils
 import logging
 import logging.handlers
-import subprocess
 
 from . import common_constants
+from . import utils
 
 
 class BufferingSMTPHandler(logging.handlers.BufferingHandler):
@@ -60,37 +58,15 @@ class BufferingSMTPHandler(logging.handlers.BufferingHandler):
 
             msg = msg + 'SUMMARY: ' + ', '.join(['%d %s(s)' % (v, k) for (k, v) in summary.items()]) + "\r\n"
 
-            # build the email
-            m = email.message.Message()
-            m['From'] = self.fromaddr
-            m['To'] = ','.join(self.toaddrs)
-            m['Reply-To'] = self.replytoaddr
-            m['Bcc'] = common_constants.ALWAYS_BCC
-            m['Subject'] = self.subject
-            m['Message-Id'] = email.utils.make_msgid()
-            m['Date'] = email.utils.formatdate()
-            m['X-Calm'] = '1'
-
-            # use utf-8 only if the message can't be ascii encoded
-            charset = 'ascii'
-            try:
-                msg.encode('ascii')
-            except UnicodeError:
-                charset = 'utf-8'
-            m.set_payload(msg, charset=charset)
-
-            # if toaddrs consists of the single address 'debug', just dump the mail we would have sent
-            if self.toaddrs == ['debug']:
-                print('-' * 40)
-                for k in m:
-                    print('%s: %s' % (k, m[k]))
-                print('-' * 40)
-                print(msg)
-                print('-' * 40)
-            else:
-                with subprocess.Popen(['/usr/sbin/sendmail', '-t', '-oi', '-f', self.fromaddr], stdin=subprocess.PIPE) as p:
-                    p.communicate(m.as_bytes())
-                    logging.debug('sendmail: msgid %s, exit status %d' % (m['Message-Id'], p.returncode))
+            hdr = {}
+            hdr['From'] = self.fromaddr
+            hdr['To'] = ','.join(self.toaddrs)
+            hdr['Reply-To'] = self.replytoaddr
+            hdr['Bcc'] = common_constants.ALWAYS_BCC
+            hdr['Subject'] = self.subject
+            hdr['X-Calm-Report'] = '1'
+
+            utils.sendmail(hdr, msg)
 
             self.buffer = []
 
diff --git a/calm/calm.py b/calm/calm.py
index 7ea4739..45fa09b 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -53,6 +53,7 @@
 #
 
 import argparse
+import codecs
 import functools
 import logging
 import lzma
@@ -63,6 +64,8 @@ import sys
 import tempfile
 import time
 
+import xtarfile
+
 from . import common_constants
 from . import db
 from . import irk
@@ -167,7 +170,11 @@ def process_uploads(args, state):
     def deploy_upload(r):
         m = mlist[r.user]
         with logfilters.AttrFilter(maint=m.name):
-            return process_maintainer_uploads(args, state, all_packages, m, os.path.join(args.stagingdir, str(r.id)), 'staging', scrub=True)
+            announce = ('announce' in r.tokens) and ('noannounce' not in r.tokens)
+            if announce and r.announce:
+                announce = r.announce
+
+            return process_maintainer_uploads(args, state, all_packages, m, os.path.join(args.stagingdir, str(r.id)), 'staging', scrub=True, announce=announce)
 
     scallywag_db.do_deploys(deploy_upload)
 
@@ -177,7 +184,7 @@ def process_uploads(args, state):
     return state.packages
 
 
-def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scrub=False):
+def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scrub=False, announce=False):
     # for each arch and noarch
     scan_result = {}
     success = True
@@ -198,6 +205,10 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scru
     if success:
         success = _process_maintainer_uploads(scan_result, args, state, all_packages, m, basedir, desc)
 
+    # automatically generate announce email if requested
+    if announce and success and any([scan_result[a].to_relarea for a in scan_result]):
+        _announce_upload(args, scan_result, m, announce)
+
     # remove upload files on success in homedir, always in stagingdir
     for arch in common_constants.ARCHES + ['noarch', 'src']:
         if scrub or success:
@@ -210,6 +221,92 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scru
     return success
 
 
+def _announce_upload(args, scan_result, maintainer, announce):
+    srcpkg = None
+    pkglist = set()
+    for arch in common_constants.ARCHES + ['noarch', 'src']:
+        for po in scan_result[arch].packages.values():
+            if po.kind == package.Kind.source:
+                srcpkg = po
+                assert len(po.versions()) == 1
+                version = list(po.versions())[0]
+                ldesc = po.version_hints[version]['ldesc'].strip('"')
+                test = 'test' in po.version_hints[version]
+
+            pkglist.add(po.orig_name)
+
+    if not srcpkg:
+        logging.error("could not locate source package in upload")
+        return
+    logging.debug("source package is %s, version %s, test %s", srcpkg.orig_name, version, test)
+
+    # find source tarfile for this particular package version
+    to = srcpkg.tar(version)
+    tf = to.repopath.abspath(args.rel_area)
+
+    if isinstance(announce, str):
+        # use announce message extracted from cygport, if present
+        cl = announce
+    else:
+        # otherwise, look in the source tar file for one of the files we know
+        # contains an announce message
+        cl = ''
+        with xtarfile.open(tf, mode='r') as a:
+            files = a.getnames()
+            for readme in ['README', srcpkg.orig_name + '.README', 'ANNOUNCE']:
+                fn = srcpkg.orig_name + '-' + version + '.src/' + readme
+                if fn in files:
+                    logging.debug("extracting %s from archive for changelog" % readme)
+
+                    f = codecs.getreader("utf-8")(a.extractfile(fn))
+
+                    # use the contents of an ANNOUNCE file verbatim
+                    if readme == 'ANNOUNCE':
+                        cl = f.read()
+                        break
+
+                    # otherwise, extract relevant part of ChangeLog from README
+                    # (between one '---- .* <version> ----' and the next '----' line)
+                    found = False
+                    for l in f:
+                        if not found:
+                            if l.startswith('----') and (version in l):
+                                cl = l
+                                found = True
+                        else:
+                            if l.startswith('----'):
+                                break
+                            cl = cl + '\n' + l
+
+                    break
+
+    # TODO: maybe other mechanisms for getting package ChangeLog?
+    # NEWS inside upstream source tarball?
+
+    # build the email
+    hdr = {}
+    hdr['From'] = maintainer.name + ' via Cygwin package uploader <cygwin-no-reply@cygwin.com>'
+    hdr['To'] = 'cygwin-announce@cygwin.com'
+    hdr['Reply-To'] = 'cygwin@cygwin.com'
+    hdr['Bcc'] = ','.join(maintainer.email)
+    hdr['Subject'] = srcpkg.orig_name + ' ' + version + (' (TEST)' if test else '')
+    hdr['X-Calm-Announce'] = '1'
+
+    msg = '''
+The following packages have been uploaded to the Cygwin distribution:
+
+%s
+
+%s
+
+%s
+''' % ('\n'.join('* ' + p + '-' + version for p in sorted(pkglist)), ldesc, cl)
+
+    # TODO: add an attachment: sha512 hashes of packages, gpg signed?
+
+    utils.sendmail(hdr, msg)
+
+
 def _process_maintainer_uploads(scan_result, args, state, all_packages, m, basedir, desc):
     name = m.name
 
@@ -301,7 +398,7 @@ def _process_maintainer_uploads(scan_result, args, state, all_packages, m, based
         # use merged package list
         state.packages[arch] = merged_packages[arch]
 
-    # report what we've done
+    # report what we've done to irc
     added = []
     for arch in common_constants.ARCHES + ['noarch', 'src']:
         added.append('%d (%s)' % (len(scan_result[arch].packages), arch))
@@ -754,7 +851,7 @@ def main():
 
     parser = argparse.ArgumentParser(description='Upset replacement')
     parser.add_argument('-d', '--daemon', action='store', nargs='?', const=pidfile_default, help="daemonize (PIDFILE defaults to " + pidfile_default + ")", metavar='PIDFILE')
-    parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help="email output to maintainer and ADDRS (ADDRS defaults to '" + common_constants.EMAILS + "')", metavar='ADDRS')
+    parser.add_argument('--email', action='store', dest='email', nargs='?', default='', const=common_constants.EMAILS, help="email output to maintainer and ADDRS (ADDRS defaults to '" + common_constants.EMAILS + "')", metavar='ADDRS')
     parser.add_argument('--force', action='count', help="force regeneration of static htdocs content", default=0)
     parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default)
     parser.add_argument('--htdocs', action='store', metavar='DIR', help="htdocs output directory (default: " + htdocs_default + ")", default=htdocs_default)
diff --git a/calm/untest.py b/calm/untest.py
index 7d6d774..7744108 100644
--- a/calm/untest.py
+++ b/calm/untest.py
@@ -53,7 +53,7 @@ def untest(pvr):
                         content = fh.read()
 
                     if re.search(r'^test:', content, re.MULTILINE):
-                        content = re.sub(r'^test:\s*$', '', content, 0, re.MULTILINE)
+                        content = re.sub(r'^test:\s*$', '', content, count=0, flags=re.MULTILINE)
 
                         with open(fn, 'w') as fh:
                             fh.write(content)
diff --git a/calm/utils.py b/calm/utils.py
index 9f75813..47085fd 100644
--- a/calm/utils.py
+++ b/calm/utils.py
@@ -25,6 +25,8 @@
 # utility functions
 #
 
+import email.message
+import email.utils
 import filecmp
 import logging
 import os
@@ -158,3 +160,39 @@ def mtime_cache(user_function):
         return result
 
     return wrapper
+
+
+def sendmail(hdr, msg):
+    # sending email not enabled
+    if not hdr['To']:
+        return
+
+    # build the email
+    m = email.message.Message()
+
+    for h in hdr:
+        m[h] = hdr[h]
+    m['Message-Id'] = email.utils.make_msgid()
+    m['Date'] = email.utils.formatdate()
+    m['X-Calm'] = '1'
+
+    # use utf-8 only if the message can't be ascii encoded
+    charset = 'ascii'
+    try:
+        msg.encode('ascii')
+    except UnicodeError:
+        charset = 'utf-8'
+    m.set_payload(msg, charset=charset)
+
+    # if To: header consists of the single address 'debug', just dump the mail we would have sent
+    if m['To'] == 'debug':
+        logging.debug('-' * 40)
+        for k in m:
+            logging.debug('%s: %s' % (k, m[k]))
+        logging.debug('-' * 40)
+        logging.debug(msg)
+        logging.debug('-' * 40)
+    else:
+        with subprocess.Popen(['/usr/sbin/sendmail', '-t', '-oi', '-f', hdr['From']], stdin=subprocess.PIPE) as p:
+            p.communicate(m.as_bytes())
+            logging.debug('sendmail: msgid %s, exit status %d' % (m['Message-Id'], p.returncode))
diff --git a/calm/version.py b/calm/version.py
index c61e8fc..5a4ce84 100644
--- a/calm/version.py
+++ b/calm/version.py
@@ -58,7 +58,7 @@ class SetupVersion:
             setattr(self, i, split[j])
             sequences = re.finditer(r'(\d+|[a-zA-Z]+|[^a-zA-Z\d]+)', split[j])
             sequences = [m for m in sequences if not re.match(r'[^a-zA-Z\d]+', m.group(1))]
-            sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in sequences]
+            sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), count=1) for m in sequences]
             setattr(self, '_' + i, sequences)
 
     def __str__(self):
diff --git a/test/test_calm.py b/test/test_calm.py
index 11697dc..819ce64 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -366,8 +366,8 @@ class CalmTest(unittest.TestCase):
         with open(args.inifile) as inifile:
             results = inifile.read()
             # fix the timestamp to match expected
-            results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1458221800', results, 1)
-            results = re.sub('generated at .*', 'generated at 2016-03-17 13:36:40 GMT', results, 1)
+            results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1458221800', results, count=1)
+            results = re.sub('generated at .*', 'generated at 2016-03-17 13:36:40 GMT', results, count=1)
             compare_with_expected_file(self, 'testdata/inifile', (results,), 'setup.ini')
 
         # XXX: delete a needed package, and check validate fails
@@ -453,8 +453,8 @@ class CalmTest(unittest.TestCase):
         with open(os.path.join(args.rel_area, 'setup.ini')) as inifile:
             results = inifile.read()
             # fix the timestamp to match expected
-            results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1473797080', results, 1)
-            results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, 1)
+            results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1473797080', results, count=1)
+            results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, count=1)
             compare_with_expected_file(self, 'testdata/process_arch', (results,), 'setup.ini')
 
         for d in ARGDIRS:
diff --git a/test/test_entrypoints.py b/test/test_entrypoints.py
index 82b1f23..030ef96 100644
--- a/test/test_entrypoints.py
+++ b/test/test_entrypoints.py
@@ -67,8 +67,8 @@ class EntryPointsTest(unittest.TestCase):
             results = inifile.read()
 
             # fix the timestamp to match expected
-            results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1680890562', results, 1)
-            results = re.sub('generated at .*', 'generated at 2023-04-07 18:02:42 GMT.', results, 1)
+            results = re.sub('setup-timestamp: .*', 'setup-timestamp: 1680890562', results, count=1)
+            results = re.sub('generated at .*', 'generated at 2023-04-07 18:02:42 GMT.', results, count=1)
 
             compare_with_expected_file(self, 'testdata/mksetupini', results, 'setup.ini')
 



More information about the Cygwin-apps-cvs mailing list