This is the mail archive of the cygwin-apps-cvs mailing list for the cygwin-apps project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-64-g5d7e122




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=5d7e122c1e2647fae84cd13ec25466fefd031d43

commit 5d7e122c1e2647fae84cd13ec25466fefd031d43
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Apr 6 19:25:49 2017 +0100

    Add daemon mode for calm
    
    SIGUSR1 tells it to re-read the upload area
    SIGUSR2 tells it to re-read the release area
    
    This avoids maintainers having to wait a random amount of time to discover
    the result of their uploads
    
    For the moment, we still re-read everything every 30 minutes, but this could
    be increased, which would be a good deal more efficient, as we won't be
    re-reading the state of the relarea when nothing has changed.


Diff:
---
 TODO              |    1 -
 calm/calm.py      |  233 +++++++++++++++++++++++++++++++++++++++++++----------
 test/test_calm.py |    4 +-
 3 files changed, 194 insertions(+), 44 deletions(-)

diff --git a/TODO b/TODO
index 63afee3..74a729d 100644
--- a/TODO
+++ b/TODO
@@ -1,5 +1,4 @@
 * more than 2 versions possible
-* run more often, option to not do anything if no uploads (to avoid reading the release area if we don't need to), lockfile to avoid colliding runs
 * use irkerd to report when calm failed due to an error?
 * upload a hash at the same time as package, and pass that through to setup.ini
 * mksetupini should have an okmissing option for override.hint which names non-existent versions
diff --git a/calm/calm.py b/calm/calm.py
index 08410d1..33bc05d 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -58,8 +58,10 @@ import argparse
 import logging
 import os
 import shutil
+import signal
 import sys
 import tempfile
+import time
 
 from .abeyance_handler import AbeyanceHandler
 from .buffering_smtp_handler import BufferingSMTPHandler
@@ -76,17 +78,21 @@ from . import uploads
 #
 #
 
-def process(args):
-    subject = 'calm%s: cygwin package upload report from %s' % (' [dry-run]' if args.dryrun else '', os.uname()[1])
+class CalmState(object):
+    def __init__(self):
+        self.subject = ''
+        self.packages = {}
 
-    # send one email per run to leads, if any errors occurred
-    with mail_logs(args.email, toaddrs=args.email, subject='%s' % (subject), thresholdLevel=logging.ERROR) as leads_email:
-        if args.dryrun:
-            logging.warning("--dry-run is in effect, nothing will really be done")
 
-        # for each arch
-        error = False
+#
+#
+#
+
+def process_relarea(args):
         packages = {}
+        error = False
+
+        # for each arch
         for arch in common_constants.ARCHES:
             logging.debug("reading existing packages for arch %s" % (arch))
 
@@ -113,6 +119,14 @@ def process(args):
                 logging.error("error while evaluating stale packages")
                 return None
 
+        return packages
+
+
+#
+#
+#
+
+def process_uploads(args, state):
         # read maintainer list
         mlist = maintainers.Maintainer.read(args)
 
@@ -124,7 +138,7 @@ def process(args):
             m = mlist[name]
 
             # also send a mail to each maintainer about their packages
-            with mail_logs(args.email, toaddrs=m.email, subject='%s for %s' % (subject, name), thresholdLevel=logging.INFO) as maint_email:
+            with mail_logs(args.email, toaddrs=m.email, subject='%s for %s' % (state.subject, name), thresholdLevel=logging.INFO) as maint_email:
 
                 # for each arch and noarch
                 scan_result = {}
@@ -162,7 +176,7 @@ def process(args):
                     logging.debug("merging %s package set with uploads from maintainer %s" % (arch, name))
 
                     # merge package sets
-                    merged_packages[arch] = package.merge(packages[arch], scan_result[arch].packages, scan_result['noarch'].packages)
+                    merged_packages[arch] = package.merge(state.packages[arch], scan_result[arch].packages, scan_result['noarch'].packages)
                     if not merged_packages[arch]:
                         logging.error("error while merging uploaded %s packages for %s" % (arch, name))
                         valid = False
@@ -231,13 +245,32 @@ def process(args):
                 # for each arch
                 for arch in common_constants.ARCHES:
                     # use merged package list
-                    packages[arch] = merged_packages[arch]
+                    state.packages[arch] = merged_packages[arch]
                     logging.debug("added %d + %d packages from maintainer %s" % (len(scan_result[arch].packages), len(scan_result['noarch'].packages), name))
 
         # record updated reminder times for maintainers
         maintainers.Maintainer.update_reminder_times(mlist)
 
-    return packages
+        return state.packages
+
+
+#
+#
+#
+
+def process(args, state):
+    # send one email per run to leads, if any errors occurred
+    with mail_logs(args.email, toaddrs=args.email, subject='%s' % (state.subject), thresholdLevel=logging.ERROR) as leads_email:
+        if args.dryrun:
+            logging.warning("--dry-run is in effect, nothing will really be done")
+
+        state.packages = process_relarea(args)
+        if not state.packages:
+            return None
+
+        state.packages = process_uploads(args, state)
+
+    return state.packages
 
 
 #
@@ -310,19 +343,29 @@ def report_movelist_conflicts(a, b, reason):
 #
 #
 
-def do_main(args):
+def do_main(args, state):
     # read package set and process uploads
-    packages = process(args)
+    packages = process(args, state)
 
     if not packages:
         logging.error("not processing uploads or writing setup.ini")
         return
 
+    state.packages = packages
+
+    do_output(args, state)
+
+
+#
+#
+#
+
+def do_output(args, state):
     # for each arch
     for arch in common_constants.ARCHES:
         # update packages listings
         # XXX: perhaps we need a --[no]listing command line option to disable this from being run?
-        pkg2html.update_package_listings(args, packages[arch], arch)
+        pkg2html.update_package_listings(args, state.packages[arch], arch)
 
     # for each arch
     for arch in common_constants.ARCHES:
@@ -342,7 +385,7 @@ def do_main(args):
             changed = False
 
             # write setup.ini
-            package.write_setup_ini(args, packages[arch], arch)
+            package.write_setup_ini(args, state.packages[arch], arch)
 
             if not os.path.exists(inifile):
                 # if the setup.ini file doesn't exist yet
@@ -380,7 +423,7 @@ def do_main(args):
                         elif ext == '.xz':
                             os.system('/usr/bin/xz -6e <%s >%s' % (inifile, os.path.splitext(inifile)[0] + ext))
 
-                        os.system('/usr/bin/gpg --batch --yes -b ' + os.path.join(basedir, 'setup' + ext))
+                        os.system('/usr/bin/gpg --batch --yes -b </dev/null ' + os.path.join(basedir, 'setup' + ext))
 
                     # arrange for checksums to be recomputed
                     for sumfile in ['md5.sum', 'sha512.sum']:
@@ -394,6 +437,99 @@ def do_main(args):
 
 
 #
+# daemonization loop
+#
+
+def do_daemon(args, state):
+    import daemon
+    import lockfile.pidlockfile
+
+    context = daemon.DaemonContext(
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        pidfile=lockfile.pidlockfile.PIDLockFile('/var/run/calm.pid'))
+
+    running = True
+    read_relarea = True
+    read_uploads = True
+
+    # signals! the first, and best, interprocess communications mechanism! :)
+    def sigusr1(signum, frame):
+        logging.info("SIGUSR1")
+        nonlocal read_uploads
+        read_uploads = True
+
+    def sigusr2(signum, frame):
+        logging.info("SIGUSR2")
+        nonlocal read_relarea
+        read_relarea = True
+
+    def sigalrm(signum, frame):
+        logging.info("SIGALRM")
+        nonlocal read_relarea
+        read_relarea = True
+        nonlocal read_uploads
+        read_uploads = True
+
+    def sigterm(signum, frame):
+        logging.info("SIGTERM")
+        nonlocal running
+        running = False
+
+    context.signal_map = {
+        signal.SIGUSR1: sigusr1,
+        signal.SIGUSR2: sigusr2,
+        signal.SIGALRM: sigalrm,
+        signal.SIGTERM: sigterm,
+    }
+
+    with context:
+        logging_setup(args)
+        logging.info("calm daemon started, pid %d" % (os.getpid()))
+
+        state.packages = {}
+
+        while running:
+            with mail_logs(args.email, toaddrs=args.email, subject='%s' % (state.subject), thresholdLevel=logging.ERROR) as leads_email:
+                # re-read relarea on SIGALRM or SIGUSR2
+                if read_relarea:
+                    read_relarea = False
+                    state.packages = process_relarea(args)
+
+                if not state.packages:
+                    logging.error("not processing uploads or writing setup.ini")
+                else:
+                    if read_uploads:
+                        # read uploads on SIGUSR1
+                        read_uploads = False
+                        state.packages = process_uploads(args, state)
+
+                    do_output(args, state)
+
+                    # if there is more work to do, but don't spin if we can't do it
+                    if read_uploads:
+                        continue
+
+            # we wake at a 10 minute offset from the next 30 minute boundary
+            # (i.e. at :10 or :40 past the hour) to check the state of the
+            # release area, in case someone has ninja-ed in a change there...
+            interval = 30*60
+            offset = 10*60
+            delay = interval - ((time.time() - offset) % interval)
+            signal.alarm(int(delay))
+
+            # wait until interrupted by a signal
+            logging.info("sleeping for %d seconds" % (delay))
+            signal.pause()
+            logging.info("woken")
+
+            # cancel any pending alarm
+            signal.alarm(0)
+
+        logging.info("calm daemon stopped")
+
+
+#
 # we only want to mail the logs if the email option was used
 # (otherwise use ExitStack() as a 'do nothing' context)
 #
@@ -406,6 +542,35 @@ def mail_logs(enabled, toaddrs, subject, thresholdLevel, retainLevel=None):
 
 
 #
+# setup logging configuration
+#
+
+def logging_setup(args):
+    # set up logging to a file
+    try:
+        os.makedirs(args.logdir, exist_ok=True)
+    except FileExistsError:
+        pass
+    rfh = logging.handlers.TimedRotatingFileHandler(os.path.join(args.logdir, 'calm.log'), backupCount=48, when='midnight')
+    rfh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s'))
+    rfh.setLevel(logging.DEBUG)
+    logging.getLogger().addHandler(rfh)
+
+    # setup logging to stdout, of WARNING messages or higher (INFO if verbose)
+    ch = logging.StreamHandler(sys.stdout)
+    ch.setFormatter(logging.Formatter(os.path.basename(sys.argv[0])+': %(message)s'))
+    if args.verbose:
+        ch.setLevel(logging.INFO)
+    else:
+        ch.setLevel(logging.WARNING)
+    logging.getLogger().addHandler(ch)
+
+    # change root logger level from the default of WARNING to NOTSET so it
+    # doesn't filter out any log messages due to level
+    logging.getLogger().setLevel(logging.NOTSET)
+
+
+#
 #
 #
 
@@ -421,6 +586,7 @@ def main():
     queuedir_default = '/sourceware/cygwin-staging/queue'
 
     parser = argparse.ArgumentParser(description='Upset replacement')
+    parser.add_argument('-d', '--daemon', action='store_true', help="daemonize")
     parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help='email output to maintainer and ADDRS (default: ' + common_constants.EMAILS + ')', metavar='ADDRS')
     parser.add_argument('--force', action='store_true', help="overwrite existing files")
     parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default)
@@ -439,34 +605,17 @@ def main():
     parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output')
     (args) = parser.parse_args()
 
-    # set up logging to a file
-    try:
-        os.makedirs(args.logdir, exist_ok=True)
-    except FileExistsError:
-        pass
-    rfh = logging.handlers.RotatingFileHandler(os.path.join(args.logdir, 'calm.log'), backupCount=48)
-    rfh.doRollover()  # force a rotate on every run
-    rfh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s'))
-    rfh.setLevel(logging.DEBUG)
-    logging.getLogger().addHandler(rfh)
-
-    # setup logging to stdout, of WARNING messages or higher (INFO if verbose)
-    ch = logging.StreamHandler(sys.stdout)
-    ch.setFormatter(logging.Formatter(os.path.basename(sys.argv[0])+': %(message)s'))
-    if args.verbose:
-        ch.setLevel(logging.INFO)
-    else:
-        ch.setLevel(logging.WARNING)
-    logging.getLogger().addHandler(ch)
-
-    # change root logger level from the default of WARNING to NOTSET so it
-    # doesn't filter out any log messages due to level
-    logging.getLogger().setLevel(logging.NOTSET)
-
     if args.email:
         args.email = args.email.split(',')
 
-    do_main(args)
+    state = CalmState()
+    state.subject = 'calm%s: cygwin package upload report from %s' % (' [dry-run]' if args.dryrun else '', os.uname()[1])
+
+    if args.daemon:
+        do_daemon(args, state)
+    else:
+        logging_setup(args)
+        do_main(args, state)
 
 
 #
diff --git a/test/test_calm.py b/test/test_calm.py
index 7ca99b0..30ada6f 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -254,6 +254,8 @@ class CalmTest(unittest.TestCase):
         setattr(args, 'setup_version', '3.1415')
         setattr(args, 'stale', True)
 
+        state = calm.calm.CalmState()
+
         shutil.copytree('testdata/relarea', getattr(args, 'rel_area'))
         shutil.copytree('testdata/homes', getattr(args, 'homedir'))
 
@@ -269,7 +271,7 @@ class CalmTest(unittest.TestCase):
         for (f, t) in ready_fns:
             os.system('touch %s "%s"' % (t, f))
 
-        packages = calm.calm.process(args)
+        packages = calm.calm.process(args, state)
         self.assertTrue(packages)
 
         pkg2html.update_package_listings(args, packages['x86'], 'x86')


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]