[calm - Cygwin server-side packaging maintenance script] branch master, updated. 0782de43170a1a81f0cb51ea83e60a0b53b28c6b

jturney@sourceware.org jturney@sourceware.org
Fri Mar 18 17:13:00 GMT 2016




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0782de43170a1a81f0cb51ea83e60a0b53b28c6b

commit 0782de43170a1a81f0cb51ea83e60a0b53b28c6b
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Fri Mar 18 16:47:40 2016 +0000

    Write detailed calm log output to a file
    
    Also write detailed calm log output to a file
    Rotate this log file on each run
    Send normal output to stdout, rather than stderr
    Downgrade a verycommon message from update_package_listings() to DEBUG

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=615aa0700618bd7d70f11854294e9823b7d420eb

commit 615aa0700618bd7d70f11854294e9823b7d420eb
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Mar 17 23:46:04 2016 +0000

    Remove deletion cookies after files are vaulted
    
    In uploads.scan(), rename 'ready' to 'remove'
    Add 'remove_success' (again) to hold the list of deletion cookies
    Remove those files after the vaulting has occured

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=5ad04e2302bff312cb5d72b520be1fad7448fafb

commit 5ad04e2302bff312cb5d72b520be1fad7448fafb
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Mar 17 23:46:21 2016 +0000

    Tolerate requests to delete files which don't exist

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=14871cd9ee4bdbc4777adc80227bd530690ac7b1

commit 14871cd9ee4bdbc4777adc80227bd530690ac7b1
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Mar 17 23:09:52 2016 +0000

    Sort requires: before showing differences
    
    This is a cosmetic improvement to reporting


Diff:
---
 calm        |   25 ++++++++++++++++++++++---
 package.py  |    9 +++++++--
 pkg2html.py |    2 +-
 tests.py    |    3 ++-
 upload-scan |    3 ++-
 uploads.py  |   23 ++++++++++++++---------
 6 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/calm b/calm
index fa095bd..92fbdcb 100755
--- a/calm
+++ b/calm
@@ -91,7 +91,7 @@ def main(args):
             # also send a mail to each maintainer about their packages
             with mail_logs(args.email, toaddrs=m.email, subject='calm messages for %s [%s]' % (name, details)) as maint_email:
 
-                (error, mpackages, to_relarea, to_vault, remove_always) = uploads.scan(m, all_packages, args)
+                (error, mpackages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, all_packages, args)
 
                 uploads.remove(args, remove_always)
 
@@ -117,6 +117,7 @@ def main(args):
                     if package.validate_packages(args, merged_packages):
                         # process the move list
                         uploads.move_to_vault(args, to_vault)
+                        uploads.remove(args, remove_success)
                         uploads.move_to_relarea(m, args, to_relarea)
                         # use merged package list
                         packages = merged_packages
@@ -163,10 +164,28 @@ if __name__ == "__main__":
     parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output')
     (args) = parser.parse_args()
 
+    # set up logging to a file
+    try:
+        os.makedirs('/var/log/cygwin/', exist_ok=True)
+    except FileExistsError:
+        pass
+    rfh = logging.handlers.RotatingFileHandler('/var/log/cygwin/calm.log', backupCount=24)
+    rfh.doRollover()  # force a rotate on every run
+    rfh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s'))
+    rfh.setLevel(logging.INFO)
+    logging.getLogger().addHandler(rfh)
+
+    # setup logging to stdout, of WARNING messages or higher (INFO if verbose)
+    ch = logging.StreamHandler(sys.stdout)
+    ch.setFormatter(logging.Formatter(os.path.basename(sys.argv[0])+': %(message)s'))
     if args.verbose:
-        logging.getLogger().setLevel(logging.INFO)
+        ch.setLevel(logging.INFO)
+    else:
+        ch.setLevel(logging.WARNING)
+    logging.getLogger().addHandler(ch)
 
-    logging.basicConfig(format=os.path.basename(sys.argv[0])+': %(message)s')
+    # change root logger level from the default of WARNING
+    logging.getLogger().setLevel(logging.INFO)
 
     if args.email:
         args.email = args.email.split(',')
diff --git a/package.py b/package.py
index 27fae03..1213a67 100755
--- a/package.py
+++ b/package.py
@@ -592,15 +592,20 @@ def merge(a, b):
                     else:
                         c[p].tars[t] = b[p].tars[t]
 
-                # use hints from b, but warn that they have changed
+                # use hints from b, but warn if they have changed
                 if a[p].hints != b[p].hints:
                     c[p].hints = b[p].hints
 
+                    # sort requires: as differences in ordering are uninteresting
+                    for hints in [a[p].hints, b[p].hints]:
+                        if 'requires' in hints:
+                            hints['requires'] = ' '.join(sorted(hints['requires'].split()))
+
                     diff = '\n'.join(difflib.ndiff(
                         pprint.pformat(a[p].hints).splitlines(),
                         pprint.pformat(b[p].hints).splitlines()))
 
-                    logging.warning("package '%s' hints changed\n%s\n" % (p, diff))
+                    logging.warning("package '%s' hints changed\n%s" % (p, diff))
 
     return c
 
diff --git a/pkg2html.py b/pkg2html.py
index 549ac8c..7f593c4 100755
--- a/pkg2html.py
+++ b/pkg2html.py
@@ -170,7 +170,7 @@ def update_package_listings(args, packages):
                                                  </pre></tt>
                                                  </html>'''), file=f)
             else:
-                logging.info('Not writing %s, already exists' % html)
+                logging.debug('Not writing %s, already exists' % html)
 
             # this file should exist, so remove from the toremove list
             if html in toremove:
diff --git a/tests.py b/tests.py
index 6453337..5b97297 100755
--- a/tests.py
+++ b/tests.py
@@ -184,11 +184,12 @@ class TestMain(unittest.TestCase):
         for (f, t) in ready_fns:
             os.system('touch %s "%s"' % (t, f))
 
-        (error, packages, to_relarea, to_vault, remove_always) = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args)
+        (error, packages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args)
 
         self.assertEqual(error, False)
         compare_with_expected_file(self, 'testdata/uploads', to_relarea, 'move')
         self.assertCountEqual(remove_always, [f for (f, t) in ready_fns])
+        self.assertEqual(remove_success, [])
         compare_with_expected_file(self, 'testdata/uploads', packages, 'pkglist')
 
     def test_package_set(self):
diff --git a/upload-scan b/upload-scan
index 033574e..08ab677 100755
--- a/upload-scan
+++ b/upload-scan
@@ -64,7 +64,7 @@ def main(args):
             with mail_logs(args.email, toaddrs=(args.email or []) + m.email, subject='upset messages') as maint_email:
 
                 # search for and validate uploaded packages
-                (error, packages, to_relarea, to_vault, remove_always) = uploads.scan(m, all_packages, args)
+                (error, packages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, all_packages, args)
 
                 # always remove all the !ready files
                 uploads.remove(args, remove_always)
@@ -72,6 +72,7 @@ def main(args):
                 # but only move something if there were no errors
                 if not error:
                     uploads.move_to_vault(args, to_vault)
+                    uploads.remove(args, remove_success)
                     uploads.move_to_relarea(m, args, to_relarea)
 
 
diff --git a/uploads.py b/uploads.py
index af784c5..33ae511 100644
--- a/uploads.py
+++ b/uploads.py
@@ -45,7 +45,8 @@ def scan(m, all_packages, args):
     packages = defaultdict(package.Package)
     move = defaultdict(list)
     vault = defaultdict(list)
-    readys = []
+    remove = []
+    remove_success = []
     error = False
     mtimes = [('', 0)]
 
@@ -57,7 +58,7 @@ def scan(m, all_packages, args):
             mtime = os.path.getmtime(ready)
             mtimes.append(('', mtime))
             logging.info('processing files with mtime older than %d' % (mtime))
-            readys.append(ready)
+            remove.append(ready)
 
     # scan package directories
     for (dirpath, subdirs, files) in os.walk(os.path.join(basedir, 'release')):
@@ -74,7 +75,7 @@ def scan(m, all_packages, args):
             ready = os.path.join(dirpath, '!ready')
             mtime = os.path.getmtime(ready)
             mtimes.append((relpath + '/', mtime))
-            readys.append(ready)
+            remove.append(ready)
             files.remove('!ready')
             logging.info("processing files below '%s' with mtime older than %d" % (relpath, mtime))
         else:
@@ -124,6 +125,7 @@ def scan(m, all_packages, args):
             if f.startswith('-'):
                 vault[relpath].append(f[1:])
                 files.remove(f)
+                remove_success.append(fn)
             else:
                 dest = os.path.join(releasedir, relpath, f)
                 if os.path.isfile(dest):
@@ -150,15 +152,15 @@ def scan(m, all_packages, args):
             if package.read_package(packages, basedir, dirpath, files, strict=True):
                 error = True
 
-    return (error, packages, move, vault, readys)
+    return (error, packages, move, vault, remove, remove_success)
 
 
 #
 #
 #
 
-def remove(args, readys):
-    for f in readys:
+def remove(args, remove):
+    for f in remove:
         logging.info("rm %s", f)
         if not args.dryrun:
             os.unlink(f)
@@ -177,9 +179,12 @@ def move(args, movelist, fromdir, todir):
             except FileExistsError:
                 pass
         for f in movelist[p]:
-            logging.warning("move %s to %s" % (os.path.join(fromdir, p, f), os.path.join(todir, p, f)))
-            if not args.dryrun:
-                os.rename(os.path.join(fromdir, p, f), os.path.join(todir, p, f))
+            if os.path.exists(os.path.join(fromdir, p, f)):
+                logging.warning("move %s to %s" % (os.path.join(fromdir, p, f), os.path.join(todir, p, f)))
+                if not args.dryrun:
+                    os.rename(os.path.join(fromdir, p, f), os.path.join(todir, p, f))
+            else:
+                logging.error("%s can't be moved as it doesn't exist" % (f))
 
 
 def move_to_relarea(m, args, movelist):



More information about the Cygwin-apps-cvs mailing list