This is the mail archive of the
cygwin-apps-cvs
mailing list for the cygwin-apps project.
[calm - Cygwin server-side packaging maintenance script] branch master, updated. 0782de43170a1a81f0cb51ea83e60a0b53b28c6b
- From: jturney at sourceware dot org
- To: cygwin-apps-cvs at sourceware dot org
- Date: 18 Mar 2016 17:13:30 -0000
- Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 0782de43170a1a81f0cb51ea83e60a0b53b28c6b
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=0782de43170a1a81f0cb51ea83e60a0b53b28c6b
commit 0782de43170a1a81f0cb51ea83e60a0b53b28c6b
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Fri Mar 18 16:47:40 2016 +0000
Write detailed calm log output to a file
Also write detailed calm log output to a file
Rotate this log file on each run
Send normal output to stdout, rather than stderr
Downgrade a verycommon message from update_package_listings() to DEBUG
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=615aa0700618bd7d70f11854294e9823b7d420eb
commit 615aa0700618bd7d70f11854294e9823b7d420eb
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Thu Mar 17 23:46:04 2016 +0000
Remove deletion cookies after files are vaulted
In uploads.scan(), rename 'ready' to 'remove'
Add 'remove_success' (again) to hold the list of deletion cookies
Remove those files after the vaulting has occured
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=5ad04e2302bff312cb5d72b520be1fad7448fafb
commit 5ad04e2302bff312cb5d72b520be1fad7448fafb
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Thu Mar 17 23:46:21 2016 +0000
Tolerate requests to delete files which don't exist
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=14871cd9ee4bdbc4777adc80227bd530690ac7b1
commit 14871cd9ee4bdbc4777adc80227bd530690ac7b1
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Thu Mar 17 23:09:52 2016 +0000
Sort requires: before showing differences
This is a cosmetic improvement to reporting
Diff:
---
calm | 25 ++++++++++++++++++++++---
package.py | 9 +++++++--
pkg2html.py | 2 +-
tests.py | 3 ++-
upload-scan | 3 ++-
uploads.py | 23 ++++++++++++++---------
6 files changed, 48 insertions(+), 17 deletions(-)
diff --git a/calm b/calm
index fa095bd..92fbdcb 100755
--- a/calm
+++ b/calm
@@ -91,7 +91,7 @@ def main(args):
# also send a mail to each maintainer about their packages
with mail_logs(args.email, toaddrs=m.email, subject='calm messages for %s [%s]' % (name, details)) as maint_email:
- (error, mpackages, to_relarea, to_vault, remove_always) = uploads.scan(m, all_packages, args)
+ (error, mpackages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, all_packages, args)
uploads.remove(args, remove_always)
@@ -117,6 +117,7 @@ def main(args):
if package.validate_packages(args, merged_packages):
# process the move list
uploads.move_to_vault(args, to_vault)
+ uploads.remove(args, remove_success)
uploads.move_to_relarea(m, args, to_relarea)
# use merged package list
packages = merged_packages
@@ -163,10 +164,28 @@ if __name__ == "__main__":
parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output')
(args) = parser.parse_args()
+ # set up logging to a file
+ try:
+ os.makedirs('/var/log/cygwin/', exist_ok=True)
+ except FileExistsError:
+ pass
+ rfh = logging.handlers.RotatingFileHandler('/var/log/cygwin/calm.log', backupCount=24)
+ rfh.doRollover() # force a rotate on every run
+ rfh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s'))
+ rfh.setLevel(logging.INFO)
+ logging.getLogger().addHandler(rfh)
+
+ # setup logging to stdout, of WARNING messages or higher (INFO if verbose)
+ ch = logging.StreamHandler(sys.stdout)
+ ch.setFormatter(logging.Formatter(os.path.basename(sys.argv[0])+': %(message)s'))
if args.verbose:
- logging.getLogger().setLevel(logging.INFO)
+ ch.setLevel(logging.INFO)
+ else:
+ ch.setLevel(logging.WARNING)
+ logging.getLogger().addHandler(ch)
- logging.basicConfig(format=os.path.basename(sys.argv[0])+': %(message)s')
+ # change root logger level from the default of WARNING
+ logging.getLogger().setLevel(logging.INFO)
if args.email:
args.email = args.email.split(',')
diff --git a/package.py b/package.py
index 27fae03..1213a67 100755
--- a/package.py
+++ b/package.py
@@ -592,15 +592,20 @@ def merge(a, b):
else:
c[p].tars[t] = b[p].tars[t]
- # use hints from b, but warn that they have changed
+ # use hints from b, but warn if they have changed
if a[p].hints != b[p].hints:
c[p].hints = b[p].hints
+ # sort requires: as differences in ordering are uninteresting
+ for hints in [a[p].hints, b[p].hints]:
+ if 'requires' in hints:
+ hints['requires'] = ' '.join(sorted(hints['requires'].split()))
+
diff = '\n'.join(difflib.ndiff(
pprint.pformat(a[p].hints).splitlines(),
pprint.pformat(b[p].hints).splitlines()))
- logging.warning("package '%s' hints changed\n%s\n" % (p, diff))
+ logging.warning("package '%s' hints changed\n%s" % (p, diff))
return c
diff --git a/pkg2html.py b/pkg2html.py
index 549ac8c..7f593c4 100755
--- a/pkg2html.py
+++ b/pkg2html.py
@@ -170,7 +170,7 @@ def update_package_listings(args, packages):
</pre></tt>
</html>'''), file=f)
else:
- logging.info('Not writing %s, already exists' % html)
+ logging.debug('Not writing %s, already exists' % html)
# this file should exist, so remove from the toremove list
if html in toremove:
diff --git a/tests.py b/tests.py
index 6453337..5b97297 100755
--- a/tests.py
+++ b/tests.py
@@ -184,11 +184,12 @@ class TestMain(unittest.TestCase):
for (f, t) in ready_fns:
os.system('touch %s "%s"' % (t, f))
- (error, packages, to_relarea, to_vault, remove_always) = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args)
+ (error, packages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args)
self.assertEqual(error, False)
compare_with_expected_file(self, 'testdata/uploads', to_relarea, 'move')
self.assertCountEqual(remove_always, [f for (f, t) in ready_fns])
+ self.assertEqual(remove_success, [])
compare_with_expected_file(self, 'testdata/uploads', packages, 'pkglist')
def test_package_set(self):
diff --git a/upload-scan b/upload-scan
index 033574e..08ab677 100755
--- a/upload-scan
+++ b/upload-scan
@@ -64,7 +64,7 @@ def main(args):
with mail_logs(args.email, toaddrs=(args.email or []) + m.email, subject='upset messages') as maint_email:
# search for and validate uploaded packages
- (error, packages, to_relarea, to_vault, remove_always) = uploads.scan(m, all_packages, args)
+ (error, packages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, all_packages, args)
# always remove all the !ready files
uploads.remove(args, remove_always)
@@ -72,6 +72,7 @@ def main(args):
# but only move something if there were no errors
if not error:
uploads.move_to_vault(args, to_vault)
+ uploads.remove(args, remove_success)
uploads.move_to_relarea(m, args, to_relarea)
diff --git a/uploads.py b/uploads.py
index af784c5..33ae511 100644
--- a/uploads.py
+++ b/uploads.py
@@ -45,7 +45,8 @@ def scan(m, all_packages, args):
packages = defaultdict(package.Package)
move = defaultdict(list)
vault = defaultdict(list)
- readys = []
+ remove = []
+ remove_success = []
error = False
mtimes = [('', 0)]
@@ -57,7 +58,7 @@ def scan(m, all_packages, args):
mtime = os.path.getmtime(ready)
mtimes.append(('', mtime))
logging.info('processing files with mtime older than %d' % (mtime))
- readys.append(ready)
+ remove.append(ready)
# scan package directories
for (dirpath, subdirs, files) in os.walk(os.path.join(basedir, 'release')):
@@ -74,7 +75,7 @@ def scan(m, all_packages, args):
ready = os.path.join(dirpath, '!ready')
mtime = os.path.getmtime(ready)
mtimes.append((relpath + '/', mtime))
- readys.append(ready)
+ remove.append(ready)
files.remove('!ready')
logging.info("processing files below '%s' with mtime older than %d" % (relpath, mtime))
else:
@@ -124,6 +125,7 @@ def scan(m, all_packages, args):
if f.startswith('-'):
vault[relpath].append(f[1:])
files.remove(f)
+ remove_success.append(fn)
else:
dest = os.path.join(releasedir, relpath, f)
if os.path.isfile(dest):
@@ -150,15 +152,15 @@ def scan(m, all_packages, args):
if package.read_package(packages, basedir, dirpath, files, strict=True):
error = True
- return (error, packages, move, vault, readys)
+ return (error, packages, move, vault, remove, remove_success)
#
#
#
-def remove(args, readys):
- for f in readys:
+def remove(args, remove):
+ for f in remove:
logging.info("rm %s", f)
if not args.dryrun:
os.unlink(f)
@@ -177,9 +179,12 @@ def move(args, movelist, fromdir, todir):
except FileExistsError:
pass
for f in movelist[p]:
- logging.warning("move %s to %s" % (os.path.join(fromdir, p, f), os.path.join(todir, p, f)))
- if not args.dryrun:
- os.rename(os.path.join(fromdir, p, f), os.path.join(todir, p, f))
+ if os.path.exists(os.path.join(fromdir, p, f)):
+ logging.warning("move %s to %s" % (os.path.join(fromdir, p, f), os.path.join(todir, p, f)))
+ if not args.dryrun:
+ os.rename(os.path.join(fromdir, p, f), os.path.join(todir, p, f))
+ else:
+ logging.error("%s can't be moved as it doesn't exist" % (f))
def move_to_relarea(m, args, movelist):