[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20180131-17-g1add34c

jturney@sourceware.org jturney@sourceware.org
Mon May 14 20:55:00 GMT 2018




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1add34cdb5d4713f808fa9e11805ec8d5a3b7f75

commit 1add34cdb5d4713f808fa9e11805ec8d5a3b7f75
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon May 14 19:46:29 2018 +0100

    Add support for 'missing-curr' token in disabled-check: key in pvr.hint
    
    in pvr.hint, disable-check can contain:
    
        missing-curr:
           check that package has a current version
    
    Move identification of best-version earlier to support this.
    
    Also fix looking for empty-obsolete in disable-check.

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=360ce8ab8fad46abe3611abf207c643595ca5fd8

commit 360ce8ab8fad46abe3611abf207c643595ca5fd8
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon May 14 19:44:59 2018 +0100

    Ignore test-only packages in non-unique current versions check

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=df1e7872cba3de9b6f564c585d4e423aa1452581

commit df1e7872cba3de9b6f564c585d4e423aa1452581
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon May 14 18:13:48 2018 +0100

    Fix some escaping issues reported by latest pycodestyle
    
    Also disable W504 (line break after binary operator).  The existing code is
    formatted to satisify W503 (Line break occurred before a binary operator).

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7ee154d67d14272433a7ab2dd87e2e7378e763f2

commit 7ee154d67d14272433a7ab2dd87e2e7378e763f2
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon May 14 17:54:03 2018 +0100

    Also remove versions which have no packages left, after expiry

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=61b82600b59f89ff0894da5357b837e502d04815

commit 61b82600b59f89ff0894da5357b837e502d04815
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Tue Jan 30 18:53:54 2018 +0000

    Drop strict option to read_package()
    
    We no longer need the ability to not issue warnings about problems in the
    relarea, since the problems are all fixed.


Diff:
---
 calm/package.py   |   64 +++++++++++++++++++++++++++++------------------------
 calm/uploads.py   |    2 +-
 pep8              |    3 +-
 test/test_calm.py |    4 +-
 4 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/calm/package.py b/calm/package.py
index 85d5368..cbae949 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -136,7 +136,7 @@ def read_hints(p, fn, kind):
 
 
 # helper function to clean up hints
-def clean_hints(p, hints, strict_lvl, warnings):
+def clean_hints(p, hints, warnings):
     #
     # fix some common defects in the hints
     #
@@ -152,7 +152,7 @@ def clean_hints(p, hints, strict_lvl, warnings):
         if colon:
             package_basename = re.sub(r'^lib(.*?)(|-devel|\d*)$', r'\1', p)
             if package_basename.upper().startswith(colon.group(1).upper()):
-                logging.log(strict_lvl, "package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2))))
+                logging.error("package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2))))
                 warnings = True
 
     return warnings
@@ -161,8 +161,7 @@ def clean_hints(p, hints, strict_lvl, warnings):
 #
 # read a single package
 #
-def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upload=False):
-    strict_lvl = logging.ERROR if strict else logging.WARNING
+def read_package(packages, basedir, dirpath, files, remove=[], upload=False):
     relpath = os.path.relpath(dirpath, basedir)
     warnings = False
 
@@ -187,7 +186,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
             hints = read_hints(p, os.path.join(dirpath, 'setup.hint'), hint.setup)
             if not hints:
                 return True
-            warnings = clean_hints(p, hints, strict_lvl, warnings)
+            warnings = clean_hints(p, hints, warnings)
             files.remove('setup.hint')
         else:
             hints = {}
@@ -260,7 +259,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
             #
             # P must match the package name, V can contain anything, R must
             # start with a number
-            match = re.match(r'^' + re.escape(p) + '-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.' + match.group(1) + '$', f)
+            match = re.match(r'^' + re.escape(p) + r'-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.' + match.group(1) + '$', f)
             if not match:
                 logging.error("file '%s' in package '%s' doesn't follow naming convention" % (f, p))
                 return True
@@ -273,14 +272,14 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
                 # idea.
                 if '-' in v:
                     if p not in past_mistakes.hyphen_in_version:
-                        lvl = strict_lvl
+                        lvl = logging.ERROR
                         warnings = True
                     else:
                         lvl = logging.INFO
                     logging.log(lvl, "file '%s' in package '%s' contains '-' in version" % (f, p))
 
                 if not v[0].isdigit():
-                    logging.log(strict_lvl, "file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p))
+                    logging.error("file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p))
                     warnings = True
 
                 # if not there already, add to version-release list
@@ -313,7 +312,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
                 pvr_hint = read_hints(p, os.path.join(dirpath, hint_fn), hint.pvr)
                 if not pvr_hint:
                     return True
-                warnings = clean_hints(p, pvr_hint, strict_lvl, warnings)
+                warnings = clean_hints(p, pvr_hint, warnings)
                 files.remove(hint_fn)
             elif legacy:
                 # otherwise, use setup.hint
@@ -344,7 +343,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
         # warn about unexpected files, including tarfiles which don't match the
         # package name
         if files:
-            logging.log(strict_lvl, "unexpected files in %s: %s" % (p, ', '.join(files)))
+            logging.error("unexpected files in %s: %s" % (p, ', '.join(files)))
             warnings = True
 
         if not upload and legacy and not legacy_used:
@@ -365,13 +364,10 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
                 files.remove(s)
 
         if len(files) > 0:
-            logging.log(strict_lvl, "no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
+            logging.error("no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
             warnings = True
 
-    if strict:
-        return warnings
-
-    return False
+    return warnings
 
 
 #
@@ -602,12 +598,25 @@ def validate_packages(args, packages):
                     packages[p].version_hints[v][l] = ''
                     break
 
+        # identify a 'best' version to take certain information from: this is
+        # the curr version, if we have one, otherwise, the highest version.
+        if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].vermap):
+            packages[p].best_version = packages[p].stability['curr']
+        elif len(packages[p].vermap):
+            packages[p].best_version = sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True)[0]
+        else:
+            logging.error("package '%s' doesn't have any versions" % (p))
+            packages[p].best_version = None
+            error = True
+
         # the package must have some versions
         if not packages[p].stability:
             logging.error("no versions at any stability level for package '%s'" % (p))
             error = True
         # it's also probably a really good idea if a curr version exists
-        elif 'curr' not in packages[p].stability and 'missing-curr' not in getattr(args, 'disable_check', []):
+        elif (('curr' not in packages[p].stability) and
+              ('missing-curr' not in packages[p].version_hints[packages[p].best_version].get('disable-check', '')) and
+              ('missing-curr' not in getattr(args, 'disable_check', []))):
             logging.warning("package '%s' doesn't have a curr version" % (p))
 
         # error if the curr: version isn't the most recent non-test: version
@@ -636,17 +645,6 @@ def validate_packages(args, packages):
 
             break
 
-        # identify a 'best' version to take certain information from: this is
-        # the curr version, if we have one, otherwise, the highest version.
-        if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].vermap):
-            packages[p].best_version = packages[p].stability['curr']
-        elif len(packages[p].vermap):
-            packages[p].best_version = sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True)[0]
-        else:
-            logging.error("package '%s' doesn't have any versions" % (p))
-            packages[p].best_version = None
-            error = True
-
         if 'replace-versions' in packages[p].override_hints:
             for rv in packages[p].override_hints['replace-versions'].split():
                 # warn if replace-versions lists a version which is less than
@@ -670,7 +668,7 @@ def validate_packages(args, packages):
                         if 'install' in packages[p].vermap[vr]:
                             if packages[p].tar(vr, 'install').is_empty:
                                 if ((p in past_mistakes.empty_but_not_obsolete) or
-                                    ('empty-obsolete' in packages[p].version_hints.get('disable-check', ''))):
+                                    ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))):
                                     lvl = logging.DEBUG
                                 else:
                                     lvl = logging.ERROR
@@ -760,6 +758,11 @@ def validate_packages(args, packages):
             if re.match(r'^lib.*\d', install_p):
                 continue
 
+            # ignore packages which don't have a current version (i.e. are test
+            # only)
+            if 'curr' not in packages[install_p].stability:
+                continue
+
             # ignore specific packages we disable this check for
             if ((install_p in past_mistakes.nonunique_versions) or
                 ('unique-version' in packages[install_p].version_hints[packages[install_p].best_version].get('disable-check', ''))):
@@ -1079,9 +1082,12 @@ def delete(packages, path, fn):
                 for t in packages[p].tars[vr]:
                     if t == fn:
                         del packages[p].tars[vr][t]
-                    # XXX: should also remove from vermap
                         break
 
+                # if no packages remain for this vr, also remove from vermap
+                if not packages[p].tars[vr]:
+                    packages[p].vermap.pop(vr, None)
+
             for h in packages[p].hint_files:
                 if packages[p].hint_files[h] == fn:
                     del packages[p].hint_files[h]
diff --git a/calm/uploads.py b/calm/uploads.py
index 679fb9a..79a9e7f 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -247,7 +247,7 @@ def scan(m, all_packages, arch, args):
         # read and validate package
         if files:
             # strict means we consider warnings as fatal for upload
-            if package.read_package(packages, m.homedir(), dirpath, files, strict=True, remove=removed_files, upload=True):
+            if package.read_package(packages, m.homedir(), dirpath, files, remove=removed_files, upload=True):
                 error = True
 
     # always consider timestamp as checked during a dry-run, so it is never
diff --git a/pep8 b/pep8
index db9c9c7..6230d52 100755
--- a/pep8
+++ b/pep8
@@ -1,4 +1,3 @@
 #!/bin/sh
 set -e
-grep -s -l '^#!/usr/bin/env python' * calm/* test/* | xargs python3 -m pycodestyle --count --show-source --max-line-length=240 --ignore=E129,E741
-flake8 --ignore E129,E501,E741,F841
+grep -s -l '^#!/usr/bin/env python' * calm/* test/* | xargs python3 -m pycodestyle --count --show-source --max-line-length=240 --ignore=E129,E741,W504
diff --git a/test/test_calm.py b/test/test_calm.py
index f514b7d..322d776 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -415,11 +415,11 @@ class CalmTest(unittest.TestCase):
         os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
         # ensure sha512.sum files exist
-        os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \;")
+        os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \\;")
         # should remove a sha512.sum file so that we test functioning when it's absent
         os.unlink('testdata/relarea/x86/release/arc/sha512.sum')
         # remove !ready files
-        os.system("find testdata/homes -name !ready -exec rm {} \;")
+        os.system("find testdata/homes -name !ready -exec rm {} \\;")
 
         # fix up package timestamps so highest version is also latest
         # (git doesn't store timestamps, so they will all be dated the time of checkout)



More information about the Cygwin-apps-cvs mailing list