This is the mail archive of the
cygwin-apps-cvs
mailing list for the cygwin-apps project.
[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20180131-17-g1add34c
- From: jturney at sourceware dot org
- To: cygwin-apps-cvs at sourceware dot org
- Date: 14 May 2018 20:55:32 -0000
- Subject: [calm - Cygwin server-side packaging maintenance script] branch master, updated. 20180131-17-g1add34c
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1add34cdb5d4713f808fa9e11805ec8d5a3b7f75
commit 1add34cdb5d4713f808fa9e11805ec8d5a3b7f75
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 14 19:46:29 2018 +0100
Add support for 'missing-curr' token in disabled-check: key in pvr.hint
in pvr.hint, disable-check can contain:
missing-curr:
check that package has a current version
Move identification of best-version earlier to support this.
Also fix looking for empty-obsolete in disable-check.
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=360ce8ab8fad46abe3611abf207c643595ca5fd8
commit 360ce8ab8fad46abe3611abf207c643595ca5fd8
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 14 19:44:59 2018 +0100
Ignore test-only packages in non-unique current versions check
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=df1e7872cba3de9b6f564c585d4e423aa1452581
commit df1e7872cba3de9b6f564c585d4e423aa1452581
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 14 18:13:48 2018 +0100
Fix some escaping issues reported by latest pycodestyle
Also disable W504 (line break after binary operator). The existing code is
formatted to satisify W503 (Line break occurred before a binary operator).
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7ee154d67d14272433a7ab2dd87e2e7378e763f2
commit 7ee154d67d14272433a7ab2dd87e2e7378e763f2
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Mon May 14 17:54:03 2018 +0100
Also remove versions which have no packages left, after expiry
https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=61b82600b59f89ff0894da5357b837e502d04815
commit 61b82600b59f89ff0894da5357b837e502d04815
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date: Tue Jan 30 18:53:54 2018 +0000
Drop strict option to read_package()
We no longer need the ability to not issue warnings about problems in the
relarea, since the problems are all fixed.
Diff:
---
calm/package.py | 64 +++++++++++++++++++++++++++++------------------------
calm/uploads.py | 2 +-
pep8 | 3 +-
test/test_calm.py | 4 +-
4 files changed, 39 insertions(+), 34 deletions(-)
diff --git a/calm/package.py b/calm/package.py
index 85d5368..cbae949 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -136,7 +136,7 @@ def read_hints(p, fn, kind):
# helper function to clean up hints
-def clean_hints(p, hints, strict_lvl, warnings):
+def clean_hints(p, hints, warnings):
#
# fix some common defects in the hints
#
@@ -152,7 +152,7 @@ def clean_hints(p, hints, strict_lvl, warnings):
if colon:
package_basename = re.sub(r'^lib(.*?)(|-devel|\d*)$', r'\1', p)
if package_basename.upper().startswith(colon.group(1).upper()):
- logging.log(strict_lvl, "package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2))))
+ logging.error("package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2))))
warnings = True
return warnings
@@ -161,8 +161,7 @@ def clean_hints(p, hints, strict_lvl, warnings):
#
# read a single package
#
-def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upload=False):
- strict_lvl = logging.ERROR if strict else logging.WARNING
+def read_package(packages, basedir, dirpath, files, remove=[], upload=False):
relpath = os.path.relpath(dirpath, basedir)
warnings = False
@@ -187,7 +186,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
hints = read_hints(p, os.path.join(dirpath, 'setup.hint'), hint.setup)
if not hints:
return True
- warnings = clean_hints(p, hints, strict_lvl, warnings)
+ warnings = clean_hints(p, hints, warnings)
files.remove('setup.hint')
else:
hints = {}
@@ -260,7 +259,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
#
# P must match the package name, V can contain anything, R must
# start with a number
- match = re.match(r'^' + re.escape(p) + '-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.' + match.group(1) + '$', f)
+ match = re.match(r'^' + re.escape(p) + r'-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.' + match.group(1) + '$', f)
if not match:
logging.error("file '%s' in package '%s' doesn't follow naming convention" % (f, p))
return True
@@ -273,14 +272,14 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
# idea.
if '-' in v:
if p not in past_mistakes.hyphen_in_version:
- lvl = strict_lvl
+ lvl = logging.ERROR
warnings = True
else:
lvl = logging.INFO
logging.log(lvl, "file '%s' in package '%s' contains '-' in version" % (f, p))
if not v[0].isdigit():
- logging.log(strict_lvl, "file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p))
+ logging.error("file '%s' in package '%s' has a version which doesn't start with a digit" % (f, p))
warnings = True
# if not there already, add to version-release list
@@ -313,7 +312,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
pvr_hint = read_hints(p, os.path.join(dirpath, hint_fn), hint.pvr)
if not pvr_hint:
return True
- warnings = clean_hints(p, pvr_hint, strict_lvl, warnings)
+ warnings = clean_hints(p, pvr_hint, warnings)
files.remove(hint_fn)
elif legacy:
# otherwise, use setup.hint
@@ -344,7 +343,7 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
# warn about unexpected files, including tarfiles which don't match the
# package name
if files:
- logging.log(strict_lvl, "unexpected files in %s: %s" % (p, ', '.join(files)))
+ logging.error("unexpected files in %s: %s" % (p, ', '.join(files)))
warnings = True
if not upload and legacy and not legacy_used:
@@ -365,13 +364,10 @@ def read_package(packages, basedir, dirpath, files, strict=False, remove=[], upl
files.remove(s)
if len(files) > 0:
- logging.log(strict_lvl, "no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
+ logging.error("no .hint files in %s but has files: %s" % (dirpath, ', '.join(files)))
warnings = True
- if strict:
- return warnings
-
- return False
+ return warnings
#
@@ -602,12 +598,25 @@ def validate_packages(args, packages):
packages[p].version_hints[v][l] = ''
break
+ # identify a 'best' version to take certain information from: this is
+ # the curr version, if we have one, otherwise, the highest version.
+ if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].vermap):
+ packages[p].best_version = packages[p].stability['curr']
+ elif len(packages[p].vermap):
+ packages[p].best_version = sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True)[0]
+ else:
+ logging.error("package '%s' doesn't have any versions" % (p))
+ packages[p].best_version = None
+ error = True
+
# the package must have some versions
if not packages[p].stability:
logging.error("no versions at any stability level for package '%s'" % (p))
error = True
# it's also probably a really good idea if a curr version exists
- elif 'curr' not in packages[p].stability and 'missing-curr' not in getattr(args, 'disable_check', []):
+ elif (('curr' not in packages[p].stability) and
+ ('missing-curr' not in packages[p].version_hints[packages[p].best_version].get('disable-check', '')) and
+ ('missing-curr' not in getattr(args, 'disable_check', []))):
logging.warning("package '%s' doesn't have a curr version" % (p))
# error if the curr: version isn't the most recent non-test: version
@@ -636,17 +645,6 @@ def validate_packages(args, packages):
break
- # identify a 'best' version to take certain information from: this is
- # the curr version, if we have one, otherwise, the highest version.
- if ('curr' in packages[p].stability) and (packages[p].stability['curr'] in packages[p].vermap):
- packages[p].best_version = packages[p].stability['curr']
- elif len(packages[p].vermap):
- packages[p].best_version = sorted(packages[p].vermap.keys(), key=lambda v: SetupVersion(v), reverse=True)[0]
- else:
- logging.error("package '%s' doesn't have any versions" % (p))
- packages[p].best_version = None
- error = True
-
if 'replace-versions' in packages[p].override_hints:
for rv in packages[p].override_hints['replace-versions'].split():
# warn if replace-versions lists a version which is less than
@@ -670,7 +668,7 @@ def validate_packages(args, packages):
if 'install' in packages[p].vermap[vr]:
if packages[p].tar(vr, 'install').is_empty:
if ((p in past_mistakes.empty_but_not_obsolete) or
- ('empty-obsolete' in packages[p].version_hints.get('disable-check', ''))):
+ ('empty-obsolete' in packages[p].version_hints[vr].get('disable-check', ''))):
lvl = logging.DEBUG
else:
lvl = logging.ERROR
@@ -760,6 +758,11 @@ def validate_packages(args, packages):
if re.match(r'^lib.*\d', install_p):
continue
+ # ignore packages which don't have a current version (i.e. are test
+ # only)
+ if 'curr' not in packages[install_p].stability:
+ continue
+
# ignore specific packages we disable this check for
if ((install_p in past_mistakes.nonunique_versions) or
('unique-version' in packages[install_p].version_hints[packages[install_p].best_version].get('disable-check', ''))):
@@ -1079,9 +1082,12 @@ def delete(packages, path, fn):
for t in packages[p].tars[vr]:
if t == fn:
del packages[p].tars[vr][t]
- # XXX: should also remove from vermap
break
+ # if no packages remain for this vr, also remove from vermap
+ if not packages[p].tars[vr]:
+ packages[p].vermap.pop(vr, None)
+
for h in packages[p].hint_files:
if packages[p].hint_files[h] == fn:
del packages[p].hint_files[h]
diff --git a/calm/uploads.py b/calm/uploads.py
index 679fb9a..79a9e7f 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -247,7 +247,7 @@ def scan(m, all_packages, arch, args):
# read and validate package
if files:
# strict means we consider warnings as fatal for upload
- if package.read_package(packages, m.homedir(), dirpath, files, strict=True, remove=removed_files, upload=True):
+ if package.read_package(packages, m.homedir(), dirpath, files, remove=removed_files, upload=True):
error = True
# always consider timestamp as checked during a dry-run, so it is never
diff --git a/pep8 b/pep8
index db9c9c7..6230d52 100755
--- a/pep8
+++ b/pep8
@@ -1,4 +1,3 @@
#!/bin/sh
set -e
-grep -s -l '^#!/usr/bin/env python' * calm/* test/* | xargs python3 -m pycodestyle --count --show-source --max-line-length=240 --ignore=E129,E741
-flake8 --ignore E129,E501,E741,F841
+grep -s -l '^#!/usr/bin/env python' * calm/* test/* | xargs python3 -m pycodestyle --count --show-source --max-line-length=240 --ignore=E129,E741,W504
diff --git a/test/test_calm.py b/test/test_calm.py
index f514b7d..322d776 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -415,11 +415,11 @@ class CalmTest(unittest.TestCase):
os.chdir(os.path.dirname(os.path.abspath(__file__)))
# ensure sha512.sum files exist
- os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \;")
+ os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \\;")
# should remove a sha512.sum file so that we test functioning when it's absent
os.unlink('testdata/relarea/x86/release/arc/sha512.sum')
# remove !ready files
- os.system("find testdata/homes -name !ready -exec rm {} \;")
+ os.system("find testdata/homes -name !ready -exec rm {} \\;")
# fix up package timestamps so highest version is also latest
# (git doesn't store timestamps, so they will all be dated the time of checkout)