This is the mail archive of the cygwin-apps-cvs mailing list for the cygwin-apps project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20160705-95-g63037d2




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=63037d2cfeb1caf0929d30fd170927dcb1d0a84e

commit 63037d2cfeb1caf0929d30fd170927dcb1d0a84e
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Tue Oct 3 21:48:58 2017 +0100

    Fix a problem with version sorting
    
    Non-alphanumeric separators (i.e. '.') were not ignored as per rpmvercmp.
    
    This led to incorrect sorting for e.g. 15.8b-1 vs 15.8.0.1-2

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1edd7bfc022bc1d1e16517ee3af0d6c3765e1e73

commit 1edd7bfc022bc1d1e16517ee3af0d6c3765e1e73
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Oct 2 23:10:38 2017 +0100

    Remove unused SetupVersion._warn_ambiguous_compare

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=484dafa7e4ff07fb4fb423697a96b46532ebaaf3

commit 484dafa7e4ff07fb4fb423697a96b46532ebaaf3
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Jun 8 16:02:12 2017 +0100

    Add a tool for de-duplicating a source package

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=7cb33c04ecb1ddd0aee793864a95d89d5221fa49

commit 7cb33c04ecb1ddd0aee793864a95d89d5221fa49
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Sat Apr 8 19:52:27 2017 +0100

    Generate package listing page for source-only packages
    
    Note that we may not have a good sdesc for these packages

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=d51e7d75d48e1940840a000b50473175e0748e35

commit d51e7d75d48e1940840a000b50473175e0748e35
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Fri Jan 20 15:43:50 2017 +0000

    Make source packages a thing
    
    Look for source packages in src/
    Write Source: lines in setup.ini to reference them
    
    Future work: could do with an indication that a package is a source package,
    rather than a heuristic based on package name?


Diff:
---
 calm/calm.py      |   30 ++++++-----
 calm/dedupsrc.py  |  156 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 calm/package.py   |   21 +++++---
 calm/pkg2html.py  |   27 ++++++---
 calm/version.py   |   38 ++-----------
 setup.py          |    1 +
 test/test_calm.py |   18 +++++-
 7 files changed, 225 insertions(+), 66 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index cfffb2f..72325f7 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -112,7 +112,7 @@ def process_relarea(args):
         if args.stale:
             stale_to_vault = remove_stale_packages(args, packages)
             if stale_to_vault:
-                for arch in common_constants.ARCHES + ['noarch']:
+                for arch in common_constants.ARCHES + ['noarch', 'src']:
                     logging.info("vaulting %d old package(s) for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch))
                     uploads.move_to_vault(args, stale_to_vault[arch])
             else:
@@ -143,7 +143,7 @@ def process_uploads(args, state):
                 # for each arch and noarch
                 scan_result = {}
                 skip_maintainer = False
-                for arch in common_constants.ARCHES + ['noarch']:
+                for arch in common_constants.ARCHES + ['noarch', 'src']:
                     logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name))
 
                     # read uploads
@@ -176,7 +176,7 @@ def process_uploads(args, state):
                     logging.debug("merging %s package set with uploads from maintainer %s" % (arch, name))
 
                     # merge package sets
-                    merged_packages[arch] = package.merge(state.packages[arch], scan_result[arch].packages, scan_result['noarch'].packages)
+                    merged_packages[arch] = package.merge(state.packages[arch], scan_result[arch].packages, scan_result['noarch'].packages, scan_result['src'].packages)
                     if not merged_packages[arch]:
                         logging.error("error while merging uploaded %s packages for %s" % (arch, name))
                         valid = False
@@ -211,7 +211,7 @@ def process_uploads(args, state):
 
                 # check for conflicting movelists
                 conflicts = False
-                for arch in common_constants.ARCHES + ['noarch']:
+                for arch in common_constants.ARCHES + ['noarch', 'src']:
                     conflicts = conflicts or report_movelist_conflicts(scan_result[arch].to_relarea, scan_result[arch].to_vault, "manually")
                     if args.stale:
                         conflicts = conflicts or report_movelist_conflicts(scan_result[arch].to_relarea, stale_to_vault[arch], "automatically")
@@ -223,7 +223,7 @@ def process_uploads(args, state):
                     continue
 
                 # for each arch and noarch
-                for arch in common_constants.ARCHES + ['noarch']:
+                for arch in common_constants.ARCHES + ['noarch', 'src']:
                     logging.debug("moving %s packages for maintainer %s" % (arch, name))
 
                     # process the move lists
@@ -237,7 +237,7 @@ def process_uploads(args, state):
 
                 # for each arch
                 if args.stale:
-                    for arch in common_constants.ARCHES + ['noarch']:
+                    for arch in common_constants.ARCHES + ['noarch', 'src']:
                         if stale_to_vault[arch]:
                             logging.info("vaulting %d old package(s) for arch %s, which are no longer accessible by installer" % (len(stale_to_vault[arch]), arch))
                             uploads.move_to_vault(args, stale_to_vault[arch])
@@ -246,7 +246,7 @@ def process_uploads(args, state):
                 for arch in common_constants.ARCHES:
                     # use merged package list
                     state.packages[arch] = merged_packages[arch]
-                    logging.debug("added %d + %d packages from maintainer %s" % (len(scan_result[arch].packages), len(scan_result['noarch'].packages), name))
+                    logging.debug("added %d (%s) + %d (noarch) + %d (src) packages from maintainer %s" % (len(scan_result[arch].packages), arch, len(scan_result['noarch'].packages), len(scan_result['src'].packages), name))
 
         # record updated reminder times for maintainers
         maintainers.Maintainer.update_reminder_times(mlist)
@@ -280,6 +280,7 @@ def process(args, state):
 def remove_stale_packages(args, packages):
     to_vault = {}
     to_vault['noarch'] = defaultdict(list)
+    to_vault['src'] = defaultdict(list)
 
     for arch in common_constants.ARCHES:
         logging.debug("checking for stale packages for arch %s" % (arch))
@@ -308,16 +309,17 @@ def remove_stale_packages(args, packages):
     if error:
         return None
 
-    # since noarch packages are included in the package set for both arch, we
-    # will build (hopefully) identical move lists for those packages for each
-    # arch.
+    # since noarch and src packages are included in the package set for both
+    # arch, we will build (hopefully) identical move lists for those packages
+    # for each arch.
     #
     # de-duplicate these package moves, as rather awkward workaround for that
     for path in list(to_vault[common_constants.ARCHES[0]]):
-        if path.startswith('noarch'):
-            to_vault['noarch'][path] = to_vault[common_constants.ARCHES[0]][path]
-            for arch in common_constants.ARCHES:
-                del to_vault[arch][path]
+        for prefix in ['noarch', 'src']:
+            if path.startswith(prefix):
+                to_vault[prefix][path] = to_vault[common_constants.ARCHES[0]][path]
+                for arch in common_constants.ARCHES:
+                    del to_vault[arch][path]
 
     return to_vault
 
diff --git a/calm/dedupsrc.py b/calm/dedupsrc.py
new file mode 100755
index 0000000..f531a2e
--- /dev/null
+++ b/calm/dedupsrc.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2017 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+#
+# Move a given source archive to src/ (assuming it is indentical in x86/ and
+# x86_64/) and adjust hints appropriately.
+#
+
+import argparse
+import copy
+import os
+import re
+import sys
+
+from . import common_constants
+from . import hint
+
+#
+#
+#
+
+
+def hint_file_write(fn, hints):
+    with open(fn, 'w') as f:
+        for k, v in hints.items():
+            print("%s: %s" % (k, v), file=f)
+
+#
+#
+#
+
+
+def dedup(archive, relarea):
+    # split path and filename
+    (path, filename) = os.path.split(archive)
+
+    # parse tarfile name
+    match = re.match(r'^(.+?)-(\d.*)-src\.tar\.(bz2|gz|lzma|xz)$', filename)
+
+    if not match:
+        print('tarfile name %s does not meet expectations' % (filename))
+        sys.exit(1)
+
+    p = match.group(1)
+    vr = match.group(2)
+    ext = match.group(3)
+
+    # compute filenames
+    to_filename = p + '-src-' + vr + '.tar.' + ext
+    hint_filename = p + '-' + vr + '.hint'
+    to_hint_filename = p + '-src-' + vr + '.hint'
+
+    # read hints for both arches
+    hints = {}
+    for arch in ['x86', 'x86_64']:
+        hint_pathname = os.path.join(relarea, arch, path, hint_filename)
+
+        if not os.path.exists(hint_pathname):
+            print('%s not found' % (hint_pathname))
+            return 1
+
+        hints[arch] = hint.hint_file_parse(hint_pathname, hint.pvr)
+
+    if hints['x86'] != hints['x86_64']:
+        print('hints for %s-%s differ between arches' % (p, vr))
+        return 1
+
+    # ensure target directory exists
+    try:
+        os.makedirs(os.path.join(relarea, 'src', path, p + '-src'))
+    except FileExistsError:
+        pass
+
+    # move the src files to src/
+    for arch in ['x86', 'x86_64']:
+        print('%s -> %s' % (os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename)))
+        os.rename(os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename))
+
+    # write .hint file for new -src package
+    src_hints = copy.copy(hints['x86'])
+
+    if 'source' not in src_hints['sdesc']:
+        sdesc = re.sub(r'"(.*)"', r'\1', src_hints['sdesc'])
+        sdesc += ' (source code)'
+        src_hints['sdesc'] = '"' + sdesc + '"'
+
+    if 'requires' in src_hints:
+        del src_hints['requires']
+
+    if 'external-source' in src_hints:
+        del src_hints['external-source']
+
+    to_hint_pathname = os.path.join(relarea, 'src', path, p + '-src', to_hint_filename)
+    print('writing %s' % (to_hint_pathname))
+    hint_file_write(to_hint_pathname, src_hints)
+
+    # adjust external-source in .hint for all subpackages
+    for arch in ['x86', 'x86_64']:
+        for (dirpath, subdirs, files) in os.walk(os.path.join(relarea, arch, path)):
+            subpkg = os.path.basename(dirpath)
+            filename = subpkg + '-' + vr + '.hint'
+            if filename in files:
+                hint_pathname = os.path.join(dirpath, filename)
+                hints = hint.hint_file_parse(hint_pathname, hint.pvr)
+                if ('skip' in hints):
+                    # p was source only, so no package remains
+                    print('removing %s' % (hint_pathname))
+                    os.remove(hint_pathname)
+                elif ('external-source' not in hints) or (hints['external-source'] == p):
+                    hints['external-source'] = p + '-src'
+                    print('writing %s' % (hint_pathname))
+                    hint_file_write(hint_pathname, hints)
+
+    return 0
+
+#
+#
+#
+
+
+def main():
+    relarea_default = common_constants.FTP
+
+    parser = argparse.ArgumentParser(description='Source package deduplicator')
+    parser.add_argument('archive', metavar='ARCHIVE', nargs=1, help="source archive to deduplicate")
+    parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area')
+    (args) = parser.parse_args()
+
+    return dedup(args.archive[0], args.rel_area)
+
+#
+#
+#
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/calm/package.py b/calm/package.py
index 07ef02b..23954dd 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -82,8 +82,8 @@ class Tar(object):
 def read_packages(rel_area, arch):
     packages = defaultdict(Package)
 
-    # both noarch/ and <arch>/ directories are considered
-    for root in ['noarch', arch]:
+    # <arch>/ noarch/ and src/ directories are considered
+    for root in ['noarch', 'src', arch]:
         releasedir = os.path.join(rel_area, root)
         logging.debug('reading packages from %s' % releasedir)
 
@@ -423,7 +423,7 @@ def validate_packages(args, packages):
 
         for (t, tar) in packages[p].tars.items():
             # categorize each tarfile as either 'source' or 'install'
-            if re.search(r'-src\.tar', t):
+            if re.search(r'-src.*\.tar', t):
                 category = 'source'
             else:
                 category = 'install'
@@ -736,7 +736,7 @@ def write_setup_ini(args, packages, arch):
         # for each package
         for p in sorted(packages.keys(), key=sort_key):
             # do nothing if 'skip'
-            if packages[p].skip:
+            if packages[p].skip and not p.endswith('-src'):
                 continue
 
             # write package data
@@ -784,11 +784,16 @@ def write_setup_ini(args, packages, arch):
                     # if that doesn't exist, follow external-source
                     elif 'external-source' in packages[p].version_hints[version]:
                         s = packages[p].version_hints[version]['external-source']
-                        if 'source' in packages[s].vermap[version]:
-                            t = packages[s].vermap[version]['source']
-                            tar_line('source', packages[s], t, f)
+                        # external-source points to a real source package (-src)
+                        if s.endswith('-src'):
+                            print("Source: %s" % (s), file=f)
+                        # external-source points to a source file in another package
                         else:
-                            logging.warning("package '%s' version '%s' has no source in external-source '%s'" % (p, version, s))
+                            if 'source' in packages[s].vermap[version]:
+                                t = packages[s].vermap[version]['source']
+                                tar_line('source', packages[s], t, f)
+                            else:
+                                logging.warning("package '%s' version '%s' has no source in external-source '%s'" % (p, version, s))
 
 
 # helper function to output details for a particular tar file
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index 9b23a94..83308b6 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -55,6 +55,22 @@ from . import package
 
 
 #
+# get sdesc for a package
+#
+# some source-only packages don't have an sdesc, since they consist of just
+# 'skip':', in which case we try to make a reasonable one
+#
+
+def desc(packages, p, bv):
+    if 'sdesc' in packages[p].version_hints[bv]:
+        header = packages[p].version_hints[bv]['sdesc']
+    else:
+        header = p
+
+    return header.replace('"', '')
+
+
+#
 #
 #
 
@@ -88,10 +104,6 @@ def update_package_listings(args, packages, arch):
 
     for p in packages:
 
-        # do nothing for packages marked 'skip'
-        if packages[p].skip:
-            continue
-
         dir = os.path.join(base, p)
         if not args.dryrun:
             try:
@@ -140,7 +152,7 @@ def update_package_listings(args, packages, arch):
                 if not args.dryrun:
                     with open(listing, 'w') as f:
                         bv = packages[p].best_version
-                        header = p + ": " + packages[p].version_hints[bv]['sdesc'].replace('"', '')
+                        header = p + ": " + desc(packages, p, bv)
 
                         if fver.endswith('-src'):
                             header = header + " (source code)"
@@ -210,12 +222,9 @@ def update_package_listings(args, packages, arch):
                                      <table class="pkglist">''') % (arch, arch), file=index)
 
             for p in sorted(packages.keys(), key=package.sort_key):
-                # don't write anything if 'skip'
-                if packages[p].skip:
-                    continue
 
                 bv = packages[p].best_version
-                header = packages[p].version_hints[bv]['sdesc'].replace('"', '')
+                header = desc(packages, p, bv)
 
                 print('<tr><td><a href="' + arch + '/' + p + '">' + p + '</a></td><td>' + html.escape(header, quote=False) + '</td></tr>', file=index)
 
diff --git a/calm/version.py b/calm/version.py
index 027372f..f00ddbb 100644
--- a/calm/version.py
+++ b/calm/version.py
@@ -22,7 +22,6 @@
 #
 
 import itertools
-import logging
 import re
 
 
@@ -45,10 +44,14 @@ class SetupVersion:
         # split version into [V, R], on the last '-', if any
         split = list(itertools.chain(version_string.rsplit('-', 1), ['']))[:2]
 
-        # then split each part into numeric and non-numeric sequences.
+        # then split each part into numeric and alphabetic sequences
+        # non-alphanumeric separators are discarded
         # numeric sequences have leading zeroes discarded
         for j, i in enumerate(['V', 'R']):
-            setattr(self, '_' + i, [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in re.finditer(r'(\d+|\D+)', split[j])])
+            sequences = re.finditer(r'(\d+|[a-zA-Z]+|[^a-zA-Z\d]+)', split[j])
+            sequences = [m for m in sequences if not re.match(r'[^a-zA-Z\d]+', m.group(1))]
+            sequences = [re.sub(r'^0+(\d)', r'\1', m.group(1), 1) for m in sequences]
+            setattr(self, '_' + i, sequences)
 
     def __str__(self):
         return '%s (V=%s R=%s)' % (self._version_string, str(self._V), str(self._R))
@@ -63,8 +66,6 @@ class SetupVersion:
         return self.__cmp__(other) == -1
 
     def __cmp__(self, other):
-        # warn about ill-specified comparisons
-        # SetupVersion._warn_ambiguous_compare(self, other)
 
         # compare V
         c = SetupVersion._compare(self._V, other._V)
@@ -101,30 +102,3 @@ class SetupVersion:
         # if equal length, all components have matched, so equal
         # otherwise, the version with a suffix remaining is greater
         return cmp(len(a), len(b))
-
-    # warn if the comparison of these versions is historically under-specified
-    @staticmethod
-    def _warn_ambiguous_compare(a, b):
-        def classify(s):
-            if len(s) == 0:
-                return 'e'
-            elif s[0].isdigit():
-                return 'n'
-            elif s[0] in '.-_':
-                return s[0]
-            elif s[0].isalpha():
-                return 'a'
-            return 'o'
-
-        def is_ambiguous(a, b):
-            ambiguous = False
-
-            for i in range(0, min(len(a), len(b))):
-                if classify(a[i]) != classify(b[i]):
-                    ambiguous = True
-                    break
-
-            return ambiguous
-
-        if is_ambiguous(a._V, b._V) or is_ambiguous(a._R, b._R):
-            logging.warning("ordering of versions '%s' and '%s' may not be what you expect" % (a._version_string, b._version_string))
diff --git a/setup.py b/setup.py
index 2941fa4..cdc8121 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,7 @@ setup(
             'calm = calm.calm:main',
             'mksetupini = calm.mksetupini:main',
             'calm-mkgitoliteconf = calm.mkgitoliteconf:main',
+            'dedup-source = calm.dedupsrc:main',
         ],
     },
     url='https://cygwin.com/git/?p=cygwin-apps/calm.git',
diff --git a/test/test_calm.py b/test/test_calm.py
index 30ada6f..d1cf3af 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -153,13 +153,25 @@ class CalmTest(unittest.TestCase):
             ["1.3.30c-2", "1.3.30c-10", -1],
             ["2.24.51-1", "2.25-1", -1],
             ["2.1.5+20120813+gitdcbe778-1", "2.1.5-3", 1],
-            ["3.4.1-1", "3.4b1-1", -1],
+            ["3.4.1-1", "3.4b1-1", 1],
             ["041206-1", "200090325-1", -1],
             ["0.6.2+git20130413-2", "0.6.2-1", 1],
             ["2.6.0+bzr6602-1", "2.6.0-2", 1],
-            ["2.6.0-2", "2.6b2-1", -1],
-            ["2.6.0+bzr6602-1", "2.6b2-1", -1],
+            ["2.6.0-2", "2.6b2-1", 1],
+            ["2.6.0+bzr6602-1", "2.6b2-1", 1],
             ["0.6.7+20150214+git3a710f9-1", "0.6.7-1", 1],
+            ["15.8b-1", "15.8.0.1-2", -1],
+            ["1.2rc1-1","1.2.0-2", -1],
+            # examples from https://fedoraproject.org/wiki/Archive:Tools/RPM/VersionComparison
+            ["1.0010", "1.9", 1],
+            ["1.05", "1.5", 0],
+            ["1.0", "1", 1],
+            ["2.50", "2.5", 1],
+            ["fc4", "fc.4", 0],
+            ["FC5", "fc4", -1],
+            ["2a", "2.0", -1],
+            ["1.0", "1.fc4", 1],
+            ["3.0.0_fc", "3.0.0.fc", 0],
         ]
 
         for d in test_data:


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]