This is the mail archive of the cygwin-apps-cvs mailing list for the cygwin-apps project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[calm - Cygwin server-side packaging maintenance script] branch master, updated. 2bf6a11a86d1dcb2d5229bd723e47ffb0fba3e12




https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=2bf6a11a86d1dcb2d5229bd723e47ffb0fba3e12

commit 2bf6a11a86d1dcb2d5229bd723e47ffb0fba3e12
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Tue Apr 12 11:07:52 2016 +0100

    Add a queue interface to a package validator
    
    Place a copy of each package upload into a filesystem queue, to be processed
    by something else which does checks on the package

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=996236245f9fbaf4d665cd16dc413e6cc69ede2a

commit 996236245f9fbaf4d665cd16dc413e6cc69ede2a
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Apr 11 22:06:31 2016 +0100

    In read_package(), log as an error if treated as an error due to strict
    
    Also fix typos of 'warning' for 'warnings'

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=9ee4bb2c74cd77fdf444c709e3ab8afbfeadbf44

commit 9ee4bb2c74cd77fdf444c709e3ab8afbfeadbf44
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Apr 4 14:30:16 2016 +0100

    Tolerate random files in the homes directory
    
    Tolerate and ignore random files in the homes directory
    Add appropriate test

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=ba55e84fa5b660e292b16d310376485431cc424d

commit ba55e84fa5b660e292b16d310376485431cc424d
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Apr 11 22:13:01 2016 +0100

    Extract version correctly from .lzma files
    
    ... if we had any


Diff:
---
 .travis.yml                            |    2 +-
 calm.py                                |    6 +++
 maintainers.py                         |    3 +
 package.py                             |   14 ++++---
 queue.py                               |   70 ++++++++++++++++++++++++++++++++
 testdata/process_arch/homedir.expected |    2 +-
 uploads.py                             |   23 ++++++++++
 7 files changed, 112 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 72a7ef5..e6c1776 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,7 @@ language: python
 sudo: false
 python:
  - "3.4"
-install: "pip install pep8"
+install: "pip install pep8 dirq"
 script:
  - ./pep8
  - ./tests.py
diff --git a/calm.py b/calm.py
index a383d46..38e8cec 100755
--- a/calm.py
+++ b/calm.py
@@ -59,6 +59,7 @@ import common_constants
 import maintainers
 import package
 import pkg2html
+import queue
 import setup_exe
 import uploads
 
@@ -111,6 +112,9 @@ def process_arch(args):
                     logging.debug("nothing to do for maintainer %s" % (name))
                     continue
 
+                # queue for source package validator
+                queue.add(args, to_relarea, os.path.join(m.homedir(), args.arch))
+
                 # merge package set
                 merged_packages = package.merge(packages, mpackages)
 
@@ -243,6 +247,7 @@ if __name__ == "__main__":
     setupdir_default = common_constants.HTDOCS
     vault_default = common_constants.VAULT
     logdir_default = '/sourceware/cygwin-staging/logs'
+    queuedir_default = '/sourceware/cygwin-staging/queue'
 
     parser = argparse.ArgumentParser(description='Upset replacement')
     parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help='email output to maintainer and ADDRS (default: ' + common_constants.EMAILS + ')', metavar='ADDRS')
@@ -252,6 +257,7 @@ if __name__ == "__main__":
     parser.add_argument('--logdir', action='store', metavar='DIR', help="log directory (default: '" + logdir_default + "')", default=logdir_default)
     parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default)
     parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default)
+    parser.add_argument('--queuedir', action='store', metavar='DIR', help="queue directory (default: '" + queuedir_default + "')", default=queuedir_default)
     parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin')
     parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area')
     parser.add_argument('--setupdir', action='store', metavar='DIR', help="setup executable directory (default: " + setupdir_default + ")", default=setupdir_default)
diff --git a/maintainers.py b/maintainers.py
index 8bc9ec7..a078376 100644
--- a/maintainers.py
+++ b/maintainers.py
@@ -71,6 +71,9 @@ class Maintainer(object):
         self._homedirs = homedirs
 
         for n in os.listdir(homedirs):
+            if not os.path.isdir(os.path.join(homedirs, n)):
+                continue
+
             m = Maintainer._find(mlist, n)
 
             for e in ['!email', '!mail']:
diff --git a/package.py b/package.py
index 8e6a405..cddbdb5 100755
--- a/package.py
+++ b/package.py
@@ -101,6 +101,7 @@ def sha512_file(fn, block_size=256*128):
 # read a single package
 #
 def read_package(packages, basedir, dirpath, files, strict=False):
+    strict_lvl = logging.ERROR if strict else logging.WARNING
     relpath = os.path.relpath(dirpath, basedir)
     warnings = False
 
@@ -161,8 +162,8 @@ def read_package(packages, basedir, dirpath, files, strict=False):
             # start with a number
             match = re.match(r'^' + re.escape(p) + '-(.+)-(\d[0-9a-zA-Z.]*)(-src|)\.tar\.(bz2|gz|lzma|xz)$', f)
             if not match:
-                logging.warning("tar file '%s' in package '%s' doesn't follow naming convention" % (f, p))
-                warning = True
+                logging.lvl(strict_lvl, "tar file '%s' in package '%s' doesn't follow naming convention" % (f, p))
+                warnings = True
             else:
                 # historically, V can contain a '-' (since we can use the fact
                 # we already know P to split unambiguously), but this is a bad
@@ -187,8 +188,8 @@ def read_package(packages, basedir, dirpath, files, strict=False):
         # warn about unexpected files, including tarfiles which don't match the
         # package name
         if files:
-            logging.warning("unexpected files in %s: %s" % (p, ', '.join(files)))
-            warning = True
+            logging.log(strict_lvl, "unexpected files in %s: %s" % (p, ', '.join(files)))
+            warnings = True
 
         packages[p].hints = hints
         packages[p].tars = tars
@@ -214,7 +215,7 @@ def read_package(packages, basedir, dirpath, files, strict=False):
             if colon:
                 package_basename = re.sub(r'^lib(.*?)(|-devel|\d*)$', r'\1', p)
                 if package_basename.upper().startswith(colon.group(1).upper()):
-                    logging.warning("package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2))))
+                    logging.log(strict_lvl, "package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2))))
                     warnings = True
 
     elif (len(files) > 0) and (relpath.count(os.path.sep) > 0):
@@ -222,6 +223,7 @@ def read_package(packages, basedir, dirpath, files, strict=False):
 
     if strict:
         return warnings
+
     return False
 
 
@@ -301,7 +303,7 @@ def validate_packages(args, packages):
 
             # extract just the version part from tar filename
             v = re.sub(r'^' + re.escape(p) + '-', '', t)
-            v = re.sub(r'(-src|)\.tar\.(xz|bz2|gz)$', '', v)
+            v = re.sub(r'(-src|)\.tar\.(bz2|gz|lzma|xz)$', '', v)
 
             # for each version, a package can contain at most one source tar
             # file and at most one install tar file.  warn if we have too many
diff --git a/queue.py b/queue.py
new file mode 100644
index 0000000..ed4f89a
--- /dev/null
+++ b/queue.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2016 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+from dirq.QueueSimple import QueueSimple
+import logging
+import os
+import re
+
+import uploads
+
+QUEUE = 'package_queue'
+
+
+#
+#
+#
+
+def add(args, movelist, fromdir):
+    if not hasattr(args, 'queuedir'):
+        return
+
+    queue_root = os.path.join(args.queuedir, 'dirq')
+    upload_root = os.path.join(args.queuedir, 'uploads')
+
+    dirq = QueueSimple(os.path.join(queue_root, QUEUE))
+
+    # clean up empty directories
+    dirq.purge()
+    os.system('find %s -depth -mindepth 1 -type d -empty -delete' % upload_root)
+
+    # are there any source packages in the filelist?
+    srcpkgs = []
+    for p in movelist:
+        for f in movelist[p]:
+            if re.search(r'-src.tar.(bz2|gz|lzma|xz)$', f):
+                srcpkgs.append(os.path.join(args.arch, p, f))
+
+    # if so...
+    #
+    # XXX: really this should break things up into the set of files for each
+    # source file
+    if len(srcpkgs) >= 1:
+        # keep all the files for comparison
+        uploads.copy(args, movelist, fromdir, os.path.join(upload_root, args.arch))
+
+        # queue any srcpkgs
+        for p in srcpkgs:
+            if not args.dryrun:
+                logging.debug("queuing source package %s for validation" % (p))
+                dirq.add(p)
diff --git a/testdata/homes/an_unexpected_file b/testdata/homes/an_unexpected_file
new file mode 100644
index 0000000..e69de29
diff --git a/testdata/process_arch/homedir.expected b/testdata/process_arch/homedir.expected
index 39fb742..091867f 100644
--- a/testdata/process_arch/homedir.expected
+++ b/testdata/process_arch/homedir.expected
@@ -1,4 +1,4 @@
-{'.': [],
+{'.': ['an_unexpected_file'],
  'Blooey McFooey': [],
  'Blooey McFooey/x86': [],
  'Blooey McFooey/x86/release': [],
diff --git a/uploads.py b/uploads.py
index 0df5d39..6909d76 100644
--- a/uploads.py
+++ b/uploads.py
@@ -30,6 +30,7 @@ import filecmp
 import os
 import logging
 import re
+import shutil
 import time
 
 import package
@@ -236,3 +237,25 @@ def move_to_relarea(m, args, movelist):
 
 def move_to_vault(args, movelist):
     move(args, movelist, os.path.join(args.rel_area, args.arch), os.path.join(args.vault, args.arch))
+
+
+#
+#
+#
+
+def copy(args, movelist, fromdir, todir):
+    for p in sorted(movelist):
+        logging.debug("mkdir %s" % os.path.join(todir, p))
+        if not args.dryrun:
+            try:
+                os.makedirs(os.path.join(todir, p), exist_ok=True)
+            except FileExistsError:
+                pass
+        logging.debug("copy from '%s' to '%s':" % (os.path.join(fromdir, p), os.path.join(todir, p)))
+        for f in sorted(movelist[p]):
+            if os.path.exists(os.path.join(fromdir, p, f)):
+                logging.debug("%s" % (f))
+                if not args.dryrun:
+                    shutil.copy2(os.path.join(fromdir, p, f), os.path.join(todir, p, f))
+            else:
+                logging.error("%s can't be copied as it doesn't exist" % (f))


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]