Re: setup

[Achim Gratz <Stromeko at nexgo dot de>]

Achim Gratz writes:
> I'll have to write a ChangeLog and clean up the code a bit before I'll
> post the patches.

Here's the patch for setup:

--8<---------------cut here---------------start------------->8---
>From 3ce2ac3109f7acb38491b74bd48751ef76eae42c Mon Sep 17 00:00:00 2001
From: Achim Gratz <Stromeko@Stromeko.DE>
Date: Sun, 7 Jun 2015 09:10:13 +0200
Subject: [PATCH] Implement Base64URL-encoded SHA512 checksums

	* ini.h: Add macros for use within the implementation of the
	checksum parsers.  Hexdigest requires a 2-to-1 and Base64 a 4-to-3
	conversion.  Base64 uses the filename and URL safe alphabet from
	RFC4648.  It would be trivial to additionally process the normal
	Base64 alphabet but we don't want to allow that in order to be
	able to use the checksums as filenames without further conversion.
	(hexnibble, b64url): Process single input character to input value.
	(nibbled1): Convert 2 processed input values into 1 output value.
	(b64d1, b64d2, b64d3): Convert 4 input values into 3 output
	values.
	* iniparse.yy: Add SHA512B64URL checksum in addition to MD5 and
	SHA512.  Keep symmetry with MD5LINE and also define SHA512LINE
	syntax.
	* inilex.ll: Implement existing MD5 and SHA512 checksum parsers
	using new macros.  Implement new SHA512B64URL parser using new
	macros.  Enforce Base64 alphabet as defined in RFC4648 with no
	padding to enable direct use of checksum values as filenames.
	Implement SHA512LINE parser.
---
 ChangeLog   | 21 +++++++++++++++++++++
 ini.h       | 14 ++++++++++++++
 inilex.ll   | 55 +++++++++++++++++++++++++++++++++++++++----------------
 iniparse.yy |  9 +++++++--
 4 files changed, 81 insertions(+), 18 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 854ba05..b2b3ae0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,24 @@
+2015-06-07  Achim Gratz  <Stromeko@NexGo.DE>
+
+	* ini.h: Add macros for use within the implementation of the
+	checksum parsers.  Hexdigest requires a 2-to-1 and Base64 a 4-to-3
+	conversion.  Base64 uses the filename and URL safe alphabet from
+	RFC4648.  It would be trivial to additionally process the normal
+	Base64 alphabet but we don't want to allow that in order to be
+	able to use the checksums as filenames without further conversion.
+	(hexnibble, b64url): Process single input character to input value.
+	(nibbled1): Convert 2 processed input values into 1 output value.
+	(b64d1, b64d2, b64d3): Convert 4 input values into 3 output
+	values.
+	* iniparse.yy: Add SHA512B64URL checksum in addition to MD5 and
+	SHA512.  Keep symmetry with MD5LINE and also define SHA512LINE
+	syntax.
+	* inilex.ll: Implement existing MD5 and SHA512 checksum parsers
+	using new macros.  Implement new SHA512B64URL parser using new
+	macros.  Enforce Base64 alphabet as defined in RFC4648 with no
+	padding to enable direct use of checksum values as filenames.
+	Implement SHA512LINE parser.
+
 2015-03-25  Corinna Vinschen  <corinna@vinschen.de>
 
 	* install.cc (sha512_one): Raise buffer size to 64K for performance
diff --git a/ini.h b/ini.h
index 232c600..ec09def 100644
--- a/ini.h
+++ b/ini.h
@@ -52,4 +52,18 @@ extern std::string current_ini_sig_name;  /* current filename/URL for sig file *
 extern std::string yyerror_messages;  /* textual parse error messages */
 extern int yyerror_count;             /* number of parse errors */
 
+/* The following definitions are used in the parser implementation */
+
+#define hexnibble(val) (255 & (val > '9') ? val - 'a' + 10 : val - '0');
+#define nibbled1(v1,v2) (255 & ((v1 << 4) & v2));
+#define b64url(val)						\
+  (63 & ((  val == '_') ? 63					\
+	 : (val == '-') ? 62					\
+	 : (val >= 'a') ? val - 'a' + 26			\
+	 : (val >= 'A') ? val - 'A' +  0			\
+	 :                val - '0' + 52))
+#define b64d1(v1,v2,v3,v4) (255 & ((v1 << 2) | (v2 >> 4)));
+#define b64d2(v1,v2,v3,v4) (255 & ((v2 << 4) | (v3 >> 2)));
+#define b64d3(v1,v2,v3,v4) (255 & ((v3 << 6) |  v4));
+
 #endif /* SETUP_INI_H */
diff --git a/inilex.ll b/inilex.ll
index 81a7f24..2e767ec 100644
--- a/inilex.ll
+++ b/inilex.ll
@@ -48,36 +48,58 @@ STR	[!a-zA-Z0-9_./:\+~-]+
 
 %%
 
-[0123456789abcdef]{32,32}       {
+[0-9a-f]{32} {
     yylval = (char *) new unsigned char[16];
     memset (yylval, 0, 16);
-    for (int i = 0; i < 32; ++i)
+    int i, j;
+    unsigned char v1, v2;
+    for (i = 0, j = 0; i < 32; i += 2, ++j)
       {
-	unsigned char val = (unsigned char) yytext[i];
-	if (val > '9')
-	  val = val - 'a' + 10;
-	else
-	  val = val - '0';
-	((unsigned char *) yylval) [i / 2] += val << ((i % 2) ? 0 : 4);
+	v1 = hexnibble((unsigned char) yytext[i+0]);
+	v2 = hexnibble((unsigned char) yytext[i+1]);
+	((unsigned char *) yylval) [j] = nibbled1(v1, v2);
       }
     return MD5;
 }
 
-[0123456789abcdef]{128,128}       {
+[0-9a-f]{128} {
     yylval = (char *) new unsigned char[SHA512_DIGEST_LENGTH];
     memset (yylval, 0, SHA512_DIGEST_LENGTH);
-    for (int i = 0; i < SHA512_BLOCK_LENGTH; ++i)
+    int i, j;
+    unsigned char v1, v2;
+    for (i = 0, j = 0; i < SHA512_BLOCK_LENGTH; i += 2, ++j)
       {
-	unsigned char val = (unsigned char) yytext[i];
-	if (val > '9')
-	  val = val - 'a' + 10;
-	else
-	  val = val - '0';
-	((unsigned char *) yylval) [i / 2] += val << ((i % 2) ? 0 : 4);
+	v1 = hexnibble((unsigned char) yytext[i+0]);
+	v2 = hexnibble((unsigned char) yytext[i+1]);
+	((unsigned char *) yylval) [j] = nibbled1(v1, v2);
       }
     return SHA512;
 }
 
+[a-zA-Z0-9_-]{86} {
+    /* base64url as defined in RFC4648 */
+    yylval = (char *) new unsigned char[SHA512_DIGEST_LENGTH];
+    memset (yylval, 0, SHA512_DIGEST_LENGTH);
+    int i, j;
+    unsigned char v1, v2, v3, v4;
+    for (i = 0, j = 0; i < 4*(SHA512_DIGEST_LENGTH/3); i += 4, j += 3)
+      {
+	v1 = b64url(((unsigned char) yytext[i+0]));
+	v2 = b64url(((unsigned char) yytext[i+1]));
+	v3 = b64url(((unsigned char) yytext[i+2]));
+	v4 = b64url(((unsigned char) yytext[i+3]));
+	((unsigned char *) yylval) [j+0] = b64d1(v1, v2, v3, v4);
+	((unsigned char *) yylval) [j+1] = b64d2(v1, v2, v3, v4);
+	((unsigned char *) yylval) [j+2] = b64d3(v1, v2, v3, v4);
+      }
+    v1 = b64url((unsigned char) yytext[i+0]);
+    v2 = b64url((unsigned char) yytext[i+1]);
+    v3 = 0;
+    v4 = 0;
+    ((unsigned char *) yylval) [j+0] = b64d1(v1, v2, v3, v4);
+    return SHA512B64URL;
+}
+
 \"[^"]*\"		{ yylval = new char [strlen (yytext+1) + 1];
 			  strcpy (yylval, yytext+1);
 			  yylval[strlen (yylval)-1] = 0;
@@ -98,6 +120,7 @@ STR	[!a-zA-Z0-9_./:\+~-]+
 "Description:"		BEGIN (descriptionstate); return DESCTAG;
 "Size:"			return FILESIZE;
 "MD5sum:"		return MD5LINE;
+"SHA512:"		return SHA512LINE;
 "Installed-Size:"	return INSTALLEDSIZE;
 "Maintainer:"		BEGIN (eolstate); return MAINTAINER;
 "Architecture:"		return ARCHITECTURE;
diff --git a/iniparse.yy b/iniparse.yy
index cab84f2..0cd1c64 100644
--- a/iniparse.yy
+++ b/iniparse.yy
@@ -42,8 +42,9 @@ void add_correct_version();
 %token CATEGORY DEPENDS REQUIRES
 %token APATH PPATH INCLUDE_SETUP EXCLUDE_PACKAGE DOWNLOAD_URL
 %token T_PREV T_CURR T_TEST
-%token SHA512 MD5 INSTALLEDSIZE MAINTAINER PRIORITY
-%token DESCTAG DESCRIPTION FILESIZE ARCHITECTURE SOURCEPACKAGE MD5LINE 
+%token SHA512 SHA512B64URL MD5 INSTALLEDSIZE MAINTAINER PRIORITY
+%token MD5LINE SHA512LINE
+%token DESCTAG DESCRIPTION FILESIZE ARCHITECTURE SOURCEPACKAGE
 %token RECOMMENDS PREDEPENDS
 %token SUGGESTS CONFLICTS REPLACES PROVIDES PACKAGENAME STRTOEOL PARAGRAPH
 %token EMAIL COMMA OR NL AT
@@ -107,6 +108,8 @@ singleitem /* non-empty */
  | DIRECTORY STRING NL		{ /* TODO */ }
  | STANDARDSVERSION STRING NL	{ /* TODO */ }
  | MD5LINE MD5 NL	{ iniBuilder->buildInstallMD5 ((unsigned char *)$2); }
+ | MD5LINE SHA512 NL	{ iniBuilder->buildInstallSHA512 ((unsigned char *)$2); }
+ | MD5LINE SHA512B64URL NL	{ iniBuilder->buildInstallSHA512 ((unsigned char *)$2); }
  | SOURCEPACKAGE source NL
  | CATEGORY categories NL
  | INSTALL STRING { iniBuilder->buildPackageInstall ($2); } installmeta NL
@@ -146,11 +149,13 @@ installmeta: /* empty */
 installchksum: /* empty */
  | MD5 			{ iniBuilder->buildInstallMD5 ((unsigned char *)$1);}
  | SHA512		{ iniBuilder->buildInstallSHA512 ((unsigned char *)$1);}
+ | SHA512B64URL		{ iniBuilder->buildInstallSHA512 ((unsigned char *)$1);}
  ;
 
 sourcechksum: /* empty */
  | MD5 			{ iniBuilder->buildSourceMD5 ((unsigned char *)$1); }
  | SHA512 		{ iniBuilder->buildSourceSHA512 ((unsigned char *)$1); }
+ | SHA512B64URL		{ iniBuilder->buildSourceSHA512 ((unsigned char *)$1); }
  ;
 
 source /* non-empty */
-- 
2.3.3.GIT
--8<---------------cut here---------------end--------------->8---

And this is the change to genini so you can produce whichever digest you
want and default to SHA512-hex:

--8<---------------cut here---------------start------------->8---
Index: genini
===================================================================
RCS file: /cvs/cygwin-apps/genini/genini,v
retrieving revision 1.15
diff -u -p -r1.15 genini
--- genini      7 Oct 2013 21:11:35 -0000       1.15
+++ genini      7 Jun 2015 20:14:29 -0000
@@ -7,6 +7,7 @@
 #
 use File::Basename;
 use Digest::MD5;
+use Digest::SHA;
 use Getopt::Long;
 
 use strict;
@@ -19,10 +20,17 @@ sub arch_handler(@);
 my @okmissing = qw'message ldesc';
 my ($outfile, $help, $recursive);
 my $arch = 'x86';
+my $digest = 'sha512';
 my $release;
-my @cmp_fmts = qw(gz bz2 lzma xz);
+my @cmp_fmts = qw(xz bz2 lzma gz);
 
-GetOptions('okmissing=s'=>\@okmissing, 'output=s'=>\$outfile, 'help'=>\$help, 'release=s'=>\$release, 'arch=s'=>\&arch_handler, 'recursive'=>\$recursive) or usage;
+GetOptions('okmissing=s'=>\@okmissing,
+          'output=s'=>\$outfile,
+          'help'=>\$help,
+          'release=s'=>\$release,
+          'arch=s'=>\&arch_handler,
+          'digest=s'=>\&digest_handler,
+          'recursive'=>\$recursive) or usage;
 $help and usage;
 
 @main::okmissing{@okmissing} = @okmissing;
@@ -31,7 +39,14 @@ sub arch_handler (@) {
    my ($opt_name, $opt_value) = @_;
    die "invalid arch specified: '$opt_value'"
       unless $main::valid_arch{lc $opt_value};
-   $arch = $opt_value;
+   $arch = lc $opt_value;
+}
+
+sub digest_handler (@) {
+   my ($opt_name, $opt_value) = @_;
+   die "invalid digest specified: '$opt_value'"
+      unless $main::valid_digest{lc $opt_value};
+   $digest = lc $opt_value;
 }
 
 if (defined($outfile)) {
@@ -190,6 +205,7 @@ sub parsedir {
     my $setup_hint = "$d/setup.hint";
     return unless -e $setup_hint;
     parse("$setup_hint", $pname);
+    next unless exists $pkg{$pname};
     my $explicit = 0;
     for my $what ('', "[prev]\n", "[test]\n") {
        my $x = $pkg{$pname}{$what};
@@ -246,9 +262,20 @@ sub filer {
        myerror "can't open $f - $!" unless $main::okmissing{$what};
        return undef;
     };
-    my $md5 = Digest::MD5->new;
-    $md5->addfile(\*F);
-    $x->{$what} = join(' ', $f, -s $f, $md5->hexdigest);
+    my ( $chk, $sum );
+    if ('md5' eq $digest) {
+       $chk = Digest::MD5->new;
+    } elsif ('sha512' eq substr($digest, 0, 6)) {
+       $chk = Digest::SHA->new(512);
+    }
+    $chk->addfile(\*F);
+    if ('sha512b64url' eq $digest) {
+       $sum = $chk->b64digest;
+       $sum =~ tr:+/:-_:; # convert to base64url as defined in RFC4648
+    } else {
+       $sum = $chk->hexdigest;
+    }
+    $x->{$what} = join(' ', $f, -s $f, $sum);
 }
 
 sub tarball {
@@ -260,8 +287,8 @@ sub tarball {
         return "$f";
       }
     }
-    # default to .bz2 (even though we know it is missing)
-    return "$d/" . "$b" . "bz2";
+    # default to .nf (because we know it is missing)
+    return "$d/" . "$b" . "nf";
 }
 
 sub fnln {
@@ -312,13 +339,17 @@ EOF
 
 BEGIN {
     my @cats = qw'
-     Admin Archive Audio Base Comm Database Debug Devel Doc Editors Games
-     Gnome Graphics Interpreters KDE Libs Mail Math Mingw Net Perl
-     Publishing Python Science Shells Sound System Text Utils Web X11
-     _obsolete _PostInstallLast
+     Accessibility Admin Archive Audio Base Comm Database Devel Doc
+     Editors Games GNOME Graphics Interpreters KDE Libs LXDE Mail MATE
+     Math Mingw Net Perl PHP Python Publishing Ruby Science Shells
+     System Text Utils Video Web X11 Xfce
+     Debug _obsolete _PostInstallLast
      ';
     @main::categories{map {lc $_} @cats} = @cats;
 
     my @archs = qw'x86 x86_64';
     @main::valid_arch{map {lc $_} @archs} = @archs;
+
+    my @digests = qw'md5 sha512 sha512b64url';
+    @main::valid_digest{map {lc $_} @digests} = @digests;
 }
--8<---------------cut here---------------end--------------->8---

Regards,
Achim.
-- 
+<[Q+ Matrix-12 WAVE#46+305 Neuron microQkb Andromeda XTk Blofeld]>+

DIY Stuff:
http://Synth.Stromeko.net/DIY.html