Merge lp:~epics-core/epics-base/config-from-snippets into lp:~epics-core/epics-base/3.16

[Ralph Lange]

Adds the ability to create config files from snippets.

Documentation is still TBD.

Usage (e.g. in exampleApp/src/Makefile)

  DB_INSTALLS += $(COMMON_DIR)/ka.mm
  SCRIPTS += ka.nn

  COMMON_ASSEMBLE_FROM_SNIPPETS += ka.mm
  ka.mm_SNIPPETS = $(wildcard ../ka.d/*)
  ASSEMBLE_FROM_SNIPPETS += ka.nn
  ka.nn_SNIPPETS = $(wildcard ../ka.d/*)

This creates two (in this case identical) files:
ka.mm is created in the COMMON_DIR and installed in TOP/db.
ka.nn is created in the T_A dir and installed in TOP/bin/<arch>.

[Andrew Johnson (anj)]

Is this intended for 3.16 (lp:epics-base) or 3.15 (lp:epics-base/3.15)? The merge proposal implies the former.

Haven't played with it yet and (of course) I'm going to try to think up something shorter than ASSEMBLE_FROM_SNIPPETS in the Makefile names, but overall the idea looks good.

Do you need a review quickly? I will be on vacation from 2015-10-14 through 2015-10-23 but I should be able to squeeze it in before I go.

[lp:~epics-core/epics-base/config-from-snippets]

12707. By Ralph Lange on 2015-10-10

Back out 12706 (assembly integration in build system)

12708. By Ralph Lange on 2015-10-10

tools: fix in assembleSnippets script and test for Windows (getpwuid() not implemented)

12709. By Ralph Lange on 2015-10-10

configure: add assemblies to RULES_EXPAND

12710. By Ralph Lange on 2015-10-10

tools: add new builtin macros to assemblies

12711. By Andrew Johnson on 2015-10-11

Minor RULES_EXPAND changes

Added support for a _PATTERN variable which calls $(wildcard) and
prepends .. and the other SRC_DIRS automatically.
Use $(ECHO) instead of @echo so make -s works silently.
Added another meta-rule to create .d files when make needs one.

Unmerged revisions

=== modified file 'configure/RULES_BUILD'
--- configure/RULES_BUILD	2015-07-24 20:59:38 +0000
+++ configure/RULES_BUILD	2015-10-09 15:33:25 +0000
@@ -316,6 +316,22 @@
 	$(LINK.mod)
 
 #---------------------------------------------------------------
+# Assembled files (from snippets)
+
+COMMON_ASSEMBLE_FROM_SNIPPETS_ += $(addprefix $(COMMON_DIR)/,$(COMMON_ASSEMBLE_FROM_SNIPPETS))
+
+$(COMMON_ASSEMBLE_FROM_SNIPPETS_): $(COMMON_DIR)/%:
+	@echo "Assembling common file $@ from snippets"
+	@$(RM) $(notdir $@)
+	$(PERL) $(TOOLS)/assembleSnippets.pl -o $(notdir $@) $($(notdir $@)_SNIPPETS)
+	@$(MV) $(notdir $@) $@
+
+$(ASSEMBLE_FROM_SNIPPETS): %:
+	@echo "Assembling file $@ from snippets"
+	@$(RM) $(notdir $@)
+	$(PERL) $(TOOLS)/assembleSnippets.pl -o $(notdir $@) $($(notdir $@)_SNIPPETS)
+
+#---------------------------------------------------------------
 # Automated testing
 
 runtests: $(TESTSCRIPTS)

=== modified file 'src/tools/Makefile'
--- src/tools/Makefile	2015-07-10 19:02:01 +0000
+++ src/tools/Makefile	2015-10-09 15:33:25 +0000
@@ -33,6 +33,7 @@
 PERL_MODULES += DBD/Registrar.pm
 PERL_MODULES += DBD/Variable.pm
 
+PERL_SCRIPTS += assembleSnippets.pl
 PERL_SCRIPTS += convertRelease.pl
 PERL_SCRIPTS += cvsclean.pl
 PERL_SCRIPTS += dos2unix.pl

=== added file 'src/tools/assembleSnippets.pl'
--- src/tools/assembleSnippets.pl	1970-01-01 00:00:00 +0000
+++ src/tools/assembleSnippets.pl	2015-10-09 15:33:25 +0000
@@ -0,0 +1,145 @@
+#!/usr/bin/env perl
+#*************************************************************************
+# Copyright (c) 2015 ITER Organization.
+# EPICS BASE is distributed subject to a Software License Agreement found
+# in file LICENSE that is included with this distribution.
+#*************************************************************************
+
+# $Id$
+
+use strict;
+use warnings;
+
+use Getopt::Std;
+use File::Basename;
+use Data::Dumper;
+
+our ($opt_o, $opt_d, $opt_m, $opt_i, $opt_M);
+
+$Getopt::Std::OUTPUT_HELP_VERSION = 1;
+&HELP_MESSAGE if !getopts('M:i:m:o:d') || @ARGV == 0;
+
+my $out;
+my $dep;
+my %snippets;
+my $ipattern;
+
+my $datetime = localtime();
+(my $user, my @dummy) = getpwuid($<);
+my %replacements = (
+    _DATETIME_ => $datetime,
+    _USER_ => $user,
+);
+
+if ($opt_o) {
+    open $out, '>', $opt_o or
+        die "Can't create $opt_o: $!\n";
+    print STDERR "opened file $opt_o for output\n" if $opt_d;
+} else {
+    open $out, '>&', STDOUT;
+    print STDERR "using STDOUT for output\n" if $opt_d;
+}
+
+if ($opt_m) {
+    foreach my $r (split /,/, $opt_m) {
+        (my $k, my $v) = split /=/, $r;
+        $replacements{$k} = $v;
+    }
+}
+
+if ($opt_M) {
+    open $dep, '>', $opt_M or
+        die "Can't create $opt_M: $!\n";
+    print STDERR "opened dependency file $opt_M for output\n" if $opt_d;
+    print $dep basename($opt_o), ":";
+}
+
+if ($opt_i) {
+    $ipattern = qr($opt_i);
+}
+
+# %snippets is a hash {rank}
+#                  of hashes {name-after-rank}
+#                      of arrays[] [files...]
+#                          of arrays[2] [filename, command]
+print STDERR "reading input files\n" if $opt_d;
+foreach (@ARGV) {
+    my $name = basename($_);
+    if ($opt_i and not $name =~ /$ipattern/) {
+        print STDERR "  snippet $_ does not match input pattern $opt_i - ignoring\n" if $opt_d;
+        next;
+    }
+    if ($name =~ /\A([ARD]?)([0-9]+)(.*[^~])\z/) {
+        print STDERR "  considering snippet $_\n" if $opt_d;
+        if (exists $snippets{$2}) {
+            my %rank = %{$snippets{$2}};
+            my @files = @{ $rank{(keys %rank)[0]} };
+            my $existcmd = $files[0]->[1];
+            if ($1 eq "D" and $existcmd ne "D") {
+                print STDERR "    ignoring 'D' default for existing rank $2\n" if $opt_d;
+                next;
+            } elsif ($1 eq "R") {
+                print STDERR "    'R' command - deleting existing rank $2 snippets\n" if $opt_d;
+                $snippets{$2} = {};
+            } elsif ($existcmd eq "D") {
+                print STDERR "    deleting existing rank $2 default snippet\n" if $opt_d;
+                $snippets{$2} = {};
+            }
+        }
+        if ($opt_d) {
+            print STDERR "    adding snippet ";
+            print STDERR "marked as default " if $1 eq "D";
+            print STDERR "to rank $2\n";
+        }
+        $snippets{$2}{$3} = () if (not exists $snippets{$2}{$3});
+        push @{$snippets{$2}{$3}}, [ $_, $1 ];
+    }
+}
+
+if ($opt_d) {
+    print STDERR "finished reading input files\n";
+    print STDERR "dumping the final snippet structure\n";
+    print STDERR Dumper(\%snippets);
+    print STDERR "dumping the macro replacements\n";
+    print STDERR Dumper(\%replacements);
+    print STDERR "creating output\n";
+}
+
+foreach my $r (sort {$a<=>$b} keys %snippets) {
+    print STDERR "  working on rank $r\n" if $opt_d;
+    foreach my $n (sort keys %{$snippets{$r}}) {
+        foreach my $s (@{$snippets{$r}{$n}}) {
+            my $in;
+            my $f = $s->[0];
+            print STDERR "    snippet $n from file $f\n" if $opt_d;
+            open $in, '<', $f or die "Can't open $f: $!\n";
+            print $dep " \\\n $f" if $opt_M;
+            while (<$in>) {
+                chomp;
+                foreach my $k (keys %replacements) {
+                    s/$k/$replacements{$k}/g;
+                }
+                print $out $_, "\n";
+            }
+            close $in;
+        }
+    }
+}
+
+print STDERR "finished creating output, closing\n" if $opt_d;
+if ($opt_M) {
+    print $dep "\n";
+    close $dep;
+}
+close $out;
+
+sub HELP_MESSAGE {
+    print STDERR "Usage: assembleSnippets.pl [options] snippets ...\n";
+    print STDERR "Options:\n";
+    print STDERR " -o file     output file [STDOUT]\n";
+    print STDERR " -d          debug mode [no]\n";
+    print STDERR " -m macros   list of macro replacements as \"key=val,key=val\"\n";
+    print STDERR " -i pattern  pattern for input files to match\n";
+    print STDERR " -M file     write file with dependency rule suitable for make\n";
+    exit 2;
+}

=== modified file 'src/tools/test/Makefile'
--- src/tools/test/Makefile	2012-03-14 20:27:40 +0000
+++ src/tools/test/Makefile	2015-10-09 15:33:25 +0000
@@ -18,6 +18,7 @@
 TESTS += Recfield
 TESTS += Recordtype
 TESTS += Registrar
+TESTS += Snippets
 TESTS += Variable
 
 TESTSCRIPTS_HOST += $(TESTS:%=%.t)

=== added file 'src/tools/test/Snippets.plt'
--- src/tools/test/Snippets.plt	1970-01-01 00:00:00 +0000
+++ src/tools/test/Snippets.plt	2015-10-09 15:33:25 +0000
@@ -0,0 +1,102 @@
+#!/usr/bin/env perl
+
+use File::Path;
+
+use Test::More tests => 26;
+
+(my $user, my @rest) = getpwuid($<);
+
+mkdir "a$$";
+mkdir "b$$";
+
+sub mksnip {
+    my ($dir, $file, $line) = @_;
+    open(my $fh, '>', "$dir$$/$file") or die "can't open $dir$$/$file : $!";
+    print $fh $line;
+    close $fh;
+}
+
+sub assemble {
+    my @cmd = ( 'perl', '../../assembleSnippets.pl', '-o', "out$$" );
+    push @cmd, @_;
+    system(@cmd);
+    open(my $fh, '<', "out$$") or die "can't open out$$ : $!";
+    chomp(my @result = <$fh>);
+    close $fh;
+    return join (' ', @result);
+}
+
+# Adding two snippets of same rank, sorting alphabetically
+mksnip('a', '10_a', '10');
+mksnip('b', '10_c', '12');
+is assemble("a$$/10_a", "b$$/10_c"), '10 12', "adding same rank; ordered";
+is assemble("b$$/10_c", "a$$/10_a"), '10 12', "adding same rank; reverse order";
+
+# Same, with 'A' cmd
+mksnip('a', 'A10_a', '10');
+mksnip('b', 'A10_c', '12');
+is assemble("a$$/10_a", "b$$/A10_c"), '10 12', "'A' add same rank; ordered";
+is assemble("b$$/10_c", "a$$/A10_a"), '10 12', "'A' add same rank; reverse order";
+
+# Same name does not create issues
+mksnip('b', '10_a', '10x');
+is assemble("a$$/10_a", "b$$/10_a"), '10 10x', "adding same name twice; order a-b";
+is assemble("b$$/10_a", "a$$/10_a"), '10x 10', "adding same name twice; order b-a";
+
+# Backup files (trailing ~) and hidden files (leading '.') get ignored
+mksnip('b', '10_c~', '12~');
+mksnip('b', '.10_c', '.12');
+is assemble("b$$/10_c", "b$$/10_c~"), '12', "backup file (trailing ~) gets ignored";
+is assemble("b$$/10_c", "b$$/.10_c"), '12', "hidden file (leading .) gets ignored";
+
+# Non-numeric filenames get ignored
+mksnip('a', 'foo10_a', 'foo10');
+is assemble("b$$/10_c", "a$$/foo10_a"), '12', "file starting with [^ADR0-9] gets ignored";
+
+# 'R' command replaces existing snippets of same rank
+mksnip('a', 'R10_b', '11');
+is assemble("a$$/10_a", "b$$/10_c", "a$$/R10_b"), '11', "'R' cmd; replace all";
+is assemble("a$$/10_a", "a$$/R10_b", "b$$/10_c"), '11 12', "'R' cmd; replace one (ordered)";
+is assemble("b$$/10_c", "a$$/R10_b", "a$$/10_a"), '10 11', "'R' cmd; replace one (reverse order)";
+
+# 'D' command establishes default that gets overwritten or ignored
+mksnip('a', 'D10_a', 'D10');
+mksnip('b', 'D10_c', 'D12');
+is assemble("a$$/D10_a", "b$$/10_c"),  '12',  "'D' default; replaced by regular";
+is assemble("a$$/D10_a", "b$$/D10_c"), 'D12', "'D' default; replaced by new default (ordered)";
+is assemble("b$$/D10_c", "a$$/D10_a"), 'D10', "'D' default; replaced by new default (reverse order)";
+is assemble("a$$/D10_a", "a$$/R10_b"), '11',  "'D' default; replaced by 'R' cmd";
+is assemble("b$$/10_c", "a$$/D10_a"),  '12',  "'D' default; ignored when regular exists";
+
+# Ranks are sorted numerically
+mksnip('b', '09_a', '09');
+mksnip('a', '15_a', '15');
+mksnip('b', '2_a', '2');
+is assemble("a$$/10_a", "b$$/2_a", "a$$/15_a", "b$$/09_a"), '2 09 10 15',  "ranks are sorted numerically";
+
+# Builtin macros
+mksnip('a', '30_a', '_USER_');
+is assemble("a$$/30_a"), "$user", "builtin macro _USER_";
+
+# User macros
+mksnip('b', '35_a', 'Line _M1_');
+mksnip('b', '35_b', 'Line _M1_ with _M2_');
+mksnip('b', '35_c', 'Line _M2_ with _M2_');
+is assemble("-m", "_M1_=REP1", "b$$/35_a"), "Line REP1", "single user macro; single occurrence";
+is assemble("-m", "_M1_=REP1,_M2_=REP2", "b$$/35_b"), "Line REP1 with REP2", "multiple user macros";
+is assemble("-m", "_M2_=REP2", "b$$/35_c"), "Line REP2 with REP2", "single user macro; multiple occurrences";
+
+# Input pattern
+mksnip('a', '10_a.cmd', '10cmd');
+is assemble("-i", "\.cmd", "a$$/10_a", "b$$/10_c", "a$$/R10_b", "a$$/10_a.cmd"), '10cmd', "input pattern";
+
+# Dependency file generation
+assemble("-M", "./dep$$", "a$$/10_a", "b$$/10_c");
+open(my $fh, '<', "dep$$") or die "can't open dep$$ : $!";
+chomp(my @result = <$fh>);
+close $fh;
+is "$result[0]", "out$$: \\", "dependency file (line 1)";
+is "$result[1]", " a$$/10_a \\", "dependency file (line 2)";
+is "$result[2]", " b$$/10_c", "dependency file (line 3)";
+
+rmtree([ "a$$", "b$$", "out$$", "dep$$" ]);