common: Add support for the new extended private key format.

* agent/findkey.c (write_extended_private_key): New function.
(agent_write_private_key): Detect if an existing file is in extended
format and update the key within if it is.
(read_key_file): Handle the new format.
* agent/keyformat.txt: Document the new format.
* common/Makefile.am: Add the new files.
* common/private-keys.c: New file.
* common/private-keys.h: Likewise.
* common/t-private-keys.c: Likewise.
* common/util.h (alphap, alnump): New macros.
* tests/migrations: Add test demonstrating that we can cope with the
new format.

--
GnuPG 2.3+ will use a new format to store private keys that is both
more flexible and easier to read and edit by human beings.  The new
format stores name,value-pairs using the common mail and http header
convention.

This patch adds the parser and support code and prepares GnuPG 2.1 for
the new format.

Signed-off-by: Justus Winter <justus@g10code.com>

2 files changed, 232 insertions, 9 deletions

diff --git a/agent/findkey.c b/agent/findkey.c
index 3cf8d0cc1..a78709cc2 100644
--- a/agent/findkey.c
+++ b/agent/findkey.c
@@ -35,6 +35,7 @@
 #include "agent.h"
 #include "i18n.h"
 #include "../common/ssh-utils.h"
+#include "../common/private-keys.h"
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -51,6 +52,75 @@ struct try_unprotect_arg_s
 };
 
 
+static gpg_error_t
+write_extended_private_key (char *fname, estream_t fp,
+                            const void *buf, size_t len)
+{
+  gpg_error_t err;
+  pkc_t pk = NULL;
+  gcry_sexp_t key = NULL;
+  int remove = 0;
+  int line;
+
+  err = pkc_parse (&pk, &line, fp);
+  if (err)
+    {
+      log_error ("error parsing '%s' line %d: %s\n",
+                 fname, line, gpg_strerror (err));
+      goto leave;
+    }
+
+  err = gcry_sexp_sscan (&key, NULL, buf, len);
+  if (err)
+    goto leave;
+
+  err = pkc_set_private_key (pk, key);
+  if (err)
+    goto leave;
+
+  err = es_fseek (fp, 0, SEEK_SET);
+  if (err)
+    goto leave;
+
+  err = pkc_write (pk, fp);
+  if (err)
+    {
+      log_error ("error writing '%s': %s\n", fname, gpg_strerror (err));
+      remove = 1;
+      goto leave;
+    }
+
+  if (ftruncate (es_fileno (fp), es_ftello (fp)))
+    {
+      err = gpg_error_from_syserror ();
+      log_error ("error truncating '%s': %s\n", fname, gpg_strerror (err));
+      remove = 1;
+      goto leave;
+    }
+
+  if (es_fclose (fp))
+    {
+      err = gpg_error_from_syserror ();
+      log_error ("error closing '%s': %s\n", fname, gpg_strerror (err));
+      remove = 1;
+      goto leave;
+    }
+  else
+    fp = NULL;
+
+  bump_key_eventcounter ();
+
+ leave:
+  if (fp)
+    es_fclose (fp);
+  if (remove)
+    gnupg_remove (fname);
+  xfree (fname);
+  gcry_sexp_release (key);
+  pkc_release (pk);
+  return err;
+}
+
 /* Write an S-expression formatted key to our key storage.  With FORCE
    passed as true an existing key with the given GRIP will get
    overwritten.  */
@@ -77,7 +147,7 @@ agent_write_private_key (const unsigned char *grip,
       return gpg_error (GPG_ERR_EEXIST);
     }
 
-  fp = es_fopen (fname, force? "wb,mode=-rw" : "wbx,mode=-rw");
+  fp = es_fopen (fname, force? "rb+,mode=-rw" : "wbx,mode=-rw");
   if (!fp)
     {
       gpg_error_t tmperr = gpg_error_from_syserror ();
@@ -86,6 +156,38 @@ agent_write_private_key (const unsigned char *grip,
       return tmperr;
     }
 
+  /* See if an existing key is in extended format.  */
+  if (force)
+    {
+      gpg_error_t rc;
+      char first;
+
+      if (es_fread (&first, 1, 1, fp) != 1)
+        {
+          rc = gpg_error_from_syserror ();
+          log_error ("error reading first byte from '%s': %s\n",
+                     fname, strerror (errno));
+          xfree (fname);
+          es_fclose (fp);
+          return rc;
+        }
+
+      rc = es_fseek (fp, 0, SEEK_SET);
+      if (rc)
+        {
+          log_error ("error seeking in '%s': %s\n", fname, strerror (errno));
+          xfree (fname);
+          es_fclose (fp);
+          return rc;
+        }
+
+      if (first != '(')
+        {
+          /* Key is in extended format.  */
+          return write_extended_private_key (fname, fp, buffer, length);
+        }
+    }
+
   if (es_fwrite (buffer, length, 1, fp) != 1)
     {
       gpg_error_t tmperr = gpg_error_from_syserror ();
@@ -95,6 +197,18 @@ agent_write_private_key (const unsigned char *grip,
       xfree (fname);
       return tmperr;
     }
+
+  /* When force is given, the file might have to be truncated.  */
+  if (force && ftruncate (es_fileno (fp), es_ftello (fp)))
+    {
+      gpg_error_t tmperr = gpg_error_from_syserror ();
+      log_error ("error truncating '%s': %s\n", fname, gpg_strerror (tmperr));
+      es_fclose (fp);
+      gnupg_remove (fname);
+      xfree (fname);
+      return tmperr;
+    }
+
   if (es_fclose (fp))
     {
       gpg_error_t tmperr = gpg_error_from_syserror ();
@@ -531,6 +645,7 @@ read_key_file (const unsigned char *grip, gcry_sexp_t *result)
   size_t buflen, erroff;
   gcry_sexp_t s_skey;
   char hexgrip[40+4+1];
+  char first;
 
   *result = NULL;
 
@@ -548,6 +663,50 @@ read_key_file (const unsigned char *grip, gcry_sexp_t *result)
       return rc;
     }
 
+  if (es_fread (&first, 1, 1, fp) != 1)
+    {
+      rc = gpg_error_from_syserror ();
+      log_error ("error reading first byte from '%s': %s\n",
+                 fname, strerror (errno));
+      xfree (fname);
+      es_fclose (fp);
+      return rc;
+    }
+
+  rc = es_fseek (fp, 0, SEEK_SET);
+  if (rc)
+    {
+      log_error ("error seeking in '%s': %s\n", fname, strerror (errno));
+      xfree (fname);
+      es_fclose (fp);
+      return rc;
+    }
+
+  if (first != '(')
+    {
+      /* Key is in extended format.  */
+      pkc_t pk;
+      int line;
+
+      rc = pkc_parse (&pk, &line, fp);
+      es_fclose (fp);
+
+      if (rc)
+        log_error ("error parsing '%s' line %d: %s\n",
+                   fname, line, gpg_strerror (rc));
+      else
+        {
+          rc = pkc_get_private_key (pk, result);
+          pkc_release (pk);
+          if (rc)
+            log_error ("error getting private key from '%s': %s\n",
+                       fname, gpg_strerror (rc));
+        }
+
+      xfree (fname);
+      return rc;
+    }
+
   if (fstat (es_fileno (fp), &st))
     {
       rc = gpg_error_from_syserror ();
diff --git a/agent/keyformat.txt b/agent/keyformat.txt
index 5e15ecf03..c1a59ce15 100644
--- a/agent/keyformat.txt
+++ b/agent/keyformat.txt
@@ -16,7 +16,71 @@ and should have permissions 700.
 The secret keys are stored in files with a name matching the
 hexadecimal representation of the keygrip[2] and suffixed with ".key".
 
-* Unprotected Private Key Format
+* Extended Private Key Format
+
+GnuPG 2.3+ will use a new format to store private keys that is both
+more flexible and easier to read and edit by human beings.  The new
+format stores name,value-pairs using the common mail and http header
+convention.  Example (here indented with two spaces):
+
+  Description: Key to sign all GnuPG released tarballs.
+    The key is actually stored on a smart card.
+  Use-for-ssh: yes
+  OpenSSH-cert: long base64 encoded string wrapped so that this
+    key file can be easily edited with a standard editor.
+  Key: (shadowed-private-key
+    (rsa
+    (n #00AA1AD2A55FD8C8FDE9E1941772D9CC903FA43B268CB1B5A1BAFDC900
+    2961D8AEA153424DC851EF13B83AC64FBE365C59DC1BD3E83017C90D4365B4
+    83E02859FC13DB5842A00E969480DB96CE6F7D1C03600392B8E08EF0C01FC7
+    19F9F9086B25AD39B4F1C2A2DF3E2BE317110CFFF21D4A11455508FE407997
+    601260816C8422297C0637BB291C3A079B9CB38A92CE9E551F80AA0EBF4F0E
+    72C3F250461E4D31F23A7087857FC8438324A013634563D34EFDDCBF2EA80D
+    F9662C9CCD4BEF2522D8BDFED24CEF78DC6B309317407EAC576D889F88ADA0
+    8C4FFB480981FB68C5C6CA27503381D41018E6CDC52AAAE46B166BDC10637A
+    E186A02BA2497FDC5D1221#)
+    (e #00010001#)
+    (shadowed t1-v1
+     (#D2760001240102000005000011730000# OPENPGP.1)
+    )))
+
+GnuPG 2.2 is able to read and update keys using the new format, but
+will not create new files using the new format.  Furthermore, it only
+makes use of the value stored under the name 'Key:'.
+
+Keys in the extended format can be recognized by looking at the first
+byte of the file.  If it starts with a '(' it is a naked S-expression,
+otherwise it is a key in extended format.
+
+** Names
+
+A name must start with a letter and end with a colon.  Valid
+characters are all ASCII letters, numbers and the hyphen.  Comparison
+of names is done case insensitively.  Names may be used several times
+to represent an array of values.
+
+The name "Key:" is special in that it may occur only once and the
+associated value holds the actual S-expression with the cryptographic
+key.  The S-expression is formatted using the 'Advanced Format'
+(GCRYSEXP_FMT_ADVANCED) that avoids non-printable characters so that
+the file can be easily inspected and edited.  See section 'Private Key
+Format' below for details.
+
+** Values
+
+Values are UTF-8 encoded strings.  Values can be wrapped at any point,
+and continued in the next line indicated by leading whitespace.  A
+continuation line with one leading space does not introduce a blank so
+that the lines can be effectively concatenated.  A blank line as part
+of a continuation line encodes a newline.
+
+** Comments
+
+Lines containing only whitespace, and lines starting with whitespace
+followed by '#' are considered to be comments and are ignored.
+
+* Private Key Format
+** Unprotected Private Key Format
 
 The content of the file is an S-Expression like the ones used with
 Libgcrypt.  Here is an example of an unprotected file:
@@ -42,7 +106,7 @@ optional but required for some operations to calculate the fingerprint
 of the key.  This timestamp should be a string with the number of
 seconds since Epoch or an ISO time string (yyyymmddThhmmss).
 
-* Protected Private Key Format
+** Protected Private Key Format
 
 A protected key is like this:
 
@@ -67,7 +131,7 @@ optional; the isotimestamp is 15 bytes long (e.g. "19610711T172000").
 
 The currently defined protection modes are:
 
-** openpgp-s2k3-sha1-aes-cbc
+*** openpgp-s2k3-sha1-aes-cbc
 
   This describes an algorithm using using AES in CBC mode for
   encryption, SHA-1 for integrity protection and the String to Key
@@ -116,7 +180,7 @@ The currently defined protection modes are:
   the stored one - If they don't match the integrity of the key is not
   given.
 
-** openpgp-s2k3-ocb-aes
+*** openpgp-s2k3-ocb-aes
 
   This describes an algorithm using using AES-128 in OCB mode, a nonce
   of 96 bit, a taglen of 128 bit, and the String to Key algorithm 3
@@ -154,7 +218,7 @@ The currently defined protection modes are:
    (protected-at "18950523T000000")
   )
 
-** openpgp-native
+*** openpgp-native
 
   This is a wrapper around the OpenPGP Private Key Transport format
   which resembles the standard OpenPGP format and allows the use of an
@@ -191,7 +255,7 @@ The currently defined protection modes are:
    (uri http://foo.bar x-foo:whatever_you_want)
    (comment whatever))
 
-* Shadowed Private Key Format
+** Shadowed Private Key Format
 
 To keep track of keys stored on IC cards we use a third format for
 private kyes which are called shadow keys as they are only a reference
@@ -219,7 +283,7 @@ readers don't allow passing a variable length PIN.
 
 More items may be added to the list.
 
-* OpenPGP Private Key Transfer Format
+** OpenPGP Private Key Transfer Format
 
 This format is used to transfer keys between gpg and gpg-agent.
 
@@ -251,7 +315,7 @@ This format is used to transfer keys between gpg and gpg-agent.
  * S2KSALT is the 8 byte salt
  * S2KCOUNT is the count value from RFC-4880.
 
-* Persistent Passphrase Format
+** Persistent Passphrase Format
 
 Note: That this has not yet been implemented.