wkd: New command --mirror for gpg-wks-client.

* tools/gpg-wks-client.c (aMirror,oBlacklist,oNoAutostart): New.
(opts): Add ----mirror, --no-autostart, and --blacklist.
(parse_arguments): Parse new options.
(main): Parse common.conf.  Implement aMirror.
(mirror_one_key_parm): New.
(mirror_one_keys_userid, mirror_one_key): New.
(command_mirror): New.

* tools/gpg-wks.h (struct uidinfo_list_s): Add fields flags.
* tools/wks-util.c (wks_cmd_install_key): Factor some code out to ...
(wks_install_key_core): new.

* tools/call-dirmngr.c (wkd_dirmngr_ks_get): New.
--

This implements the basic LDAP to WKD mirroring.  The blacklist
option and domain restrictions are not yet fully implemented.

Take care: In OpenLDAP you may need to increase the paged result limit
by using a configuration like:

  dn: olcDatabase={1}mdb,cn=config
  changetype: modify
  replace: olcLimits
  olcLimits: dn.subtree="dc=example,dc=org" size.prtotal=unlimited

GnuPG-bug-id: 6224

5 files changed, 316 insertions, 30 deletions

diff --git a/tools/call-dirmngr.c b/tools/call-dirmngr.c
index c21990533..4eef9b264 100644
--- a/tools/call-dirmngr.c
+++ b/tools/call-dirmngr.c
@@ -1,5 +1,5 @@
 /* call-dirmngr.c - Interact with the Dirmngr.
- * Copyright (C) 2016 g10 Code GmbH
+ * Copyright (C) 2016, 2022 g10 Code GmbH
  * Copyright (C) 2016 Bundesamt für Sicherheit in der Informationstechnik
  *
  * This file is part of GnuPG.
@@ -311,3 +311,71 @@ wkd_get_key (const char *addrspec, estream_t *r_key)
   assuan_release (ctx);
   return err;
 }
+
+
+/* Send the KS_GET command to the dirmngr.  The caller provides CB
+ * which is called for each key.  The callback is called wit a stream
+ * conveying a single key and several other informational parameters.
+ * DOMAIN restricts the returned keys to this domain.  */
+gpg_error_t
+wkd_dirmngr_ks_get (const char *domain, gpg_error_t cb (estream_t key))
+{
+  gpg_error_t err;
+  assuan_context_t ctx;
+  struct wkd_get_parm_s parm;
+  char *line = NULL;
+  int any = 0;
+
+  memset (&parm, 0, sizeof parm);
+
+  err = connect_dirmngr (&ctx);
+  if (err)
+    return err;
+
+  line = es_bsprintf ("KS_GET --ldap --first %s", domain? domain:"");
+  if (!line)
+    {
+      err = gpg_error_from_syserror ();
+      goto leave;
+    }
+  if (strlen (line) + 2 >= ASSUAN_LINELENGTH)
+    {
+      err = gpg_error (GPG_ERR_TOO_LARGE);
+      goto leave;
+    }
+
+  parm.memfp = es_fopenmem (0, "rwb");
+  if (!parm.memfp)
+    {
+      err = gpg_error_from_syserror ();
+      goto leave;
+    }
+
+  for (;;)
+    {
+      err = assuan_transact (ctx, any? "KS_GET --next" : line,
+                             wkd_get_data_cb, &parm,
+                             NULL, NULL, wkd_get_status_cb, &parm);
+      if (err)
+        {
+          if (gpg_err_code (err) == GPG_ERR_NO_DATA
+              && gpg_err_source (err) == GPG_ERR_SOURCE_DIRMNGR)
+            err = any? 0 : gpg_error (GPG_ERR_NOT_FOUND);
+          goto leave;
+        }
+      any = 1;
+
+      es_rewind (parm.memfp);
+      err = cb (parm.memfp);
+      if (err)
+        break;
+      es_ftruncate (parm.memfp, 0);
+    }
+
+
+ leave:
+  es_fclose (parm.memfp);
+  xfree (line);
+  assuan_release (ctx);
+  return err;
+}
diff --git a/tools/call-dirmngr.h b/tools/call-dirmngr.h
index 4da0145e7..3acea513d 100644
--- a/tools/call-dirmngr.h
+++ b/tools/call-dirmngr.h
@@ -28,5 +28,8 @@ gpg_error_t wkd_get_policy_flags (const char *addrspec, estream_t *r_buffer);
 
 gpg_error_t wkd_get_key (const char *addrspec, estream_t *r_key);
 
+gpg_error_t wkd_dirmngr_ks_get (const char *domain,
+                                gpg_error_t cb (estream_t key));
+
 
 #endif /*GNUPG_TOOLS_CALL_DIRMNGR_H*/
diff --git a/tools/gpg-wks-client.c b/tools/gpg-wks-client.c
index b56343232..c90e86373 100644
--- a/tools/gpg-wks-client.c
+++ b/tools/gpg-wks-client.c
@@ -1,5 +1,5 @@
 /* gpg-wks-client.c - A client for the Web Key Service protocols.
- * Copyright (C) 2016 Werner Koch
+ * Copyright (C) 2016, 2022 g10 Code GmbH
  * Copyright (C) 2016 Bundesamt für Sicherheit in der Informationstechnik
  *
  * This file is part of GnuPG.
@@ -39,6 +39,7 @@
 #include "../common/exectool.h"
 #include "../common/mbox-util.h"
 #include "../common/name-value.h"
+#include "../common/comopt.h"
 #include "call-dirmngr.h"
 #include "mime-maker.h"
 #include "send-mail.h"
@@ -62,6 +63,7 @@ enum cmd_and_opt_values
     aCreate,
     aReceive,
     aRead,
+    aMirror,
     aInstallKey,
     aRemoveKey,
     aPrintWKDHash,
@@ -72,6 +74,8 @@ enum cmd_and_opt_values
     oFakeSubmissionAddr,
     oStatusFD,
     oWithColons,
+    oBlacklist,
+    oNoAutostart,
 
     oDummy
   };
@@ -91,6 +95,8 @@ static gpgrt_opt_t opts[] = {
               ("receive a MIME confirmation request")),
   ARGPARSE_c (aRead,      "read",
               ("receive a plain text confirmation request")),
+  ARGPARSE_c (aMirror, "mirror",
+              "mirror an LDAP directory"),
   ARGPARSE_c (aInstallKey, "install-key",
               "install a key into a directory"),
   ARGPARSE_c (aRemoveKey, "remove-key",
@@ -109,7 +115,9 @@ static gpgrt_opt_t opts[] = {
   ARGPARSE_s_n (oSend, "send", "send the mail using sendmail"),
   ARGPARSE_s_s (oOutput, "output", "|FILE|write the mail to FILE"),
   ARGPARSE_s_i (oStatusFD, "status-fd", N_("|FD|write status info to this FD")),
+  ARGPARSE_s_n (oNoAutostart, "no-autostart", "@"),
   ARGPARSE_s_n (oWithColons, "with-colons", "@"),
+  ARGPARSE_s_s (oBlacklist, "blacklist", "@"),
   ARGPARSE_s_s (oDirectory, "directory", "@"),
 
   ARGPARSE_s_s (oFakeSubmissionAddr, "fake-submission-addr", "@"),
@@ -150,6 +158,7 @@ static gpg_error_t read_confirmation_request (estream_t msg);
 static gpg_error_t command_receive_cb (void *opaque,
                                        const char *mediatype, estream_t fp,
                                        unsigned int flags);
+static gpg_error_t command_mirror (const char *domain);
 
 
 
@@ -236,12 +245,19 @@ parse_arguments (gpgrt_argparse_t *pargs, gpgrt_opt_t *popts)
         case oWithColons:
           opt.with_colons = 1;
           break;
+        case oNoAutostart:
+          opt.no_autostart = 1;
+          break;
+        case oBlacklist:
+          opt.blacklist = pargs->r.ret_str;
+          break;
 
 	case aSupported:
 	case aCreate:
 	case aReceive:
 	case aRead:
         case aCheck:
+        case aMirror:
         case aInstallKey:
         case aRemoveKey:
         case aPrintWKDHash:
@@ -287,6 +303,15 @@ main (int argc, char **argv)
   if (log_get_errorcount (0))
     exit (2);
 
+  /* Process common component options.  Note that we set the config
+   * dir only here so that --homedir will have an effect.  */
+  gpgrt_set_confdir (GPGRT_CONFDIR_SYS, gnupg_sysconfdir ());
+  gpgrt_set_confdir (GPGRT_CONFDIR_USER, gnupg_homedir ());
+  if (parse_comopt (GNUPG_MODULE_NAME_CONNECT_AGENT, opt.verbose > 1))
+    exit(2);
+  if (comopt.no_autostart)
+     opt.no_autostart = 1;
+
   /* Print a warning if an argument looks like an option.  */
   if (!opt.quiet && !(pargs.flags & ARGPARSE_FLAG_STOP_SEEN))
     {
@@ -305,11 +330,12 @@ main (int argc, char **argv)
     opt.directory = "openpgpkey";
 
   /* Tell call-dirmngr what options we want.  */
-  set_dirmngr_options (opt.verbose, (opt.debug & DBG_IPC_VALUE), 1);
+  set_dirmngr_options (opt.verbose, (opt.debug & DBG_IPC_VALUE),
+                       !opt.no_autostart);
 
 
   /* Check that the top directory exists.  */
-  if (cmd == aInstallKey || cmd == aRemoveKey)
+  if (cmd == aInstallKey || cmd == aRemoveKey || cmd == aMirror)
     {
       struct stat sb;
 
@@ -379,6 +405,15 @@ main (int argc, char **argv)
       err = command_check (argv[0]);
       break;
 
+    case aMirror:
+      if (!argc)
+        err = command_mirror (NULL);
+      else if (argc == 1)
+        err = command_mirror (*argv);
+      else
+        wrong_args ("--mirror [DOMAIN]");
+      break;
+
     case aInstallKey:
       if (!argc)
         err = wks_cmd_install_key (NULL, NULL);
@@ -1594,3 +1629,166 @@ command_receive_cb (void *opaque, const char *mediatype,
 
   return err;
 }
+
+
+
+/* An object used to communicate with the mirror_one_key callback.  */
+struct
+{
+  const char *domain;
+  int anyerror;
+  unsigned int nkeys;   /* Number of keys processed.  */
+  unsigned int nuids;   /* Number of published user ids.  */
+} mirror_one_key_parm;
+
+
+/* Core of mirror_one_key with the goal of mirroring just one uid.
+ * UIDLIST is used to figure out whether the given MBOX occurs several
+ * times in UIDLIST and then to single out the newwest one.  This is
+ * so that for a key with
+ *    uid: Joe Someone <joe@example.org>
+ *    uid: Joe <joe@example.org>
+ * only the news user id (and thus its self-signature) is used.
+ * UIDLIST is nodified to set all MBOX fields to NULL for a processed
+ * user id.  FPR is the fingerprint of the key.
+ */
+static gpg_error_t
+mirror_one_keys_userid (estream_t key, const char *mbox, uidinfo_list_t uidlist,
+                        const char *fpr)
+{
+  gpg_error_t err;
+  uidinfo_list_t uid, thisuid, firstuid;
+  time_t thistime;
+  estream_t newkey = NULL;
+
+  /* Find the UID we want to use.  */
+  thistime = 0;
+  thisuid = firstuid = NULL;
+  for (uid = uidlist; uid; uid = uid->next)
+    {
+      if ((uid->flags & 1) || !uid->mbox || strcmp (uid->mbox, mbox))
+        continue; /* Already processed or no matching mbox.  */
+      uid->flags |= 1;  /* Set "processed" flag.  */
+      if (!firstuid)
+        firstuid = uid;
+      if (uid->created > thistime)
+        {
+          thistime = uid->created;
+          thisuid = uid;
+        }
+    }
+  if (!thisuid)
+    thisuid = firstuid;  /* This is the case for a missing timestamp.  */
+  if (!thisuid)
+    {
+      log_error ("error finding the user id for %s (%s)\n", fpr, mbox);
+      err = gpg_error (GPG_ERR_NO_USER_ID);
+      goto leave;
+    }
+  /* FIXME: Consult blacklist.  */
+
+
+  /* Only if we have more than one user id we bother to run the
+   * filter.  In this case the result will be put into NEWKEY*/
+  es_rewind (key);
+  if (uidlist->next)
+    {
+      err = wks_filter_uid (&newkey, key, thisuid->uid, 0);
+      if (err)
+        {
+          log_error ("error filtering key %s: %s\n", fpr, gpg_strerror (err));
+          err = gpg_error (GPG_ERR_NO_PUBKEY);
+          goto leave;
+        }
+    }
+
+  err = wks_install_key_core (newkey? newkey : key, mbox);
+  if (!opt.quiet)
+    log_info ("key %s published for '%s'\n", fpr, mbox);
+  mirror_one_key_parm.nuids++;
+  if (!opt.quiet && !(mirror_one_key_parm.nuids % 25))
+    log_info ("%u user ids from %d keys so far\n",
+              mirror_one_key_parm.nuids, mirror_one_key_parm.nkeys);
+
+ leave:
+  es_fclose (newkey);
+  return err;
+}
+
+
+/* The callback used by command_mirror.  It received an estream with
+ * one key and should return success to process the next key.  */
+static gpg_error_t
+mirror_one_key (estream_t key)
+{
+  gpg_error_t err = 0;
+  char *fpr;
+  uidinfo_list_t uidlist = NULL;
+  uidinfo_list_t uid;
+
+  /* List the key to get all user ids.  */
+  err = wks_list_key (key, &fpr, &uidlist);
+  if (err)
+    {
+      log_error ("error parsing a key: %s - skipped\n",
+                 gpg_strerror (err));
+      mirror_one_key_parm.anyerror = 1;
+      err = 0;
+      goto leave;
+    }
+  for (uid = uidlist; uid; uid = uid->next)
+    {
+      if (!uid->mbox || (uid->flags & 1))
+        continue; /* No mail box or already processed.  */
+      err = mirror_one_keys_userid (key, uid->mbox, uidlist, fpr);
+      if (err)
+        {
+          log_error ("error processing key %s: %s - skipped\n",
+                     fpr, gpg_strerror (err));
+          mirror_one_key_parm.anyerror = 1;
+          err = 0;
+          goto leave;
+        }
+    }
+  mirror_one_key_parm.nkeys++;
+
+
+ leave:
+  free_uidinfo_list (uidlist);
+  xfree (fpr);
+  return err;
+}
+
+
+/* Copy the keys from the configured LDAP server into a local WKD.
+ * DOMAIN is a domain name to restrict the copy to only this domain;
+ * if it is NULL all keys are mirrored.  */
+static gpg_error_t
+command_mirror (const char *domain)
+{
+  gpg_error_t err;
+
+  if (domain)
+    {
+      /* Fixme: Do some sanity checks on the domain.  */
+    }
+  mirror_one_key_parm.domain = domain;
+  mirror_one_key_parm.anyerror = 0;
+  mirror_one_key_parm.nkeys = 0;
+  mirror_one_key_parm.nuids = 0;
+
+  err = wkd_dirmngr_ks_get (domain, mirror_one_key);
+  if (!opt.quiet)
+    log_info ("a total of %u user ids from %d keys published\n",
+              mirror_one_key_parm.nuids, mirror_one_key_parm.nkeys);
+  if (err)
+    log_error ("error mirroring LDAP directory: %s <%s>\n",
+               gpg_strerror (err), gpg_strsource (err));
+  else if (mirror_one_key_parm.anyerror)
+    log_info ("warning: errors encountered - not all keys are mirrored\n");
+
+
+
+
+  return err;
+}
diff --git a/tools/gpg-wks.h b/tools/gpg-wks.h
index 6c5dc8b17..50350eddb 100644
--- a/tools/gpg-wks.h
+++ b/tools/gpg-wks.h
@@ -38,11 +38,13 @@ struct
   int quiet;
   int use_sendmail;
   int with_colons;
+  int no_autostart;
   const char *output;
   const char *gpg_program;
   const char *directory;
   const char *default_from;
   strlist_t extra_headers;
+  const char *blacklist;
 } opt;
 
 /* Debug values and macros.  */
@@ -78,6 +80,7 @@ struct uidinfo_list_s
   struct uidinfo_list_s *next;
   time_t created; /* Time the userid was created.  */
   char *mbox;  /* NULL or the malloced mailbox from UID.  */
+  unsigned int flags;  /* These flags are cleared on creation.  */
   char uid[1];
 };
 typedef struct uidinfo_list_s *uidinfo_list_t;
@@ -102,6 +105,7 @@ void wks_free_policy (policy_flags_t policy);
 gpg_error_t wks_fname_from_userid (const char *userid, int hash_only,
                                    char **r_fname, char **r_addrspec);
 gpg_error_t wks_compute_hu_fname (char **r_fname, const char *addrspec);
+gpg_error_t wks_install_key_core (estream_t key, const char *addrspec);
 gpg_error_t wks_cmd_install_key (const char *fname, const char *userid);
 gpg_error_t wks_cmd_remove_key (const char *userid);
 gpg_error_t wks_cmd_print_wkd_hash (const char *userid);
diff --git a/tools/wks-util.c b/tools/wks-util.c
index 3f8e8206d..e1d08b9ed 100644
--- a/tools/wks-util.c
+++ b/tools/wks-util.c
@@ -119,6 +119,7 @@ append_to_uidinfo_list (uidinfo_list_t *list, const char *uid, time_t created)
 
   strcpy (sl->uid, plainuid);
   sl->created = created;
+  sl->flags = 0;
   sl->mbox = mailbox_from_userid (plainuid, 0);
   sl->next = NULL;
   if (!*list)
@@ -1031,6 +1032,43 @@ install_key_from_spec_file (const char *fname)
 }
 
 
+/* The core of the code to install a key as a file.  */
+gpg_error_t
+wks_install_key_core (estream_t key, const char *addrspec)
+{
+  gpg_error_t err;
+  char *huname = NULL;
+
+  /* Hash user ID and create filename.  */
+  err = wks_compute_hu_fname (&huname, addrspec);
+  if (err)
+    goto leave;
+
+  /* Now that wks_compute_hu_fname has created missing directories we
+   * can create a policy file if it does not exist.  */
+  err = ensure_policy_file (addrspec);
+  if (err)
+    goto leave;
+
+  /* Publish.  */
+  err = write_to_file (key, huname);
+  if (err)
+    {
+      log_error ("copying key to '%s' failed: %s\n", huname,gpg_strerror (err));
+      goto leave;
+    }
+
+  /* Make sure it is world readable.  */
+  if (gnupg_chmod (huname, "-rw-r--r--"))
+    log_error ("can't set permissions of '%s': %s\n",
+               huname, gpg_strerror (gpg_err_code_from_syserror()));
+
+ leave:
+  xfree (huname);
+  return err;
+}
+
+
 /* Install a single key into the WKD by reading FNAME and extracting
  * USERID.  If USERID is NULL FNAME is expected to be a list of fpr
  * mbox lines and for each line the respective key will be
@@ -1046,7 +1084,6 @@ wks_cmd_install_key (const char *fname, const char *userid)
   uidinfo_list_t uidlist = NULL;
   uidinfo_list_t uid, thisuid;
   time_t thistime;
-  char *huname = NULL;
   int any;
 
   if (!userid)
@@ -1137,36 +1174,12 @@ wks_cmd_install_key (const char *fname, const char *userid)
     fp = fp2;
   }
 
-  /* Hash user ID and create filename.  */
-  err = wks_compute_hu_fname (&huname, addrspec);
-  if (err)
-    goto leave;
-
-  /* Now that wks_compute_hu_fname has created missing directories we
-   * can create a policy file if it does not exist.  */
-  err = ensure_policy_file (addrspec);
-  if (err)
-    goto leave;
-
-  /* Publish.  */
-  err = write_to_file (fp, huname);
-  if (err)
-    {
-      log_error ("copying key to '%s' failed: %s\n", huname,gpg_strerror (err));
-      goto leave;
-    }
-
-  /* Make sure it is world readable.  */
-  if (gnupg_chmod (huname, "-rw-r--r--"))
-    log_error ("can't set permissions of '%s': %s\n",
-               huname, gpg_strerror (gpg_err_code_from_syserror()));
-
+  err = wks_install_key_core (fp, addrspec);
   if (!opt.quiet)
     log_info ("key %s published for '%s'\n", fpr, addrspec);
 
 
  leave:
-  xfree (huname);
   free_uidinfo_list (uidlist);
   xfree (fpr);
   xfree (addrspec);