gpg: Improve speed of secret key listing.

* agent/command.c (cmd_keyinfo): Factor some code out to ...
(get_keyinfo_on_cards): ... new.
(cmd_havekey): Add --list mode.
* g10/gpg.h (struct server_control_s): Add new caching vars.
* g10/gpg.c (gpg_deinit_default_ctrl): Release cache.
* g10/call-agent.c (agent_probe_any_secret_key): Init and try to use
the keygrip cache.
(agent_genkey): Clear the cache.
(agent_import_key): Ditto.

* g10/keylist.c (list_all, list_one): Pass ctrl to
agent_probe_any_secret_key.
* g10/getkey.c (lookup): Ditto.
--

With this change we first ask the agent for a list of all secret
keygrips and use that list instead of asking the agent for each public
key.  Speeds up my "gpg -K" with a lot of secret and public keys by
more than 25%.

Signed-off-by: Werner Koch <wk@gnupg.org>

5 files changed, 101 insertions, 19 deletions

diff --git a/g10/call-agent.c b/g10/call-agent.c
index 83355454a..782631c0d 100644
--- a/g10/call-agent.c
+++ b/g10/call-agent.c
@@ -2236,13 +2236,50 @@ agent_probe_any_secret_key (ctrl_t ctrl, kbnode_t keyblock)
   char line[ASSUAN_LINELENGTH];
   char *p;
   kbnode_t kbctx, node;
-  int nkeys;
+  int nkeys;  /* (always zero in secret_keygrips mode)  */
   unsigned char grip[KEYGRIP_LEN];
+  const unsigned char *s;
+  unsigned int n;
 
   err = start_agent (ctrl, 0);
   if (err)
     return err;
 
+  /* If we have not yet issued a "HAVEKEY --list" do that now.  We use
+   * a more or less arbitray limit of 1000 keys.  */
+  if (ctrl && !ctrl->secret_keygrips && !ctrl->no_more_secret_keygrips)
+    {
+      membuf_t data;
+
+      init_membuf (&data, 4096);
+      err = assuan_transact (agent_ctx, "HAVEKEY --list=1000",
+                             put_membuf_cb, &data,
+                             NULL, NULL, NULL, NULL);
+      if (err)
+        xfree (get_membuf (&data, NULL));
+      else
+        {
+          ctrl->secret_keygrips = get_membuf (&data,
+                                              &ctrl->secret_keygrips_len);
+          if (!ctrl->secret_keygrips)
+            err = gpg_error_from_syserror ();
+          if ((ctrl->secret_keygrips_len % 20))
+            {
+              err = gpg_error (GPG_ERR_INV_DATA);
+              xfree (ctrl->secret_keygrips);
+              ctrl->secret_keygrips = NULL;
+            }
+        }
+      if (err)
+        {
+          log_info ("problem with fast path key listing: %s - ignored\n",
+                    gpg_strerror (err));
+          err = 0;
+        }
+      /* We want to do this only once.  */
+      ctrl->no_more_secret_keygrips = 1;
+    }
+
   err = gpg_error (GPG_ERR_NO_SECKEY); /* Just in case no key was
                                           found in KEYBLOCK.  */
   p = stpcpy (line, "HAVEKEY");
@@ -2252,23 +2289,42 @@ agent_probe_any_secret_key (ctrl_t ctrl, kbnode_t keyblock)
         || node->pkt->pkttype == PKT_SECRET_KEY
         || node->pkt->pkttype == PKT_SECRET_SUBKEY)
       {
-        if (nkeys && ((p - line) + 41) > (ASSUAN_LINELENGTH - 2))
+        if (ctrl && ctrl->secret_keygrips)
           {
-            err = assuan_transact (agent_ctx, line,
-                                   NULL, NULL, NULL, NULL, NULL, NULL);
-            if (err != gpg_err_code (GPG_ERR_NO_SECKEY))
-              break; /* Seckey available or unexpected error - ready.  */
-            p = stpcpy (line, "HAVEKEY");
-            nkeys = 0;
+            /* We got an array with all secret keygrips.  Check this.  */
+            err = keygrip_from_pk (node->pkt->pkt.public_key, grip);
+            if (err)
+              return err;
+            for (s=ctrl->secret_keygrips, n = 0;
+                 n < ctrl->secret_keygrips_len;
+                 s += 20, n += 20)
+              {
+                if (!memcmp (s, grip, 20))
+                  return 0;
+              }
+            err = gpg_error (GPG_ERR_NO_SECKEY);
+            /* Keep on looping over the keyblock.  Never bump nkeys.  */
+          }
+        else
+          {
+            if (nkeys && ((p - line) + 41) > (ASSUAN_LINELENGTH - 2))
+              {
+                err = assuan_transact (agent_ctx, line,
+                                       NULL, NULL, NULL, NULL, NULL, NULL);
+                if (err != gpg_err_code (GPG_ERR_NO_SECKEY))
+                  break; /* Seckey available or unexpected error - ready.  */
+                p = stpcpy (line, "HAVEKEY");
+                nkeys = 0;
+              }
+
+            err = keygrip_from_pk (node->pkt->pkt.public_key, grip);
+            if (err)
+              return err;
+            *p++ = ' ';
+            bin2hex (grip, 20, p);
+            p += 40;
+            nkeys++;
           }
-
-        err = keygrip_from_pk (node->pkt->pkt.public_key, grip);
-        if (err)
-          return err;
-        *p++ = ' ';
-        bin2hex (grip, 20, p);
-        p += 40;
-        nkeys++;
       }
 
   if (!err && nkeys)
@@ -2419,6 +2475,14 @@ agent_genkey (ctrl_t ctrl, char **cache_nonce_addr, char **passwd_nonce_addr,
     return err;
   dfltparm.ctx = agent_ctx;
 
+  /* Do not use our cache of secret keygrips anymore - this command
+   * would otherwise requiring to update that cache.  */
+  if (ctrl && ctrl->secret_keygrips)
+    {
+      xfree (ctrl->secret_keygrips);
+      ctrl->secret_keygrips = 0;
+    }
+
   if (timestamp)
     {
       strcpy (timestamparg, " --timestamp=");
@@ -2876,6 +2940,14 @@ agent_import_key (ctrl_t ctrl, const char *desc, char **cache_nonce_addr,
     return err;
   dfltparm.ctx = agent_ctx;
 
+  /* Do not use our cache of secret keygrips anymore - this command
+   * would otherwise requiring to update that cache.  */
+  if (ctrl && ctrl->secret_keygrips)
+    {
+      xfree (ctrl->secret_keygrips);
+      ctrl->secret_keygrips = 0;
+    }
+
   if (timestamp)
     {
       strcpy (timestamparg, " --timestamp=");
diff --git a/g10/getkey.c b/g10/getkey.c
index c0f8b7c54..cf734db24 100644
--- a/g10/getkey.c
+++ b/g10/getkey.c
@@ -3916,7 +3916,7 @@ lookup (ctrl_t ctrl, getkey_ctx_t ctx, int want_secret,
 
       if (want_secret)
 	{
-	  rc = agent_probe_any_secret_key (NULL, keyblock);
+	  rc = agent_probe_any_secret_key (ctrl, keyblock);
 	  if (gpg_err_code(rc) == GPG_ERR_NO_SECKEY)
 	    goto skip; /* No secret key available.  */
 	  if (rc)
diff --git a/g10/gpg.c b/g10/gpg.c
index f5623be76..9787ca15f 100644
--- a/g10/gpg.c
+++ b/g10/gpg.c
@@ -2314,6 +2314,8 @@ gpg_deinit_default_ctrl (ctrl_t ctrl)
 
   keydb_release (ctrl->cached_getkey_kdb);
   gpg_keyboxd_deinit_session_data (ctrl);
+  xfree (ctrl->secret_keygrips);
+  ctrl->secret_keygrips = NULL;
 }
 
 
diff --git a/g10/gpg.h b/g10/gpg.h
index d4e66e72d..c51bbbb46 100644
--- a/g10/gpg.h
+++ b/g10/gpg.h
@@ -110,6 +110,14 @@ struct server_control_s
 
   /* This is used to cache a key data base handle.  */
   KEYDB_HANDLE cached_getkey_kdb;
+
+  /* Cached results from HAVEKEY --list.  They are used if the pointer
+   * is not NULL.  The length gives the length in bytes and is a
+   * multiple of 20.  If the no_more flag is set the list shall not
+   * anymore be refreshed even if it has been freed and NULLed. */
+  unsigned char *secret_keygrips;
+  size_t secret_keygrips_len;
+  int no_more_secret_keygrips;
 };
 
 
diff --git a/g10/keylist.c b/g10/keylist.c
index e222259ac..de651d671 100644
--- a/g10/keylist.c
+++ b/g10/keylist.c
@@ -553,7 +553,7 @@ list_all (ctrl_t ctrl, int secret, int mark_secret)
 	}
 
       if (secret || mark_secret)
-        any_secret = !agent_probe_any_secret_key (NULL, keyblock);
+        any_secret = !agent_probe_any_secret_key (ctrl, keyblock);
       else
         any_secret = 0;
 
@@ -645,7 +645,7 @@ list_one (ctrl_t ctrl, strlist_t names, int secret, int mark_secret)
       if (secret)
         any_secret = 1;
       else if (mark_secret)
-        any_secret = !agent_probe_any_secret_key (NULL, keyblock);
+        any_secret = !agent_probe_any_secret_key (ctrl, keyblock);
       else
         any_secret = 0;