s390x assembly pack: add KIMD/KLMD code path for sha3/shake

Signed-off-by: Patrick Steuer <patrick.steuer@de.ibm.com>
Reviewed-by: Andy Polyakov <appro@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5935)

4 files changed, 275 insertions, 7 deletions

diff --git a/crypto/evp/build.info b/crypto/evp/build.info
index 0305738011..cc33ac3c49 100644
--- a/crypto/evp/build.info
+++ b/crypto/evp/build.info
@@ -22,3 +22,4 @@ INCLUDE[e_camellia.o]=.. ../modes
 INCLUDE[e_sm4.o]=.. ../modes
 INCLUDE[e_des.o]=..
 INCLUDE[e_des3.o]=..
+INCLUDE[m_sha3.o]=..
diff --git a/crypto/evp/m_sha3.c b/crypto/evp/m_sha3.c
index bfc65b22e4..729622b814 100644
--- a/crypto/evp/m_sha3.c
+++ b/crypto/evp/m_sha3.c
@@ -137,7 +137,227 @@ static int shake_ctrl(EVP_MD_CTX *evp_ctx, int cmd, int p1, void *p2)
     }
 }
 
-#define EVP_MD_SHA3(bitlen)                     \
+#if defined(OPENSSL_CPUID_OBJ) && defined(__s390__) && defined(KECCAK1600_ASM)
+/*
+ * IBM S390X support
+ */
+# include "s390x_arch.h"
+
+# define S390X_SHA3_FC(ctx)     ((ctx)->pad)
+
+# define S390X_sha3_224_CAPABLE ((OPENSSL_s390xcap_P.kimd[0] &      \
+                                  S390X_CAPBIT(S390X_SHA3_224)) &&  \
+                                 (OPENSSL_s390xcap_P.klmd[0] &      \
+                                  S390X_CAPBIT(S390X_SHA3_224)))
+# define S390X_sha3_256_CAPABLE ((OPENSSL_s390xcap_P.kimd[0] &      \
+                                  S390X_CAPBIT(S390X_SHA3_256)) &&  \
+                                 (OPENSSL_s390xcap_P.klmd[0] &      \
+                                  S390X_CAPBIT(S390X_SHA3_256)))
+# define S390X_sha3_384_CAPABLE ((OPENSSL_s390xcap_P.kimd[0] &      \
+                                  S390X_CAPBIT(S390X_SHA3_384)) &&  \
+                                 (OPENSSL_s390xcap_P.klmd[0] &      \
+                                  S390X_CAPBIT(S390X_SHA3_384)))
+# define S390X_sha3_512_CAPABLE ((OPENSSL_s390xcap_P.kimd[0] &      \
+                                  S390X_CAPBIT(S390X_SHA3_512)) &&  \
+                                 (OPENSSL_s390xcap_P.klmd[0] &      \
+                                  S390X_CAPBIT(S390X_SHA3_512)))
+# define S390X_shake128_CAPABLE ((OPENSSL_s390xcap_P.kimd[0] &      \
+                                  S390X_CAPBIT(S390X_SHAKE_128)) && \
+                                 (OPENSSL_s390xcap_P.klmd[0] &      \
+                                  S390X_CAPBIT(S390X_SHAKE_128)))
+# define S390X_shake256_CAPABLE ((OPENSSL_s390xcap_P.kimd[0] &      \
+                                  S390X_CAPBIT(S390X_SHAKE_256)) && \
+                                 (OPENSSL_s390xcap_P.klmd[0] &      \
+                                  S390X_CAPBIT(S390X_SHAKE_256)))
+
+/* Convert md-size to block-size. */
+# define S390X_KECCAK1600_BSZ(n) ((KECCAK1600_WIDTH - ((n) << 1)) >> 3)
+
+static int s390x_sha3_init(EVP_MD_CTX *evp_ctx)
+{
+    KECCAK1600_CTX *ctx = evp_ctx->md_data;
+    const size_t bsz = evp_ctx->digest->block_size;
+
+    /*-
+     * KECCAK1600_CTX structure's pad field is used to store the KIMD/KLMD
+     * function code.
+     */
+    switch (bsz) {
+    case S390X_KECCAK1600_BSZ(224):
+        ctx->pad = S390X_SHA3_224;
+        break;
+    case S390X_KECCAK1600_BSZ(256):
+        ctx->pad = S390X_SHA3_256;
+        break;
+    case S390X_KECCAK1600_BSZ(384):
+        ctx->pad = S390X_SHA3_384;
+        break;
+    case S390X_KECCAK1600_BSZ(512):
+        ctx->pad = S390X_SHA3_512;
+        break;
+    default:
+        return 0;
+    }
+
+    memset(ctx->A, 0, sizeof(ctx->A));
+    ctx->num = 0;
+    ctx->block_size = bsz;
+    ctx->md_size = evp_ctx->digest->md_size;
+    return 1;
+}
+
+static int s390x_shake_init(EVP_MD_CTX *evp_ctx)
+{
+    KECCAK1600_CTX *ctx = evp_ctx->md_data;
+    const size_t bsz = evp_ctx->digest->block_size;
+
+    /*-
+     * KECCAK1600_CTX structure's pad field is used to store the KIMD/KLMD
+     * function code.
+     */
+    switch (bsz) {
+    case S390X_KECCAK1600_BSZ(128):
+        ctx->pad = S390X_SHAKE_128;
+        break;
+    case S390X_KECCAK1600_BSZ(256):
+        ctx->pad = S390X_SHAKE_256;
+        break;
+    default:
+        return 0;
+    }
+
+    memset(ctx->A, 0, sizeof(ctx->A));
+    ctx->num = 0;
+    ctx->block_size = bsz;
+    ctx->md_size = evp_ctx->digest->md_size;
+    return 1;
+}
+
+static int s390x_sha3_update(EVP_MD_CTX *evp_ctx, const void *_inp, size_t len)
+{
+    KECCAK1600_CTX *ctx = evp_ctx->md_data;
+    const unsigned char *inp = _inp;
+    const size_t bsz = ctx->block_size;
+    size_t num, rem;
+
+    if (len == 0)
+        return 1;
+
+    if ((num = ctx->num) != 0) {
+        rem = bsz - num;
+
+        if (len < rem) {
+            memcpy(ctx->buf + num, inp, len);
+            ctx->num += len;
+            return 1;
+        }
+        memcpy(ctx->buf + num, inp, rem);
+        inp += rem;
+        len -= rem;
+        s390x_kimd(ctx->buf, bsz, ctx->pad, ctx->A);
+        ctx->num = 0;
+    }
+    rem = len % bsz;
+
+    s390x_kimd(inp, len - rem, ctx->pad, ctx->A);
+
+    if (rem) {
+        memcpy(ctx->buf, inp + len - rem, rem);
+        ctx->num = rem;
+    }
+    return 1;
+}
+
+static int s390x_sha3_final(EVP_MD_CTX *evp_ctx, unsigned char *md)
+{
+    KECCAK1600_CTX *ctx = evp_ctx->md_data;
+
+    s390x_klmd(ctx->buf, ctx->num, NULL, 0, ctx->pad, ctx->A);
+    memcpy(md, ctx->A, ctx->md_size);
+    return 1;
+}
+
+static int s390x_shake_final(EVP_MD_CTX *evp_ctx, unsigned char *md)
+{
+    KECCAK1600_CTX *ctx = evp_ctx->md_data;
+
+    s390x_klmd(ctx->buf, ctx->num, md, ctx->md_size, ctx->pad, ctx->A);
+    return 1;
+}
+
+# define EVP_MD_SHA3(bitlen)                         \
+const EVP_MD *EVP_sha3_##bitlen(void)                \
+{                                                    \
+    static const EVP_MD s390x_sha3_##bitlen##_md = { \
+        NID_sha3_##bitlen,                           \
+        NID_RSA_SHA3_##bitlen,                       \
+        bitlen / 8,                                  \
+        EVP_MD_FLAG_DIGALGID_ABSENT,                 \
+        s390x_sha3_init,                             \
+        s390x_sha3_update,                           \
+        s390x_sha3_final,                            \
+        NULL,                                        \
+        NULL,                                        \
+        (KECCAK1600_WIDTH - bitlen * 2) / 8,         \
+        sizeof(KECCAK1600_CTX),                      \
+    };                                               \
+    static const EVP_MD sha3_##bitlen##_md = {       \
+        NID_sha3_##bitlen,                           \
+        NID_RSA_SHA3_##bitlen,                       \
+        bitlen / 8,                                  \
+        EVP_MD_FLAG_DIGALGID_ABSENT,                 \
+        sha3_init,                                   \
+        sha3_update,                                 \
+        sha3_final,                                  \
+        NULL,                                        \
+        NULL,                                        \
+        (KECCAK1600_WIDTH - bitlen * 2) / 8,         \
+        sizeof(KECCAK1600_CTX),                      \
+    };                                               \
+    return S390X_sha3_##bitlen##_CAPABLE ?           \
+           &s390x_sha3_##bitlen##_md :               \
+           &sha3_##bitlen##_md;                      \
+}
+
+# define EVP_MD_SHAKE(bitlen)                        \
+const EVP_MD *EVP_shake##bitlen(void)                \
+{                                                    \
+    static const EVP_MD s390x_shake##bitlen##_md = { \
+        NID_shake##bitlen,                           \
+        0,                                           \
+        bitlen / 8,                                  \
+        EVP_MD_FLAG_XOF,                             \
+        s390x_shake_init,                            \
+        s390x_sha3_update,                           \
+        s390x_shake_final,                           \
+        NULL,                                        \
+        NULL,                                        \
+        (KECCAK1600_WIDTH - bitlen * 2) / 8,         \
+        sizeof(KECCAK1600_CTX),                      \
+        shake_ctrl                                   \
+    };                                               \
+    static const EVP_MD shake##bitlen##_md = {       \
+        NID_shake##bitlen,                           \
+        0,                                           \
+        bitlen / 8,                                  \
+        EVP_MD_FLAG_XOF,                             \
+        shake_init,                                  \
+        sha3_update,                                 \
+        sha3_final,                                  \
+        NULL,                                        \
+        NULL,                                        \
+        (KECCAK1600_WIDTH - bitlen * 2) / 8,         \
+        sizeof(KECCAK1600_CTX),                      \
+        shake_ctrl                                   \
+    };                                               \
+    return S390X_shake##bitlen##_CAPABLE ?           \
+           &s390x_shake##bitlen##_md :               \
+           &shake##bitlen##_md;                      \
+}
+
+#else
+
+# define EVP_MD_SHA3(bitlen)                    \
 const EVP_MD *EVP_sha3_##bitlen(void)           \
 {                                               \
     static const EVP_MD sha3_##bitlen##_md = {  \
@@ -156,12 +376,7 @@ const EVP_MD *EVP_sha3_##bitlen(void)           \
     return &sha3_##bitlen##_md;                 \
 }
 
-EVP_MD_SHA3(224)
-EVP_MD_SHA3(256)
-EVP_MD_SHA3(384)
-EVP_MD_SHA3(512)
-
-#define EVP_MD_SHAKE(bitlen)                    \
+# define EVP_MD_SHAKE(bitlen)                   \
 const EVP_MD *EVP_shake##bitlen(void)           \
 {                                               \
     static const EVP_MD shake##bitlen##_md = {  \
@@ -180,6 +395,12 @@ const EVP_MD *EVP_shake##bitlen(void)           \
     };                                          \
     return &shake##bitlen##_md;                 \
 }
+#endif
+
+EVP_MD_SHA3(224)
+EVP_MD_SHA3(256)
+EVP_MD_SHA3(384)
+EVP_MD_SHA3(512)
 
 EVP_MD_SHAKE(128)
 EVP_MD_SHAKE(256)
diff --git a/crypto/s390x_arch.h b/crypto/s390x_arch.h
index 5042154470..4a775a927d 100644
--- a/crypto/s390x_arch.h
+++ b/crypto/s390x_arch.h
@@ -12,6 +12,10 @@
 
 # ifndef __ASSEMBLER__
 
+void s390x_kimd(const unsigned char *in, size_t len, unsigned int fc,
+                void *param);
+void s390x_klmd(const unsigned char *in, size_t inlen, unsigned char *out,
+                size_t outlen, unsigned int fc, void *param);
 void s390x_km(const unsigned char *in, size_t len, unsigned char *out,
               unsigned int fc, void *param);
 void s390x_kmac(const unsigned char *in, size_t len, unsigned int fc,
diff --git a/crypto/s390xcpuid.pl b/crypto/s390xcpuid.pl
index e7afb8dcf5..ec700a47d9 100755
--- a/crypto/s390xcpuid.pl
+++ b/crypto/s390xcpuid.pl
@@ -262,6 +262,48 @@ OPENSSL_vx_probe:
 .size	OPENSSL_vx_probe,.-OPENSSL_vx_probe
 ___
 
+{
+################
+# void s390x_kimd(const unsigned char *in, size_t len, unsigned int fc,
+#                 void *param)
+my ($in,$len,$fc,$param) = map("%r$_",(2..5));
+$code.=<<___;
+.globl	s390x_kimd
+.type	s390x_kimd,\@function
+.align	16
+s390x_kimd:
+	llgfr	%r0,$fc
+	lgr	%r1,$param
+
+	.long	0xb93e0002	# kimd %r0,%r2
+	brc	1,.-4		# pay attention to "partial completion"
+
+	br	$ra
+.size	s390x_kimd,.-s390x_kimd
+___
+}
+
+{
+################
+# void s390x_klmd(const unsigned char *in, size_t inlen, unsigned char *out,
+#                 size_t outlen, unsigned int fc, void *param)
+my ($in,$inlen,$out,$outlen,$fc) = map("%r$_",(2..6));
+$code.=<<___;
+.globl	s390x_klmd
+.type	s390x_klmd,\@function
+.align	32
+s390x_klmd:
+	llgfr	%r0,$fc
+	l${g}	%r1,$stdframe($sp)
+
+	.long	0xb93f0042	# klmd %r4,%r2
+	brc	1,.-4		# pay attention to "partial completion"
+
+	br	$ra
+.size	s390x_klmd,.-s390x_klmd
+___
+}
+
 ################
 # void s390x_km(const unsigned char *in, size_t len, unsigned char *out,
 #               unsigned int fc, void *param)