Make DSA_generate_parameters, and fix a couple of bug

(including another problem in the s3_srvr.c state machine).

7 files changed, 90 insertions, 112 deletions

diff --git a/crypto/bn/bn.h b/crypto/bn/bn.h
index 93e2f6f2bd..e88291d62c 100644
--- a/crypto/bn/bn.h
+++ b/crypto/bn/bn.h
@@ -292,7 +292,7 @@ typedef struct bn_recp_ctx_st
  * of Applied Cryptography [Menezes, van Oorschot, Vanstone; CRC Press 1996];
  * original paper: Damgaard, Landrock, Pomerance: Average case error estimates
  * for the strong probable prime test. -- Math. Comp. 61 (1993) 177-194) */
-#define BN_prime_checks_size(b) ((b) >= 1300 ?  2 : \
+#define BN_prime_checks_for_size(b) ((b) >= 1300 ?  2 : \
                                 (b) >=  850 ?  3 : \
                                 (b) >=  650 ?  4 : \
                                 (b) >=  550 ?  5 : \
@@ -406,6 +406,10 @@ BIGNUM *BN_generate_prime(BIGNUM *ret,int bits,int safe,BIGNUM *add,
 		BIGNUM *rem,void (*callback)(int,int,void *),void *cb_arg);
 int	BN_is_prime(BIGNUM *p,int nchecks,void (*callback)(int,int,void *),
 		BN_CTX *ctx,void *cb_arg);
+int	BN_is_prime_fasttest(BIGNUM *p,int nchecks,
+		void (*callback)(int,int,void *),
+		BN_CTX *ctx,BN_CTX *ctx2,void *cb_arg,
+		int do_trial_division);
 void	ERR_load_BN_strings(void );
 
 BN_ULONG bn_mul_add_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w);
diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c
index 84f0699b9b..e679c7c822 100644
--- a/crypto/bn/bn_prime.c
+++ b/crypto/bn/bn_prime.c
@@ -84,7 +84,7 @@ BIGNUM *BN_generate_prime(BIGNUM *ret, int bits, int safe, BIGNUM *add,
 	int found=0;
 	int i,j,c1=0;
 	BN_CTX *ctx;
-	int checks = BN_prime_checks_size(bits);
+	int checks = BN_prime_checks_for_size(bits);
 
 	ctx=BN_CTX_new();
 	if (ctx == NULL) goto err;
@@ -154,10 +154,12 @@ err:
 	return(found ? rnd : NULL);
 	}
 
-int BN_is_prime(BIGNUM *a, int checks, void (*callback)(int,int,void *),
-	     BN_CTX *ctx_passed, void *cb_arg)
+int BN_is_prime_fasttest(BIGNUM *a, int checks,
+		void (*callback)(int,int,void *),
+		BN_CTX *ctx_passed, BN_CTX *ctx2_passed, void *cb_arg,
+		int do_trial_division)
 	{
-	int i,j,c2=0,ret= -1;
+	int i,j,ret= -1;
 	BIGNUM *check;
 	BN_CTX *ctx=NULL,*ctx2=NULL;
 	BN_MONT_CTX *mont=NULL;
@@ -165,17 +167,25 @@ int BN_is_prime(BIGNUM *a, int checks, void (*callback)(int,int,void *),
 	if (checks == BN_prime_checks)
 		{
 		int bits = BN_num_bits(a);
-		checks = BN_prime_checks_size(bits);
+		checks = BN_prime_checks_for_size(bits);
 		}
 
 	if (!BN_is_odd(a))
 		return(0);
+	if (do_trial_division)
+		for (i = 1; i < NUMPRIMES; i++)
+			if (BN_mod_word(a, primes[i]) == 0) 
+				return 0;
+
 	if (ctx_passed != NULL)
 		ctx=ctx_passed;
 	else
 		if ((ctx=BN_CTX_new()) == NULL) goto err;
+	if (ctx2_passed != NULL)
+		ctx2=ctx2_passed;
+	else
+		if ((ctx2=BN_CTX_new()) == NULL) goto err;
 
-	if ((ctx2=BN_CTX_new()) == NULL) goto err;
 	if ((mont=BN_MONT_CTX_new()) == NULL) goto err;
 
 	check= &(ctx->bn[ctx->tos++]);
@@ -185,7 +195,9 @@ int BN_is_prime(BIGNUM *a, int checks, void (*callback)(int,int,void *),
 
 	for (i=0; i<checks; i++)
 		{
-		if (!BN_pseudo_rand(check,BN_num_bits(a)-1,0,0)) goto err;
+		if (!BN_pseudo_rand(check,BN_num_bits(a),0,0)) goto err;
+		if (BN_cmp(check, a) >= 0)
+			BN_sub(check, check, a);
 		j=witness(check,a,ctx,ctx2,mont);
 		if (j == -1) goto err;
 		if (j)
@@ -193,20 +205,26 @@ int BN_is_prime(BIGNUM *a, int checks, void (*callback)(int,int,void *),
 			ret=0;
 			goto err;
 			}
-		if (callback != NULL) callback(1,c2++,cb_arg);
+		if (callback != NULL) callback(1,i,cb_arg);
 		}
 	ret=1;
 err:
 	ctx->tos--;
 	if ((ctx_passed == NULL) && (ctx != NULL))
 		BN_CTX_free(ctx);
-	if (ctx2 != NULL)
+	if ((ctx2_passed == NULL) && (ctx2 != NULL))
 		BN_CTX_free(ctx2);
 	if (mont != NULL) BN_MONT_CTX_free(mont);
 		
 	return(ret);
 	}
 
+int BN_is_prime(BIGNUM *a, int checks, void (*callback)(int,int,void *),
+	BN_CTX *ctx_passed, void *cb_arg)
+	{
+	return BN_is_prime_fasttest(a, checks, callback, ctx_passed, NULL, cb_arg, 0);
+	}
+
 static int witness(BIGNUM *a, BIGNUM *n, BN_CTX *ctx, BN_CTX *ctx2,
 	     BN_MONT_CTX *mont)
 	{
@@ -274,7 +292,7 @@ err:
 static int probable_prime(BIGNUM *rnd, int bits)
 	{
 	int i;
-	MS_STATIC BN_ULONG mods[NUMPRIMES];
+	BN_ULONG mods[NUMPRIMES];
 	BN_ULONG delta,d;
 
 again:
diff --git a/crypto/dsa/dsa.h b/crypto/dsa/dsa.h
index 3da104b6dd..e59c7d293f 100644
--- a/crypto/dsa/dsa.h
+++ b/crypto/dsa/dsa.h
@@ -197,7 +197,11 @@ int	DSAparams_print_fp(FILE *fp, DSA *x);
 int	DSA_print_fp(FILE *bp, DSA *x, int off);
 #endif
 
-int DSA_is_prime(BIGNUM *q,void (*callback)(),void *cb_arg);
+#define DSS_prime_checks 50
+/* Primality test according to FIPS PUB 186[-1], Appendix 2.1:
+ * 50 rounds of Rabin-Miller */
+#define DSA_is_prime(n, callback, cb_arg) \
+	BN_is_prime(n, DSS_prime_checks, callback, NULL, cb_arg)
 
 #ifndef NO_DH
 /* Convert DSA structure (key or just parameters) into DH structure
@@ -218,7 +222,6 @@ DH *DSA_dup_DH(DSA *r);
 #define DSA_F_DSAPARAMS_PRINT_FP			 101
 #define DSA_F_DSA_DO_SIGN				 112
 #define DSA_F_DSA_DO_VERIFY				 113
-#define DSA_F_DSA_IS_PRIME				 102
 #define DSA_F_DSA_NEW					 103
 #define DSA_F_DSA_PRINT					 104
 #define DSA_F_DSA_PRINT_FP				 105
diff --git a/crypto/dsa/dsa_err.c b/crypto/dsa/dsa_err.c
index 33a8270afd..38e4af968c 100644
--- a/crypto/dsa/dsa_err.c
+++ b/crypto/dsa/dsa_err.c
@@ -70,7 +70,6 @@ static ERR_STRING_DATA DSA_str_functs[]=
 {ERR_PACK(0,DSA_F_DSAPARAMS_PRINT_FP,0),	"DSAparams_print_fp"},
 {ERR_PACK(0,DSA_F_DSA_DO_SIGN,0),	"DSA_do_sign"},
 {ERR_PACK(0,DSA_F_DSA_DO_VERIFY,0),	"DSA_do_verify"},
-{ERR_PACK(0,DSA_F_DSA_IS_PRIME,0),	"DSA_is_prime"},
 {ERR_PACK(0,DSA_F_DSA_NEW,0),	"DSA_new"},
 {ERR_PACK(0,DSA_F_DSA_PRINT,0),	"DSA_print"},
 {ERR_PACK(0,DSA_F_DSA_PRINT_FP,0),	"DSA_print_fp"},
diff --git a/crypto/dsa/dsa_gen.c b/crypto/dsa/dsa_gen.c
index 5df9132dd8..f7e0b585a1 100644
--- a/crypto/dsa/dsa_gen.c
+++ b/crypto/dsa/dsa_gen.c
@@ -59,12 +59,18 @@
 #undef GENUINE_DSA
 
 #ifdef GENUINE_DSA
+/* Parameter generation follows the original release of FIPS PUB 186,
+ * Appendix 2.2 (i.e. use SHA as defined in FIPS PUB 180) */
 #define HASH    SHA
 #else
+/* Parameter generation follows the updated Appendix 2.2 for FIPS PUB 186,
+ * also Appendix 2.2 of FIPS PUB 186-1 (i.e. use SHA as defined in
+ * FIPS PUB 180-1) */
 #define HASH    SHA1
 #endif 
 
 #ifndef NO_SHA
+
 #include <stdio.h>
 #include <time.h>
 #include "cryptlib.h"
@@ -74,8 +80,8 @@
 #include <openssl/rand.h>
 
 DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
-	     int *counter_ret, unsigned long *h_ret, void (*callback)(),
-	     void *cb_arg)
+		int *counter_ret, unsigned long *h_ret, void (*callback)(),
+		void *cb_arg)
 	{
 	int ok=0;
 	unsigned char seed[SHA_DIGEST_LENGTH];
@@ -86,19 +92,26 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 	BN_MONT_CTX *mont=NULL;
 	int k,n=0,i,b,m=0;
 	int counter=0;
-	BN_CTX *ctx=NULL,*ctx2=NULL;
+	int r=0;
+	BN_CTX *ctx=NULL,*ctx2=NULL,*ctx3=NULL,*ctx4=NULL;
 	unsigned int h=2;
 	DSA *ret=NULL;
 
 	if (bits < 512) bits=512;
 	bits=(bits+63)/64*64;
 
-	if (seed_len < 20) seed_in = NULL;
+	if (seed_len < 20)
+		seed_in = NULL; /* seed buffer too small -- ignore */
+	if (seed_len > 20) 
+		seed_len = 20; /* App. 2.2 of FIPS PUB 186 allows larger SEED,
+		                * but our internal buffers are restricted to 160 bits*/
 	if ((seed_in != NULL) && (seed_len == 20))
 		memcpy(seed,seed_in,seed_len);
 
 	if ((ctx=BN_CTX_new()) == NULL) goto err;
 	if ((ctx2=BN_CTX_new()) == NULL) goto err;
+	if ((ctx3=BN_CTX_new()) == NULL) goto err;
+	if ((ctx4=BN_CTX_new()) == NULL) goto err;
 	if ((ret=DSA_new()) == NULL) goto err;
 
 	if ((mont=BN_MONT_CTX_new()) == NULL) goto err;
@@ -116,18 +129,24 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 
 	for (;;)
 		{
-		for (;;)
+		for (;;) /* find q */
 			{
+			int seed_is_random = 0;
+
 			/* step 1 */
 			if (callback != NULL) callback(0,m++,cb_arg);
 
 			if (!seed_len)
+				{
 				RAND_pseudo_bytes(seed,SHA_DIGEST_LENGTH);
+				seed_is_random = 1;
+				}
 			else
+				/* use random seed if 'seed_in' turns out to be bad */
 				seed_len=0;
-
 			memcpy(buf,seed,SHA_DIGEST_LENGTH);
 			memcpy(buf2,seed,SHA_DIGEST_LENGTH);
+			/* precompute "SEED + 1" for step 7: */
 			for (i=SHA_DIGEST_LENGTH-1; i >= 0; i--)
 				{
 				buf[i]++;
@@ -146,7 +165,12 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 			if (!BN_bin2bn(md,SHA_DIGEST_LENGTH,q)) goto err;
 
 			/* step 4 */
-			if (BN_is_prime(q,BN_prime_checks,callback,NULL,cb_arg) > 0) break;
+			r = BN_is_prime_fasttest(q, DSS_prime_checks, callback, ctx3, ctx4, cb_arg, seed_is_random);
+			if (r > 0)
+					break;
+			if (r != 0)
+					goto err;
+
 			/* do a callback call */
 			/* step 5 */
 			}
@@ -156,16 +180,22 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 
 		/* step 6 */
 		counter=0;
+		/* "offset = 2" */
 
 		n=(bits-1)/160;
 		b=(bits-1)-n*160;
 
 		for (;;)
 			{
+			if (callback != NULL && counter != 0)
+				callback(0,counter,cb_arg);
+
 			/* step 7 */
 			BN_zero(W);
+			/* now 'buf' contains "SEED + offset - 1" */
 			for (k=0; k<=n; k++)
 				{
+				/* obtain "SEED + offset + k" by incrementing: */
 				for (i=SHA_DIGEST_LENGTH-1; i >= 0; i--)
 					{
 					buf[i]++;
@@ -196,17 +226,19 @@ DSA *DSA_generate_parameters(int bits, unsigned char *seed_in, int seed_len,
 			if (BN_cmp(p,test) >= 0)
 				{
 				/* step 11 */
-				if (BN_is_prime(p,BN_prime_checks,callback,NULL,cb_arg) > 0)
-					goto end;
+				r = BN_is_prime_fasttest(p, DSS_prime_checks, callback, ctx3, ctx4, cb_arg, 1);
+				if (r > 0)
+						goto end; /* found it */
+				if (r != 0)
+					goto err;
 				}
 
 			/* step 13 */
 			counter++;
+			/* "offset = offset + n + 1" */
 
 			/* step 14 */
 			if (counter >= 4096) break;
-
-			if (callback != NULL) callback(0,counter,cb_arg);
 			}
 		}
 end:
@@ -247,90 +279,10 @@ err:
 		if (h_ret != NULL) *h_ret=h;
 		}
 	if (ctx != NULL) BN_CTX_free(ctx);
-	if (ctx != NULL) BN_CTX_free(ctx2);
+	if (ctx2 != NULL) BN_CTX_free(ctx2);
+	if (ctx3 != NULL) BN_CTX_free(ctx3);
+	if (ctx4 != NULL) BN_CTX_free(ctx4);
 	if (mont != NULL) BN_MONT_CTX_free(mont);
 	return(ok?ret:NULL);
 	}
-
-int DSA_is_prime(BIGNUM *w, void (*callback)(), void *cb_arg)
-	{
-	int ok= -1,j,i,n;
-	BN_CTX *ctx=NULL,*ctx2=NULL;
-	BIGNUM *w_1,*b,*m,*z,*tmp,*mont_1;
-	int a;
-	BN_MONT_CTX *mont=NULL;
-
-	if (!BN_is_odd(w)) return(0);
-
-	if ((ctx=BN_CTX_new()) == NULL) goto err;
-	if ((ctx2=BN_CTX_new()) == NULL) goto err;
-	if ((mont=BN_MONT_CTX_new()) == NULL) goto err;
-
-	m=   &(ctx2->bn[2]);
-	b=   &(ctx2->bn[3]);
-	z=   &(ctx2->bn[4]);
-	w_1= &(ctx2->bn[5]);
-	tmp= &(ctx2->bn[6]);
-	mont_1= &(ctx2->bn[7]);
-
-	/* step 1 */
-	n=BN_prime_checks_size(BN_num_bits(w));
-
-	/* step 2 */
-	if (!BN_sub(w_1,w,BN_value_one())) goto err;
-	for (a=1; !BN_is_bit_set(w_1,a); a++)
-		;
-	if (!BN_rshift(m,w_1,a)) goto err;
-
-	BN_MONT_CTX_set(mont,w,ctx);
-	BN_to_montgomery(mont_1,BN_value_one(),mont,ctx);
-	BN_to_montgomery(w_1,w_1,mont,ctx);
-	for (i=1; i < n; i++)
-		{
-		/* step 3 */
-		if (!BN_pseudo_rand(b,BN_num_bits(w)-2/*-1*/,0,0))
-			goto err;
-		/* BN_set_word(b,0x10001L); */
-
-		/* step 4 */
-		j=0;
-		if (!BN_mod_exp_mont(z,b,m,w,ctx,mont)) goto err;
-
-		if (!BN_to_montgomery(z,z,mont,ctx)) goto err;
-
-		/* step 5 */
-		for (;;)
-			{
-			if (((j == 0) && (BN_cmp(z,mont_1) == 0)) ||
-				(BN_cmp(z,w_1) == 0))
-				break;
-
-			/* step 6 */
-			if ((j > 0) && (BN_cmp(z,mont_1) == 0))
-				{
-				ok=0;
-				goto err;
-				}
-
-			j++;
-			if (j >= a)
-				{
-				ok=0;
-				goto err;
-				}
-
-			if (!BN_mod_mul_montgomery(z,z,z,mont,ctx)) goto err;
-			if (callback != NULL) callback(1,j,cb_arg);
-			}
-		}
-
-	ok=1;
-err:
-	if (ok == -1) DSAerr(DSA_F_DSA_IS_PRIME,ERR_R_BN_LIB);
-	BN_CTX_free(ctx);
-	BN_CTX_free(ctx2);
-	BN_MONT_CTX_free(mont);
-	
-	return(ok);
-	}
 #endif
diff --git a/crypto/dsa/dsatest.c b/crypto/dsa/dsatest.c
index a30dae6b72..f096ed0688 100644
--- a/crypto/dsa/dsatest.c
+++ b/crypto/dsa/dsatest.c
@@ -85,6 +85,9 @@ int main(int argc, char *argv[])
 #endif
 
 static void MS_CALLBACK dsa_cb(int p, int n, char *arg);
+
+/* seed, out_p, out_q, out_g are taken from the updated Appendix 5 to
+ * FIPS PUB 186 and also appear in Appendix 5 to FIPS PIB 186-1 */
 static unsigned char seed[20]={
 	0xd5,0x01,0x4e,0x4b,0x60,0xef,0x2b,0xa8,0xb6,0x21,0x1b,0x40,
 	0x62,0xba,0x32,0x24,0xe0,0x42,0x7d,0xd3,
@@ -141,9 +144,8 @@ int main(int argc, char **argv)
 	CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 
 	BIO_printf(bio_err,"test generation of DSA parameters\n");
-	BIO_printf(bio_err,"expect '.*' followed by 3 lines of '.'s and '+'s\n");
-	dsa=DSA_generate_parameters(512,seed,20,&counter,&h,dsa_cb,
-		(char *)bio_err);
+
+	dsa=DSA_generate_parameters(512,seed,20,&counter,&h,dsa_cb,(char *)bio_err);
 
 	BIO_printf(bio_err,"seed\n");
 	for (i=0; i<20; i+=4)
diff --git a/crypto/sha/sha.h b/crypto/sha/sha.h
index 96daa968fe..5e9cf20cc8 100644
--- a/crypto/sha/sha.h
+++ b/crypto/sha/sha.h
@@ -63,7 +63,7 @@
 extern "C" {
 #endif
 
-#if defined(NO_SHA) || defined(NO_SHA0) || defined(NO_SHA1)
+#if defined(NO_SHA) || (defined(NO_SHA0) && defined(NO_SHA1))
 #error SHA is disabled.
 #endif