Debian bug 677275 - random() max value

* libmisc/salt.c: random() max value is 2^31-1 (same as RAND_MAX
	on GNU). As it is not clear whether on some systems the max value
	can exceed this number and whether some systems have max values
	which would be lower, we take this into account when defining the
	salt size and number of rounds for SHA encrypted passwords. Higher
	values are favored.

ChangeLog

@@ -1,3 +1,12 @@
+2013-08-05  Nicolas François  <nicolas.francois@centraliens.net>
+
+	* libmisc/salt.c: random() max value is 2^31-1 (same as RAND_MAX
+	on GNU). As it is not clear whether on some systems the max value
+	can exceed this number and whether some systems have max values
+	which would be lower, we take this into account when defining the
+	salt size and number of rounds for SHA encrypted passwords. Higher
+	values are favored.
+
 2013-08-04  Nicolas François  <nicolas.francois@centraliens.net>
 
 	* man/su.1.xml: With getopt, '-' does not need to be the last

libmisc/salt.c

@@ -23,7 +23,7 @@
 static void seedRNG (void);
 static /*@observer@*/const char *gensalt (size_t salt_size);
 #ifdef USE_SHA_CRYPT
-static size_t SHA_salt_size (void);
+static size_t shadow_random (size_t min, size_t max);
 static /*@observer@*/const char *SHA_salt_rounds (/*@null@*/int *prefered_rounds);
 #endif /* USE_SHA_CRYPT */
 
@@ -81,17 +81,29 @@ static void seedRNG (void)
 #define MAGNUM(array,ch)	(array)[0]=(array)[2]='$',(array)[1]=(ch),(array)[3]='\0'
 
 #ifdef USE_SHA_CRYPT
+/* It is not clear what is the maximum value of random().
+ * We assume 2^31-1.*/
+#define RANDOM_MAX 0x7FFFFFFF
+
 /*
- * Return the salt size.
+ * Return a random number between min and max (both included).
- * The size of the salt string is between 8 and 16 bytes for the SHA crypt
+ *
- * methods.
+ * It favors slightly the higher numbers.
  */
-static size_t SHA_salt_size (void)
+static size_t shadow_random (size_t min, size_t max)
 {
-	double rand_size;
+	double drand;
+	size_t ret;
 	seedRNG ();
-	rand_size = (double) 9.0 * random () / RAND_MAX;
+	drand = (double) (max - min + 1) * random () / RANDOM_MAX;
-	return (size_t) (8 + rand_size);
+	/* On systems were this is not random() range is lower, we favor
+	 * higher numbers of salt. */
+	ret = (size_t) (max + 1 - drand);
+	/* And we catch limits, and use the highest number */
+	if ((ret < min) || (ret > max)) {
+		ret = max;
+	}
+	return ret;
 }
 
 /* Default number of rounds if not explicitly specified.  */
@@ -130,10 +142,7 @@ static /*@observer@*/const char *SHA_salt_rounds (/*@null@*/int *prefered_rounds
 			max_rounds = min_rounds;
 		}
 
-		seedRNG ();
+		rounds = shadow_random (min_rounds, max_rounds);
-		rand_rounds = (double) (max_rounds-min_rounds+1.0) * random ();
-		rand_rounds /= RAND_MAX;
-		rounds = min_rounds + rand_rounds;
 	} else if (0 == *prefered_rounds) {
 		return "";
 	} else {
@@ -226,11 +235,11 @@ static /*@observer@*/const char *gensalt (size_t salt_size)
 	} else if (0 == strcmp (method, "SHA256")) {
 		MAGNUM(result, '5');
 		strcat(result, SHA_salt_rounds((int *)arg));
-		salt_len = SHA_salt_size();
+		salt_len = shadow_random (8, 16);
 	} else if (0 == strcmp (method, "SHA512")) {
 		MAGNUM(result, '6');
 		strcat(result, SHA_salt_rounds((int *)arg));
-		salt_len = SHA_salt_size();
+		salt_len = shadow_random (8, 16);
 #endif /* USE_SHA_CRYPT */
 	} else if (0 != strcmp (method, "DES")) {
 		fprintf (stderr,