From 7ce66edc572eb3c47c890bfe3ce5d37cc2b0f346 Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Wed, 15 Jun 2016 23:21:29 +0200 Subject: [PATCH 1/8] lib/util_hash.c: add xbps_file_hash_raw method this function does not mmap the target file and therefore avoids out of memory exceptions on 32bit systems. --- include/xbps.h.in | 11 +++++++++++ lib/util_hash.c | 33 ++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/include/xbps.h.in b/include/xbps.h.in index 6f707cb9..6c1a7964 100644 --- a/include/xbps.h.in +++ b/include/xbps.h.in @@ -1795,6 +1795,17 @@ bool xbps_mmap_file(const char *file, void **mmf, size_t *mmflen, size_t *filele */ char *xbps_file_hash(const char *file); +/** + * Returns a raw byte buffer with the sha256 hash for the file specified + * by \a file. + * + * @param[in] file Path to a file. + * @return A pointer to a malloc(3)ed buffer, NULL otherwise and errno + * is set appropiately. The pointer should be free(3)d when it's no + * longer needed. + */ +unsigned char *xbps_file_hash_raw(const char *file); + /** * Compares the sha256 hash of the file \a file with the sha256 * string specified by \a sha256. diff --git a/lib/util_hash.c b/lib/util_hash.c index 0c03c3c0..b170b432 100644 --- a/lib/util_hash.c +++ b/lib/util_hash.c @@ -108,24 +108,39 @@ xbps_mmap_file(const char *file, void **mmf, size_t *mmflen, size_t *filelen) return true; } +unsigned char * +xbps_file_hash_raw(const char *file) +{ + int fd; + ssize_t len; + unsigned char *digest, buf[65536]; + SHA256_CTX sha256; + + if ((fd = open(file, O_RDONLY)) < 0) + return NULL; + digest = malloc(SHA256_DIGEST_LENGTH); + assert(digest); + SHA256_Init(&sha256); + while ((len = read(fd, buf, sizeof(buf))) > 0) + SHA256_Update(&sha256, buf, len); + SHA256_Final(digest, &sha256); + close(fd); + + return digest; +} + char * xbps_file_hash(const char *file) { char *res, hash[SHA256_DIGEST_LENGTH * 2 + 1]; - unsigned char digest[SHA256_DIGEST_LENGTH]; - unsigned char *mmf = NULL; - size_t mmflen, filelen; + unsigned char *digest; - if (!xbps_mmap_file(file, (void *)&mmf, &mmflen, &filelen)) + if (!(digest = xbps_file_hash_raw(file))) return NULL; - if (SHA256(mmf, filelen, digest) == NULL) { - (void)munmap(mmf, mmflen); - return NULL; - } digest2string(digest, hash, SHA256_DIGEST_LENGTH); res = strdup(hash); - (void)munmap(mmf, mmflen); + free(digest); return res; } From df97be6a54c22377a508ade24342cec32a9db0ad Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Wed, 15 Jun 2016 23:22:44 +0200 Subject: [PATCH 2/8] lib/verifysig.c: use xbps_file_hash_raw() instead of mmap'ing the source file, xbps_file_hash_raw is used to generate a digest of the file. --- lib/verifysig.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/lib/verifysig.c b/lib/verifysig.c index 2fc56529..a48bb4a7 100644 --- a/lib/verifysig.c +++ b/lib/verifysig.c @@ -41,14 +41,12 @@ #include "xbps_api_impl.h" static bool -rsa_verify_buf(struct xbps_repo *repo, xbps_data_t pubkey, +rsa_verify_hash(struct xbps_repo *repo, xbps_data_t pubkey, unsigned char *sig, unsigned int siglen, - unsigned char *buf, unsigned int buflen) + unsigned char *sha256) { - SHA256_CTX context; BIO *bio; RSA *rsa; - unsigned char sha256[SHA256_DIGEST_LENGTH]; int rv; ERR_load_crypto_strings(); @@ -65,11 +63,7 @@ rsa_verify_buf(struct xbps_repo *repo, xbps_data_t pubkey, return false; } - SHA256_Init(&context); - SHA256_Update(&context, buf, buflen); - SHA256_Final(sha256, &context); - - rv = RSA_verify(NID_sha1, sha256, sizeof(sha256), sig, siglen, rsa); + rv = RSA_verify(NID_sha1, sha256, SHA256_DIGEST_LENGTH, sig, siglen, rsa); RSA_free(rsa); BIO_free(bio); ERR_free_strings(); @@ -83,8 +77,8 @@ xbps_verify_file_signature(struct xbps_repo *repo, const char *fname) xbps_dictionary_t repokeyd = NULL; xbps_data_t pubkey; char *hexfp = NULL; - unsigned char *buf = NULL, *sig_buf = NULL; - size_t buflen, filelen, sigbuflen, sigfilelen; + unsigned char *digest = NULL, *sig_buf = NULL; + size_t sigbuflen, sigfilelen; char *rkeyfile = NULL, *sig = NULL; bool val = false; @@ -116,7 +110,7 @@ xbps_verify_file_signature(struct xbps_repo *repo, const char *fname) /* * Prepare fname and signature data buffers. */ - if (!xbps_mmap_file(fname, (void *)&buf, &buflen, &filelen)) { + if (!(digest = xbps_file_hash_raw(fname))) { xbps_dbg_printf(repo->xhp, "can't open file %s: %s\n", fname, strerror(errno)); goto out; } @@ -128,7 +122,7 @@ xbps_verify_file_signature(struct xbps_repo *repo, const char *fname) /* * Verify fname RSA signature. */ - if (rsa_verify_buf(repo, pubkey, sig_buf, sigfilelen, buf, filelen)) + if (rsa_verify_hash(repo, pubkey, sig_buf, sigfilelen, digest)) val = true; out: @@ -136,8 +130,8 @@ out: free(hexfp); if (rkeyfile) free(rkeyfile); - if (buf) - (void)munmap(buf, buflen); + if (digest) + free(digest); if (sig_buf) (void)munmap(sig_buf, sigbuflen); if (sig) From b55ffeceaeea973e457ac1e3bdf44acb20417096 Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Thu, 16 Jun 2016 06:50:14 +0200 Subject: [PATCH 3/8] lib/util_hash.c: write directly to malloced string instead if coping it over --- lib/util_hash.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/util_hash.c b/lib/util_hash.c index b170b432..f13f1b0e 100644 --- a/lib/util_hash.c +++ b/lib/util_hash.c @@ -132,17 +132,18 @@ xbps_file_hash_raw(const char *file) char * xbps_file_hash(const char *file) { - char *res, hash[SHA256_DIGEST_LENGTH * 2 + 1]; + char *hash; unsigned char *digest; if (!(digest = xbps_file_hash_raw(file))) return NULL; + hash = malloc(SHA256_DIGEST_LENGTH * 2 + 1); + assert(hash); digest2string(digest, hash, SHA256_DIGEST_LENGTH); - res = strdup(hash); free(digest); - return res; + return hash; } int From f0f15304e10e7be3e6a47dfd97b5da740c32d72f Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Fri, 17 Jun 2016 13:05:49 +0200 Subject: [PATCH 4/8] update NEWS --- NEWS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS b/NEWS index bdec93d9..3d8db29a 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,11 @@ xbps-0.52 (???): + * libxbps: avoid mmap in cases where the mmaped file can fill up the address + space on 32bit causing out of memory errors. Patches provided by Enno + Boland in #183, reported by Christian Neukirchen in #182. See + https://github.com/voidlinux/xbps/pull/183 and + https://github.com/voidlinux/xbps/pull/182 + * xbps-create(1): accept -c/--changelog to set the changelog property of the package From cb94ffe1c06bff6d5e0d0f2f9e8f378283ef4300 Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Fri, 17 Jun 2016 16:55:02 +0200 Subject: [PATCH 5/8] util_hash.c: fail on negative result code of read. --- lib/util_hash.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/util_hash.c b/lib/util_hash.c index f13f1b0e..0e62d45e 100644 --- a/lib/util_hash.c +++ b/lib/util_hash.c @@ -125,6 +125,8 @@ xbps_file_hash_raw(const char *file) SHA256_Update(&sha256, buf, len); SHA256_Final(digest, &sha256); close(fd); + if(len < 0) + return NULL; return digest; } From 30d1d0f6077a82eb41347c154efd99f7eb33a0f1 Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Fri, 17 Jun 2016 17:00:39 +0200 Subject: [PATCH 6/8] util_hash.c: void return value --- lib/util_hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util_hash.c b/lib/util_hash.c index 0e62d45e..1c1060b9 100644 --- a/lib/util_hash.c +++ b/lib/util_hash.c @@ -124,7 +124,7 @@ xbps_file_hash_raw(const char *file) while ((len = read(fd, buf, sizeof(buf))) > 0) SHA256_Update(&sha256, buf, len); SHA256_Final(digest, &sha256); - close(fd); + (void)close(fd); if(len < 0) return NULL; From 35fa3b58087fa4e52f9c04c974a6091800dfb432 Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Fri, 17 Jun 2016 17:10:10 +0200 Subject: [PATCH 7/8] xbps-create: remove mmap to load files to archive. --- bin/xbps-create/main.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bin/xbps-create/main.c b/bin/xbps-create/main.c index 61cb1a4f..3bd30219 100644 --- a/bin/xbps-create/main.c +++ b/bin/xbps-create/main.c @@ -526,8 +526,9 @@ static void write_entry(struct archive *ar, struct archive_entry *entry) { const char *name; - char *mmf; - size_t mmflen, filelen; + int fd; + char buf[65536]; + ssize_t len; if (archive_entry_pathname(entry) == NULL) return; @@ -546,11 +547,16 @@ write_entry(struct archive *ar, struct archive_entry *entry) } name = archive_entry_sourcepath(entry); - if (!xbps_mmap_file(name, (void *)&mmf, &mmflen, &filelen)) - die("cannot read %s file", name); - archive_write_data(ar, mmf, filelen); - (void)munmap(mmf, mmflen); + if ((fd = open(name, O_RDONLY)) < 0) + die("cannot open %s file", name); + while ((len = read(fd, buf, sizeof(buf))) > 0) + archive_write_data(ar, buf, len); + (void)close(fd); + + if(len < 0) + die("cannot open %s file", name); + archive_entry_free(entry); } From 842ac7c97e26b385c3d7c3d9743911e84808238a Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Mon, 20 Jun 2016 10:03:49 +0200 Subject: [PATCH 8/8] lib/util_hash.c: fix memleak. --- lib/util_hash.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/util_hash.c b/lib/util_hash.c index 1c1060b9..985e9063 100644 --- a/lib/util_hash.c +++ b/lib/util_hash.c @@ -123,10 +123,12 @@ xbps_file_hash_raw(const char *file) SHA256_Init(&sha256); while ((len = read(fd, buf, sizeof(buf))) > 0) SHA256_Update(&sha256, buf, len); + if(len < 0) { + free(digest); + return NULL; + } SHA256_Final(digest, &sha256); (void)close(fd); - if(len < 0) - return NULL; return digest; }