Optimize repository API functions.

- Remove xbps_repo_get_plist() and try to internalize all members at
  xbps_repo_open() time.

- Added xbps_repo_open_idxfiles() to also internalize the index-files
  plist from repository, which is really huge and must only be internalized
  when needed.

- Improve how signed and verified repositories are detected.

- Misc optimizations and small performance improvements.

Bump XBPS_API_VERSION.

include/xbps.h.in

@@ -46,7 +46,7 @@
  *
  * This header documents the full API for the XBPS Library.
  */
-#define XBPS_API_VERSION	"20131005"
+#define XBPS_API_VERSION	"20131007"
 
 #ifndef XBPS_VERSION
  #define XBPS_VERSION		"UNSET"
@@ -106,15 +106,15 @@
 
 /**
  * @def XBPS_REPOIDX_FILES
- * Filename for the repository index files property list.
+ * Filename for the repository index-files property list.
  */
 #define XBPS_REPOIDX_FILES	"index-files.plist"
 
 /**
- * @def XBPS_REPOMETA
+ * @def XBPS_REPOIDX_META
- * Filename for the repository metadata property list.
+ * Filename for the repository index metadata property list.
  */
-#define XBPS_REPOMETA 		"meta.plist"
+#define XBPS_REPOIDX_META 	"index-meta.plist"
 
 /**
  * @def XBPS_SYSCONF_PATH
@@ -470,6 +470,9 @@ struct xbps_handle {
 	 * @private
 	 */
 	cfg_t *cfg;
+	xbps_dictionary_t repokeys;
+	xbps_dictionary_t pkg_metad;
+	xbps_dictionary_t pkgdb_revdeps;
 	/**
 	 * @var pkgdb
 	 *
@@ -477,14 +480,6 @@ struct xbps_handle {
 	 * stored in XBPS_META_PATH/XBPS_PKGDB.
 	 */
 	xbps_dictionary_t pkgdb;
-	/**
-	 * @private
-	 */
-	xbps_dictionary_t pkg_metad;
-	/**
-	 * @private
-	 */
-	xbps_dictionary_t pkgdb_revdeps;
 	/**
 	 * @var transd
 	 *
@@ -1117,6 +1112,12 @@ struct xbps_repo {
 	 * @private
 	 */
 	struct archive *ar;
+	/**
+	 * @var xhp
+	 *
+	 * Pointer to our xbps_handle struct passed to xbps_rpool_foreach.
+	 */
+	struct xbps_handle *xhp;
 	/**
 	 * @var idx
 	 *
@@ -1126,7 +1127,7 @@ struct xbps_repo {
 	/**
 	 * @var idxfiles
 	 *
-	 * Proplib dictionary associated with the repository index files.
+	 * Proplib dictionary associated with the repository index-files.
 	 */
 	xbps_dictionary_t idxfiles;
 	/**
@@ -1141,6 +1142,12 @@ struct xbps_repo {
 	 * URI string associated with repository.
 	 */
 	const char *uri;
+	/**
+	 * var is_remote
+	 *
+	 * True if repository is remote, false if it's a local repository.
+	 */
+	bool is_remote;
 	/**
 	 * var is_signed
 	 *
@@ -1155,12 +1162,6 @@ struct xbps_repo {
 	 * False if the stored public key did not match its signature.
 	 */
 	bool is_verified;
-	/**
-	 * @var xhp
-	 *
-	 * Pointer to our xbps_handle struct passed to xbps_rpool_foreach.
-	 */
-	struct xbps_handle *xhp;
 };
 
 /**
@@ -1273,17 +1274,6 @@ xbps_dictionary_t xbps_rpool_get_pkg_plist(struct xbps_handle *xhp,
  */
 struct xbps_repo *xbps_repo_open(struct xbps_handle *xhp, const char *url);
 
-/**
- * Returns a proplib dictionary associated with a repository object.
- *
- * @param[in] repo Pointer to the xbps_repo object.
- * @param[in] file Filename of proplib dictionary stored in the
- * repository object.
- *
- * @return The matching proplib dictionary on success, NULL otherwise.
- */
-xbps_dictionary_t xbps_repo_get_plist(struct xbps_repo *repo, const char *file);
-
 /**
  * Closes a repository object and releases resources.
  *
@@ -1291,6 +1281,14 @@ xbps_dictionary_t xbps_repo_get_plist(struct xbps_repo *repo, const char *file);
  */
 void xbps_repo_close(struct xbps_repo *repo);
 
+/**
+ * Prepares the repository index-files.plist to have access to it.
+ * The repository must be opened previously with \a xbps_repo_open().
+ *
+ * @param[in] repo The repository object to use.
+ */
+void xbps_repo_open_idxfiles(struct xbps_repo *repo);
+
 /**
  *
  * Returns a heap-allocated string with the repository local path.

lib/repo.c

@@ -47,6 +47,47 @@ xbps_repo_path(struct xbps_handle *xhp, const char *url)
 	    url, xhp->target_arch ? xhp->target_arch : xhp->native_arch);
 }
 
+static xbps_dictionary_t
+repo_get_dict(struct xbps_repo *repo, const char *fname)
+{
+	xbps_dictionary_t d;
+	struct archive_entry *entry;
+	void *buf;
+	size_t buflen;
+	ssize_t nbytes = -1;
+	int rv;
+
+	assert(repo);
+	assert(fname);
+
+	if (repo->ar == NULL)
+		return NULL;
+
+	for (;;) {
+		rv = archive_read_next_header(repo->ar, &entry);
+		if (rv == ARCHIVE_EOF || rv == ARCHIVE_FATAL)
+			break;
+		else if (rv == ARCHIVE_RETRY)
+			continue;
+
+		if (strcmp(archive_entry_pathname(entry), fname) == 0) {
+			buflen = (size_t)archive_entry_size(entry);
+			buf = malloc(buflen);
+			assert(buf);
+			nbytes = archive_read_data(repo->ar, buf, buflen);
+			if ((size_t)nbytes != buflen) {
+				free(buf);
+				return NULL;
+			}
+			d = xbps_dictionary_internalize(buf);
+			free(buf);
+			return d;
+		}
+		archive_read_data_skip(repo->ar);
+	}
+	return NULL;
+}
+
 struct xbps_repo *
 xbps_repo_open(struct xbps_handle *xhp, const char *url)
 {
@@ -54,6 +95,7 @@ xbps_repo_open(struct xbps_handle *xhp, const char *url)
 	struct stat st;
 	const char *arch;
 	char *repofile;
+	bool is_remote = false;
 
 	assert(xhp);
 	assert(url);
@@ -71,20 +113,22 @@ xbps_repo_open(struct xbps_handle *xhp, const char *url)
 			return NULL;
 		repofile = xbps_xasprintf("%s/%s/%s-repodata", xhp->metadir, rpath, arch);
 		free(rpath);
+		is_remote = true;
 	} else {
 		/* local repository */
 		repofile = xbps_repo_path(xhp, url);
 	}
 
-	repo = calloc(1, sizeof(struct xbps_repo));
+	repo = malloc(sizeof(struct xbps_repo));
 	assert(repo);
 
 	repo->xhp = xhp;
 	repo->uri = url;
 	repo->ar = archive_read_new();
+	repo->is_verified = false;
+	repo->is_signed = false;
+	repo->is_remote = is_remote;
 	archive_read_support_compression_gzip(repo->ar);
-	archive_read_support_compression_bzip2(repo->ar);
-	archive_read_support_compression_xz(repo->ar);
 	archive_read_support_format_tar(repo->ar);
 
 	if (stat(repofile, &st) == -1) {
@@ -102,50 +146,30 @@ xbps_repo_open(struct xbps_handle *xhp, const char *url)
 		archive_read_finish(repo->ar);
 		free(repo);
 		repo = NULL;
+		goto out;
 	}
+	if ((repo->idx = repo_get_dict(repo, XBPS_REPOIDX)) == NULL) {
+		xbps_dbg_printf(xhp,
+		    "[repo] `%s' failed to internalize index on archive %s: %s\n",
+		    url, repofile, strerror(archive_errno(repo->ar)));
+		archive_read_finish(repo->ar);
+		free(repo);
+		repo = NULL;
+		goto out;
+	}
+	if ((repo->meta = repo_get_dict(repo, XBPS_REPOIDX_META)))
+		repo->is_signed = true;
+
+	repo->idxfiles = NULL;
 out:
 	free(repofile);
 	return repo;
 }
 
-xbps_dictionary_t
+void
-xbps_repo_get_plist(struct xbps_repo *repo, const char *file)
+xbps_repo_open_idxfiles(struct xbps_repo *repo)
 {
-	xbps_dictionary_t d;
+	repo->idxfiles = repo_get_dict(repo, XBPS_REPOIDX_FILES);
-	struct archive_entry *entry;
-	void *buf;
-	size_t buflen;
-	ssize_t nbytes = -1;
-	int rv;
-
-	assert(repo);
-	assert(file);
-
-	if (repo->ar == NULL)
-		return NULL;
-
-	for (;;) {
-		rv = archive_read_next_header(repo->ar, &entry);
-		if (rv == ARCHIVE_EOF || rv == ARCHIVE_FATAL)
-			break;
-		else if (rv == ARCHIVE_RETRY)
-			continue;
-		if (strcmp(archive_entry_pathname(entry), file) == 0) {
-			buflen = (size_t)archive_entry_size(entry);
-			buf = malloc(buflen);
-			assert(buf);
-			nbytes = archive_read_data(repo->ar, buf, buflen);
-			if ((size_t)nbytes != buflen) {
-				free(buf);
-				return NULL;
-			}
-			d = xbps_dictionary_internalize(buf);
-			free(buf);
-			return d;
-		}
-		archive_read_data_skip(repo->ar);
-	}
-	return NULL;
 }
 
 void

lib/repo_keys.c

@@ -39,7 +39,7 @@
 int HIDDEN
 xbps_repo_key_import(struct xbps_repo *repo)
 {
-	xbps_dictionary_t repokeyd, rkeysd = NULL, newmetad = NULL;
+	xbps_dictionary_t repokeyd, newmetad = NULL;
 	xbps_data_t rpubkey;
 	const char *signedby;
 	unsigned char *fp;
@@ -52,11 +52,27 @@ xbps_repo_key_import(struct xbps_repo *repo)
 	 */
 	if (xbps_dictionary_count(repo->meta) == 0) {
 		xbps_dbg_printf(repo->xhp,
-		    "[repo] `%s' missing required metadata, ignoring.\n", repo->uri);
+		    "[repo] `%s' unsigned repository!\n", repo->uri);
-		repo->is_verified = false;
-		repo->is_signed = false;
 		return 0;
 	}
+	/*
+	 * Check if the public key has been stored for this repository.
+	 */
+	if (repo->xhp->repokeys == NULL) {
+		rkeypath = xbps_xasprintf("%s/%s", repo->xhp->metadir, XBPS_REPOKEYS);
+		repo->xhp->repokeys = xbps_dictionary_internalize_from_file(rkeypath);
+		if (xbps_object_type(repo->xhp->repokeys) != XBPS_TYPE_DICTIONARY)
+			repo->xhp->repokeys = xbps_dictionary_create();
+	}
+	repokeyd = xbps_dictionary_get(repo->xhp->repokeys, repo->uri);
+	if (xbps_object_type(repokeyd) == XBPS_TYPE_DICTIONARY) {
+		if (xbps_dictionary_get(repokeyd, "public-key")) {
+			xbps_dbg_printf(repo->xhp,
+			    "[repo] `%s' public key already stored.\n",
+			    repo->uri);
+			goto out;
+		}
+	}
 	/*
 	 * Check the repository provides a working public-key data object.
 	 */
@@ -65,28 +81,9 @@ xbps_repo_key_import(struct xbps_repo *repo)
 		rv = EINVAL;
 		xbps_dbg_printf(repo->xhp,
 		    "[repo] `%s' invalid public-key object!\n", repo->uri);
-		repo->is_verified = false;
-		repo->is_signed = false;
 		goto out;
 	}
 	repo->is_signed = true;
-	/*
-	 * Check if the public key has been stored for this repository.
-	 */
-	rkeypath = xbps_xasprintf("%s/%s", repo->xhp->metadir, XBPS_REPOKEYS);
-	rkeysd = xbps_dictionary_internalize_from_file(rkeypath);
-	if (xbps_object_type(rkeysd) != XBPS_TYPE_DICTIONARY)
-		rkeysd = xbps_dictionary_create();
-
-	repokeyd = xbps_dictionary_get(rkeysd, repo->uri);
-	if (xbps_object_type(repokeyd) == XBPS_TYPE_DICTIONARY) {
-		if (xbps_dictionary_get(repokeyd, "public-key")) {
-			xbps_dbg_printf(repo->xhp,
-			    "[repo] `%s' public key already stored.\n",
-			    repo->uri);
-			goto out;
-		}
-	}
 	/*
 	 * Notify the client and take appropiate action to import
 	 * the repository public key. Pass back the public key openssh fingerprint
@@ -99,14 +96,16 @@ xbps_repo_key_import(struct xbps_repo *repo)
 			"This repository is RSA signed by \"%s\"",
 			signedby);
 	free(fp);
-	if (import <= 0)
+	if (import <= 0) {
+		rv = EAGAIN;
 		goto out;
+	}
 	/*
 	 * Add the meta dictionary into XBPS_REPOKEYS and externalize it.
 	 */
 	newmetad = xbps_dictionary_copy_mutable(repo->meta);
 	xbps_dictionary_remove(newmetad, "signature");
-	xbps_dictionary_set(rkeysd, repo->uri, newmetad);
+	xbps_dictionary_set(repo->xhp->repokeys, repo->uri, newmetad);
 
 	if (access(repo->xhp->metadir, R_OK|W_OK) == -1) {
 		if (errno == ENOENT) {
@@ -119,7 +118,7 @@ xbps_repo_key_import(struct xbps_repo *repo)
 			goto out;
 		}
 	}
-	if (!xbps_dictionary_externalize_to_file(rkeysd, rkeypath)) {
+	if (!xbps_dictionary_externalize_to_file(repo->xhp->repokeys, rkeypath)) {
 		rv = errno;
 		xbps_dbg_printf(repo->xhp,
 		    "[repo] `%s' failed to externalize %s: %s\n",
@@ -129,8 +128,7 @@ xbps_repo_key_import(struct xbps_repo *repo)
 out:
 	if (newmetad)
 		xbps_object_release(newmetad);
-	if (xbps_object_type(rkeysd) == XBPS_TYPE_DICTIONARY)
+	if (rkeypath)
-		xbps_object_release(rkeysd);
 		free(rkeypath);
 	return rv;
 }
@@ -184,34 +182,18 @@ rsa_verify_buf(struct xbps_repo *repo, xbps_data_t sigdata,
 int HIDDEN
 xbps_repo_key_verify(struct xbps_repo *repo)
 {
-	xbps_dictionary_t rkeysd, repokeyd;
+	xbps_dictionary_t repokeyd;
 	xbps_data_t sigdata, pubkey;
-	char *rkeyspath, *idx_xml;
+	char *idx_xml;
-	bool verified = false;
 
-	/* unsigned repo */
+	if (repo->xhp->repokeys == NULL)
-	if (!repo->is_signed) {
+		return ENOENT;
-		xbps_dbg_printf(repo->xhp,
-		    "[repo] `%s' ignoring unsigned repository.\n", repo->uri);
-		return 0;
-	}
-	rkeyspath = xbps_xasprintf("%s/%s", repo->xhp->metadir, XBPS_REPOKEYS);
-	rkeysd = xbps_dictionary_internalize_from_file(rkeyspath);
-	if (xbps_dictionary_count(rkeysd) == 0) {
-		xbps_dbg_printf(repo->xhp,
-		    "[repo] `%s': failed to internalize %s: %s\n",
-		    repo->uri, rkeyspath, strerror(errno));
-		free(rkeyspath);
-		return ENODEV;
-	}
-	free(rkeyspath);
 
-	repokeyd = xbps_dictionary_get(rkeysd, repo->uri);
+	repokeyd = xbps_dictionary_get(repo->xhp->repokeys, repo->uri);
 	if (xbps_dictionary_count(repokeyd) == 0) {
 		xbps_dbg_printf(repo->xhp,
 		    "[repo] `%s': empty %s dictionary\n",
 		    repo->uri, XBPS_REPOKEYS);
-		xbps_object_release(rkeysd);
 		return ENOENT;
 	}
 
@@ -223,11 +205,10 @@ xbps_repo_key_verify(struct xbps_repo *repo)
 	pubkey = xbps_dictionary_get(repokeyd, "public-key");
 	assert(xbps_object_type(pubkey) == XBPS_TYPE_DATA);
 	/* XXX ignore 'signature-type' for now */
-	if (rsa_verify_buf(repo, sigdata, pubkey, idx_xml) == 0) {
+	if (rsa_verify_buf(repo, sigdata, pubkey, idx_xml) == 0)
 		repo->is_verified = true;
-		verified = true;
+
-	}
 	free(idx_xml);
 
-	return verified ? 0 : EPERM;
+	return repo->is_verified ? 0 : EPERM;
 }

lib/rpool.c

@@ -52,6 +52,7 @@ xbps_rpool_init(struct xbps_handle *xhp)
 {
 	struct rpool *rp;
 	const char *repouri;
+	char *p;
 	bool foundrepo = false;
 	int retval, rv = 0;
 
@@ -60,40 +61,44 @@ xbps_rpool_init(struct xbps_handle *xhp)
 	if (xhp->rpool_initialized)
 		return 0;
 
+	p = xbps_xasprintf("%s/%s", xhp->metadir, XBPS_REPOKEYS);
+	xhp->repokeys = xbps_dictionary_internalize_from_file(p);
+	free(p);
+
 	for (unsigned int i = 0; i < xbps_array_count(xhp->repositories); i++) {
 		rp = malloc(sizeof(struct rpool));
 		assert(rp);
 		xbps_array_get_cstring_nocopy(xhp->repositories, i, &repouri);
 		if ((rp->repo = xbps_repo_open(xhp, repouri)) == NULL) {
-			rp->repo = malloc(sizeof(struct xbps_repo));
+			rp->repo = calloc(1, sizeof(struct xbps_repo));
 			assert(rp->repo);
-			rp->repo->ar = NULL;
-			rp->repo->is_verified = false;
-			rp->repo->is_signed = false;
-		}
-		rp->repo->idx = xbps_repo_get_plist(rp->repo, XBPS_REPOIDX);
-		if (xbps_object_type(rp->repo->idx) == XBPS_TYPE_DICTIONARY)
-			xbps_dictionary_make_immutable(rp->repo->idx);
-
-		rp->repo->meta = xbps_repo_get_plist(rp->repo, XBPS_REPOMETA);
-		if (xbps_object_type(rp->repo->meta) == XBPS_TYPE_DICTIONARY)
-			xbps_dictionary_make_immutable(rp->repo->meta);
-
-		rp->repo->uri = repouri;
 			rp->repo->xhp = xhp;
-
+			rp->repo->uri = repouri;
-		if (xbps_repository_is_remote(repouri)) {
+			if (xbps_repository_is_remote(repouri))
+				rp->repo->is_remote = true;
+		}
+		if (rp->repo->is_remote) {
 			/*
 			 * Import the RSA public key (if it's signed).
 			 */
 			retval = xbps_repo_key_import(rp->repo);
-			if (retval != 0) {
+			if (retval == EAGAIN) {
+				/* signed but public key was not imported */
+				xbps_dbg_printf(xhp, "[rpool] `%s': public-key not yet imported.\n", repouri);
+				rp->repo->is_signed = true;
+				rp->repo->is_verified = false;
+			} else if (retval != 0 && retval != EAGAIN) {
 				/* any error */
 				xbps_dbg_printf(xhp, "[rpool] %s: key_import %s\n",
-				    rp->repo->uri, strerror(retval));
+				    repouri, strerror(retval));
 			}
+			if (!rp->repo->is_signed) {
+				/* ignore unsigned repositories */
+				xbps_repo_close(rp->repo);
+			} else {
 				/*
-			 * Check the repository signature against stored public key.
+				 * Check the repository index signature against
+				 * stored public key.
 				 */
 				retval = xbps_repo_key_verify(rp->repo);
 				if (retval == 0) {
@@ -103,13 +108,15 @@ xbps_rpool_init(struct xbps_handle *xhp)
 					/* signed, unverified */
 					xbps_set_cb_state(xhp, XBPS_STATE_REPO_SIGUNVERIFIED, 0, NULL, NULL);
 					xbps_repo_close(rp->repo);
+					rp->repo->is_verified = false;
 				} else {
 					/* any error */
 					xbps_dbg_printf(xhp, "[rpool] %s: key_verify %s\n",
-				    rp->repo->uri, strerror(retval));
+					    repouri, strerror(retval));
 					xbps_repo_close(rp->repo);
 				}
 			}
+		}
 		/*
 		 * If repository has passed signature checks, add it to the pool.
 		 */
@@ -148,6 +155,8 @@ xbps_rpool_release(struct xbps_handle *xhp)
 		free(rp->repo);
 		free(rp);
 	}
+	xbps_object_release(xhp->repokeys);
+	xhp->repokeys = NULL;
 	xhp->rpool_initialized = false;
 	xbps_dbg_printf(xhp, "[rpool] released ok.\n");
 }