Fixed issue 21 "xbps-repo find-files is awfully slow" reported by ojab.

bin/xbps-repo/Makefile

@@ -3,6 +3,7 @@ TOPDIR = ../..
 
 BIN =	xbps-repo
 OBJS =	main.o index.o show.o find-files.o list.o
+OBJS += index-files.o
 OBJS += ../xbps-bin/fetch_cb.o ../xbps-bin/util.o
 OBJS += ../xbps-bin/state_cb.o ../xbps-bin/list.o
 MAN = $(BIN).8

bin/xbps-repo/defs.h

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2011 Juan Romero Pardines.
+ * Copyright (c) 2009-2012 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -41,6 +41,9 @@ struct repo_search_data {
 /* From index.c */
 int	repo_genindex(const char *);
 
+/* From index-files.c */
+int	repo_genindex_files(const char *);
+
 /* From find-files.c */
 int	repo_find_files_in_packages(int, char **);
 

bin/xbps-repo/find-files.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2010 Juan Romero Pardines.
+ * Copyright (c) 2010-2012 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,100 +37,81 @@ struct ffdata {
 	char **patterns;
 };
 
-static int
-match_files_by_pattern(prop_dictionary_t pkg_filesd,
-		       prop_dictionary_keysym_t key,
-		       struct ffdata *ffd,
-		       const char *pkgver)
+static void
+match_files_by_pattern(prop_dictionary_t pkg_filesd, struct ffdata *ffd)
 {
 	prop_object_iterator_t iter;
-	prop_array_t array;
+	prop_array_t array, allkeys;
 	prop_object_t obj;
-	const char *keyname, *filestr, *typestr;
-	int i;
+	prop_dictionary_keysym_t key;
+	const char *keyname, *filestr, *typestr, *pkgver;
+	size_t i;
+	int x;
 
-	keyname = prop_dictionary_keysym_cstring_nocopy(key);
-	array = prop_dictionary_get_keysym(pkg_filesd, key);
-	if (prop_object_type(array) != PROP_TYPE_ARRAY)
-		return 0;
+	allkeys = prop_dictionary_all_keys(pkg_filesd);
+	for (i = 0; i < prop_array_count(allkeys); i++) {
+		key = prop_array_get(allkeys, i);
+		keyname = prop_dictionary_keysym_cstring_nocopy(key);
+		array = prop_dictionary_get_keysym(pkg_filesd, key);
+		if (prop_object_type(array) != PROP_TYPE_ARRAY)
+			break;
 
-	if (strcmp(keyname, "files") == 0)
-		typestr = "regular file";
-	else if (strcmp(keyname, "dirs") == 0)
-		typestr = "directory";
-	else if (strcmp(keyname, "links") == 0)
-		typestr = "link";
-	else
-		typestr = "configuration file";
+		if (strcmp(keyname, "files") == 0)
+			typestr = "regular file";
+		else if (strcmp(keyname, "links") == 0)
+			typestr = "link";
+		else
+			typestr = "configuration file";
 
-	iter = prop_array_iterator(array);
-	while ((obj = prop_object_iterator_next(iter))) {
-		prop_dictionary_get_cstring_nocopy(obj, "file", &filestr);
-		for (i = 1; i < ffd->npatterns; i++) {
-			if ((strcmp(filestr, ffd->patterns[i]) == 0) ||
-			    (strstr(filestr, ffd->patterns[i])) ||
-			    (xbps_pkgpattern_match(filestr,
-						   ffd->patterns[i]) == 1))
-				printf(" %s: %s (%s)\n", pkgver, filestr, typestr);
+		iter = prop_array_iterator(array);
+		while ((obj = prop_object_iterator_next(iter))) {
+			prop_dictionary_get_cstring_nocopy(obj, "file", &filestr);
+			for (x = 1; x < ffd->npatterns; x++) {
+				if ((strcmp(filestr, ffd->patterns[x]) == 0) ||
+				    (strstr(filestr, ffd->patterns[x])) ||
+				     (xbps_pkgpattern_match(filestr,
+				      ffd->patterns[x]) == 1)) {
+					prop_dictionary_get_cstring_nocopy(
+					    pkg_filesd, "pkgver", &pkgver);
+					printf(" %s: %s (%s)\n",
+					    pkgver, filestr, typestr);
+				}
+			}
 		}
+		prop_object_iterator_release(iter);
 	}
-	prop_object_iterator_release(iter);
-	return 0;
+	prop_object_release(allkeys);
 }
 
 static int
 find_files_in_package(struct repository_pool_index *rpi, void *arg, bool *done)
 {
-	prop_dictionary_t pkg_filesd;
+	prop_dictionary_t idxfilesd;
 	prop_array_t files_keys;
-	prop_object_t obj;
-	prop_object_iterator_t iter;
 	struct ffdata *ffd = arg;
-	const char *pkgname, *pkgver;
-	char *url;
-	int rv = 0;
-	unsigned int i, count;
+	char *plist;
+	unsigned int i;
 
 	(void)done;
 
-	iter = xbps_array_iter_from_dict(rpi->rpi_repod, "packages");
-	if (iter == NULL)
-		return -1;
-
 	printf("Looking in repository '%s', please wait...\n", rpi->rpi_uri);
-	while ((obj = prop_object_iterator_next(iter))) {
-		url = xbps_path_from_repository_uri(obj, rpi->rpi_uri);
-		if (url == NULL) {
-			rv = -1;
-			break;
-		}
-		prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
-		prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
-		pkg_filesd = xbps_dictionary_metadata_plist_by_url(url,
-		    XBPS_PKGFILES);
-		free(url);
-		if (pkg_filesd == NULL) {
-			xbps_error_printf("xbps-repo: couldn't read '%s' "
-			    "from '%s (%s)': %s\n", XBPS_PKGFILES, pkgname,
-			    rpi->rpi_uri, strerror(errno));
-			rv = -1;
-			break;
-		}
-		files_keys = prop_dictionary_all_keys(pkg_filesd);
-		count = prop_array_count(files_keys);
-		for (i = 0; i < count; i++) {
-			rv = match_files_by_pattern(pkg_filesd,
-			    prop_array_get(files_keys, i), ffd, pkgver);
-			if (rv == -1)
-				break;
-		}
-		prop_object_release(files_keys);
-		prop_object_release(pkg_filesd);
-		if (rv == -1)
-			break;
+	plist = xbps_pkg_index_files_plist(rpi->rpi_uri);
+	if (plist == NULL)
+		return ENOMEM;
+
+	idxfilesd = prop_dictionary_internalize_from_zfile(plist);
+	if (idxfilesd == NULL) {
+		free(plist);
+		return errno;
 	}
-	prop_object_iterator_release(iter);
-	return rv;
+	free(plist);
+
+	files_keys = prop_dictionary_get(idxfilesd, "packages");
+	for (i = 0; i < prop_array_count(files_keys); i++)
+		match_files_by_pattern(prop_array_get(files_keys, i), ffd);
+
+	prop_object_release(idxfilesd);
+	return 0;
 }
 
 int

bin/xbps-repo/index-files.c

@@ -0,0 +1,196 @@
+/*-
+ * Copyright (c) 2012 Juan Romero Pardines.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include <errno.h>
+
+#include <xbps_api.h>
+#include "defs.h"
+
+struct index_files_data {
+	prop_array_t idxfiles;
+	const char *pkgdir;
+};
+
+static int
+genindex_files_cb(prop_object_t obj, void *arg, bool *done)
+{
+	prop_dictionary_t pkg_filesd, pkgd;
+	prop_array_t array;
+	struct index_files_data *ifd = arg;
+	const char *binpkg, *pkgver;
+	char *file;
+	bool found = false;
+
+	(void)done;
+
+	prop_dictionary_get_cstring_nocopy(obj, "filename", &binpkg);
+	prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+
+	file = xbps_xasprintf("%s/%s", ifd->pkgdir, binpkg);
+	if (file == NULL)
+		return ENOMEM;
+
+	/* internalize files.plist from binary package archive */
+	pkg_filesd = xbps_dictionary_metadata_plist_by_url(file, XBPS_PKGFILES);
+	if (pkg_filesd == NULL) {
+		free(file);
+		return EINVAL;
+	}
+	free(file);
+
+	/* create pkg dictionary */
+	pkgd = prop_dictionary_create();
+	if (pkgd == NULL) {
+		prop_object_release(pkg_filesd);
+		return ENOMEM;
+	}
+	/* add conf_files array in pkgd */
+	array = prop_dictionary_get(pkg_filesd, "conf_files");
+	if (array != NULL && prop_array_count(array)) {
+		found = true;
+		if (!prop_dictionary_set(pkgd, "conf_files", array)) {
+			prop_object_release(pkgd);
+			prop_object_release(pkg_filesd);
+			return EINVAL;
+		}
+	}
+	/* add files array in pkgd */
+	array = prop_dictionary_get(pkg_filesd, "files");
+	if (array != NULL && prop_array_count(array)) {
+		found = true;
+		if (!prop_dictionary_set(pkgd, "files", array)) {
+			prop_object_release(pkgd);
+			prop_object_release(pkg_filesd);
+			return EINVAL;
+		}
+	}
+	/* add links array in pkgd */
+	array = prop_dictionary_get(pkg_filesd, "links");
+	if (array != NULL && prop_array_count(array)) {
+		found = true;
+		if (!prop_dictionary_set(pkgd, "links", array)) {
+			prop_object_release(pkgd);
+			prop_object_release(pkg_filesd);
+			return EINVAL;
+		}
+	}
+	prop_object_release(pkg_filesd);
+	if (!found) {
+		prop_object_release(pkgd);
+		return 0;
+	}
+	/* pkgver obj in pkgd */
+	if (!prop_dictionary_set_cstring(pkgd, "pkgver", pkgver)) {
+		prop_object_release(pkgd);
+		return EINVAL;
+	}
+
+	/* add pkgd into provided array */
+	if (!prop_array_add(ifd->idxfiles, pkgd)) {
+		prop_object_release(pkgd);
+		return EINVAL;
+	}
+	prop_object_release(pkgd);
+
+	return 0;
+}
+
+/*
+ * Create the index files cache for all packages in repository.
+ */
+int
+repo_genindex_files(const char *pkgdir)
+{
+	prop_dictionary_t idxdict, idxfilesd;
+	struct index_files_data *ifd;
+	char *plist, *files_plist;
+	int rv;
+
+	plist = xbps_pkg_index_plist(pkgdir);
+	if (plist == NULL)
+		return ENOMEM;
+
+	/* internalize repository index plist */
+	idxdict = prop_dictionary_internalize_from_zfile(plist);
+	if (idxdict == NULL) {
+		free(plist);
+		return errno;
+	}
+
+	ifd = malloc(sizeof(*ifd));
+	if (ifd == NULL) {
+		prop_object_release(idxdict);
+		free(plist);
+		return ENOMEM;
+	}
+	ifd->pkgdir = pkgdir;
+	ifd->idxfiles = prop_array_create();
+
+	printf("Creating repository's index files cache...\n");
+
+	/* iterate over index.plist packages array */
+	rv = xbps_callback_array_iter_in_dict(idxdict,
+	    "packages", genindex_files_cb, ifd);
+	prop_object_release(idxdict);
+	free(plist);
+	if (rv != 0) {
+		prop_object_release(ifd->idxfiles);
+		free(ifd);
+		return rv;
+	}
+	idxfilesd = prop_dictionary_create();
+	/* add array into the index-files dictionary */
+	if (!prop_dictionary_set(idxfilesd, "packages", ifd->idxfiles)) {
+		prop_object_release(ifd->idxfiles);
+		prop_object_release(idxfilesd);
+		free(ifd);
+		return EINVAL;
+	}
+	files_plist = xbps_pkg_index_files_plist(pkgdir);
+	if (files_plist == NULL) {
+		prop_object_release(ifd->idxfiles);
+		prop_object_release(idxfilesd);
+		free(ifd);
+		return ENOMEM;
+	}
+	/* externalize index-files dictionary to the plist file */
+	if (!prop_dictionary_externalize_to_zfile(idxfilesd, files_plist)) {
+		free(files_plist);
+		prop_object_release(ifd->idxfiles);
+		prop_object_release(idxfilesd);
+		free(ifd);
+		return errno;
+	}
+	free(files_plist);
+	prop_object_release(idxfilesd);
+	prop_object_release(ifd->idxfiles);
+	free(ifd);
+
+	return 0;
+}

bin/xbps-repo/main.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2008-2011 Juan Romero Pardines.
+ * Copyright (c) 2008-2012 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -239,6 +239,9 @@ main(int argc, char **argv)
 			usage(xhp);
 
 		rv = repo_genindex(argv[1]);
+		if (rv == 0)
+			rv = repo_genindex_files(argv[1]);
+
 	} else if (strcasecmp(argv[0], "sync") == 0) {
 		/* Syncs the pkg index for all registered remote repos */
 		if (argc != 1)