Introduce xbps_array_foreach_cb() and use it in random code.

This routine will spawn a thread per core to process N items stored
in the specified array, the last thread gets the remainder of items left.

Results have shown that xbps benefits if there is a considerable amount
of items and number of threads being spawned.

Use it in xbps_pkgdb_foreach_cb(), xbps-pkgdb(8), xbps-query(8)
and xbps-rindex(8).

On UP systems there's no overhead because pthread(3) is not used at all.

WIP! investigate if it can be used in libxbps (xbps_rpool_foreach()),
and finish conversion of xbps-rindex(8) -c.

bin/xbps-query/defs.h

@@ -59,8 +59,8 @@ int	repo_ownedby(struct xbps_handle *, int, char **);
 /* From list.c */
 unsigned int	find_longest_pkgver(struct xbps_handle *, xbps_object_t);
 
-int	list_pkgs_in_dict(struct xbps_handle *, xbps_object_t, void *, bool *);
-int	list_manual_pkgs(struct xbps_handle *, xbps_object_t, void *, bool *);
+int	list_pkgs_in_dict(struct xbps_handle *, xbps_object_t, const char *, void *, bool *);
+int	list_manual_pkgs(struct xbps_handle *, xbps_object_t, const char *, void *, bool *);
 int	list_orphans(struct xbps_handle *);
 int	list_pkgs_pkgdb(struct xbps_handle *);
 

bin/xbps-query/list.c

@@ -41,6 +41,7 @@ struct list_pkgver_cb {
 int
 list_pkgs_in_dict(struct xbps_handle *xhp,
 		  xbps_object_t obj,
+		  const char *key,
 		  void *arg,
 		  bool *loop_done)
 {
@@ -51,6 +52,7 @@ list_pkgs_in_dict(struct xbps_handle *xhp,
 	pkg_state_t state;
 
 	(void)xhp;
+	(void)key;
 	(void)loop_done;
 
 	xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
@@ -93,6 +95,7 @@ list_pkgs_in_dict(struct xbps_handle *xhp,
 int
 list_manual_pkgs(struct xbps_handle *xhp,
 		 xbps_object_t obj,
+		 const char *key,
 		 void *arg,
 		 bool *loop_done)
 {
@@ -100,6 +103,7 @@ list_manual_pkgs(struct xbps_handle *xhp,
 	bool automatic = false;
 
 	(void)xhp;
+	(void)key;
 	(void)arg;
 	(void)loop_done;
 
@@ -176,23 +180,19 @@ struct fflongest {
 static int
 _find_longest_pkgver_cb(struct xbps_handle *xhp,
 			xbps_object_t obj,
+			const char *key,
 			void *arg,
 			bool *loop_done)
 {
 	struct fflongest *ffl = arg;
-	xbps_dictionary_t pkgd;
 	const char *pkgver;
 	unsigned int len;
 
 	(void)xhp;
+	(void)key;
 	(void)loop_done;
 
-	if (xbps_object_type(obj) == XBPS_TYPE_DICT_KEYSYM)
-		pkgd = xbps_dictionary_get_keysym(ffl->d, obj);
-	else
-		pkgd = obj;
-
-	xbps_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver);
+	xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
 	len = strlen(pkgver);
 	if (ffl->len == 0 || len > ffl->len)
 		ffl->len = len;
@@ -212,7 +212,7 @@ find_longest_pkgver(struct xbps_handle *xhp, xbps_object_t o)
 		xbps_array_t array;
 
 		array = xbps_dictionary_all_keys(o);
-		(void)xbps_callback_array_iter(xhp, array,
+		(void)xbps_array_foreach_cb(xhp, array, o,
 		    _find_longest_pkgver_cb, &ffl);
 		xbps_object_release(array);
 	} else {

bin/xbps-query/ownedby.c

@@ -31,14 +31,18 @@
 #include <fnmatch.h>
 #include <dirent.h>
 #include <assert.h>
+#include <pthread.h>
 
 #include <xbps.h>
 #include "defs.h"
 
 struct ffdata {
+	pthread_mutex_t mtx;
 	int npatterns;
 	char **patterns;
 	const char *repouri;
+	xbps_array_t allkeys;
+	xbps_dictionary_t filesd;
 };
 
 static void
@@ -83,7 +87,7 @@ match_files_by_pattern(xbps_dictionary_t pkg_filesd,
 }
 
 static int
-ownedby_pkgdb_cb(struct xbps_handle *xhp, xbps_object_t obj, void *arg, bool *done)
+ownedby_pkgdb_cb(struct xbps_handle *xhp, xbps_object_t obj, const char *obj_key, void *arg, bool *done)
 {
 	xbps_dictionary_t pkgmetad;
 	xbps_array_t files_keys;
@@ -91,9 +95,13 @@ ownedby_pkgdb_cb(struct xbps_handle *xhp, xbps_object_t obj, void *arg, bool *do
 	unsigned int i;
 	const char *pkgver;
 
+	(void)obj_key;
 	(void)done;
 
 	xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+
+	pthread_mutex_lock(&ffd->mtx);
+
 	pkgmetad = xbps_pkgdb_get_pkg_metadata(xhp, pkgver);
 	assert(pkgmetad);
 
@@ -105,6 +113,8 @@ ownedby_pkgdb_cb(struct xbps_handle *xhp, xbps_object_t obj, void *arg, bool *do
 	xbps_object_release(pkgmetad);
 	xbps_object_release(files_keys);
 
+	pthread_mutex_unlock(&ffd->mtx);
+
 	return 0;
 }
 
@@ -117,6 +127,7 @@ ownedby(struct xbps_handle *xhp, int npatterns, char **patterns)
 
 	ffd.npatterns = npatterns;
 	ffd.patterns = patterns;
+	pthread_mutex_init(&ffd.mtx, NULL);
 
 	for (i = 0; i < npatterns; i++) {
 		rfile = realpath(patterns[i], NULL);
@@ -126,52 +137,51 @@ ownedby(struct xbps_handle *xhp, int npatterns, char **patterns)
 	return xbps_pkgdb_foreach_cb(xhp, ownedby_pkgdb_cb, &ffd);
 }
 
-static void
-repo_match_files_by_pattern(xbps_array_t files,
-			const char *pkgver,
-			struct ffdata *ffd)
+static int
+repo_match_cb(struct xbps_handle *xhp, xbps_object_t obj, const char *key, void *arg, bool *done)
 {
+	struct ffdata *ffd = arg;
 	const char *filestr;
 	unsigned int i;
 	int x;
 
-	for (i = 0; i < xbps_array_count(files); i++) {
-		xbps_array_get_cstring_nocopy(files, i, &filestr);
+	(void)xhp;
+	(void)done;
+
+	for (i = 0; i < xbps_array_count(obj); i++) {
+		xbps_array_get_cstring_nocopy(obj, i, &filestr);
 		for (x = 0; x < ffd->npatterns; x++) {
 			if ((fnmatch(ffd->patterns[x], filestr, FNM_PERIOD)) == 0) {
 				printf("%s: %s (%s)\n",
-				    pkgver, filestr, ffd->repouri);
+				    key, filestr, ffd->repouri);
 			}
 		}
 	}
+
+	return 0;
 }
 
 static int
 repo_ownedby_cb(struct xbps_repo *repo, void *arg, bool *done)
 {
-	xbps_array_t allkeys, pkgar;
+	xbps_array_t allkeys;
 	xbps_dictionary_t filesd;
-	xbps_dictionary_keysym_t ksym;
 	struct ffdata *ffd = arg;
-	const char *pkgver;
-	unsigned int i;
+	int rv;
 
 	(void)done;
 
 	filesd = xbps_repo_get_plist(repo, XBPS_PKGINDEX_FILES);
+	if (filesd == NULL)
+		return 0;
+
 	ffd->repouri = repo->uri;
 	allkeys = xbps_dictionary_all_keys(filesd);
-
-	for (i = 0; i < xbps_array_count(allkeys); i++) {
-		ksym = xbps_array_get(allkeys, i);
-		pkgar = xbps_dictionary_get_keysym(filesd, ksym);
-		pkgver = xbps_dictionary_keysym_cstring_nocopy(ksym);
-		repo_match_files_by_pattern(pkgar, pkgver, ffd);
-	}
+	rv = xbps_array_foreach_cb(repo->xhp, allkeys, filesd, repo_match_cb, ffd);
 	xbps_object_release(filesd);
 	xbps_object_release(allkeys);
 
-	return 0;
+	return rv;
 }
 
 int
@@ -179,8 +189,9 @@ repo_ownedby(struct xbps_handle *xhp, int npatterns, char **patterns)
 {
 	struct ffdata ffd;
 	char *rfile;
-	int i;
+	int i, rv;
 
+	pthread_mutex_init(&ffd.mtx, NULL);
 	ffd.npatterns = npatterns;
 	ffd.patterns = patterns;
 
@@ -189,5 +200,8 @@ repo_ownedby(struct xbps_handle *xhp, int npatterns, char **patterns)
 		if (rfile)
 			patterns[i] = rfile;
 	}
-	return xbps_rpool_foreach(xhp, repo_ownedby_cb, &ffd);
+	rv = xbps_rpool_foreach(xhp, repo_ownedby_cb, &ffd);
+	pthread_mutex_destroy(&ffd.mtx);
+
+	return rv;
 }

bin/xbps-query/search.c

@@ -40,6 +40,7 @@
 #include <libgen.h>
 #include <fnmatch.h>
 #include <assert.h>
+#include <pthread.h>
 
 #include <xbps.h>
 #include "defs.h"
@@ -48,6 +49,7 @@ struct search_data {
 	int npatterns;
 	char **patterns;
 	int maxcols;
+	pthread_mutex_t mtx;
 	xbps_array_t results;
 };
 
@@ -95,44 +97,52 @@ print_results(struct xbps_handle *xhp, struct search_data *sd)
 	}
 }
 
+static int
+search_array_cb(struct xbps_handle *xhp, xbps_object_t obj, const char *key, void *arg, bool *done)
+{
+	struct search_data *sd = arg;
+	const char *pkgver, *desc;
+	int x;
+
+	(void)xhp;
+	(void)key;
+	(void)done;
+
+	xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+	xbps_dictionary_get_cstring_nocopy(obj, "short_desc", &desc);
+
+	for (x = 0; x < sd->npatterns; x++) {
+		bool vpkgfound = false;
+
+		if (xbps_match_virtual_pkg_in_dict(obj, sd->patterns[x], false))
+			vpkgfound = true;
+
+		if ((xbps_pkgpattern_match(pkgver, sd->patterns[x])) ||
+		    (strcasestr(pkgver, sd->patterns[x])) ||
+		    (strcasestr(desc, sd->patterns[x])) || vpkgfound) {
+			pthread_mutex_lock(&sd->mtx);
+			xbps_array_add_cstring_nocopy(sd->results, pkgver);
+			xbps_array_add_cstring_nocopy(sd->results, desc);
+			pthread_mutex_unlock(&sd->mtx);
+		}
+	}
+	return 0;
+}
+
 static int
 search_pkgs_cb(struct xbps_repo *repo, void *arg, bool *done)
 {
 	xbps_array_t allkeys;
-	xbps_dictionary_t pkgd;
-	xbps_dictionary_keysym_t ksym;
 	struct search_data *sd = arg;
-	const char *pkgver, *desc;
-	unsigned int i;
-	int x;
+	int rv;
 
 	(void)done;
 
 	allkeys = xbps_dictionary_all_keys(repo->idx);
-	for (i = 0; i < xbps_array_count(allkeys); i++) {
-		ksym = xbps_array_get(allkeys, i);
-		pkgd = xbps_dictionary_get_keysym(repo->idx, ksym);
-
-		xbps_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver);
-		xbps_dictionary_get_cstring_nocopy(pkgd, "short_desc", &desc);
-
-		for (x = 0; x < sd->npatterns; x++) {
-			bool vpkgfound = false;
-
-			if (xbps_match_virtual_pkg_in_dict(pkgd, sd->patterns[x], false))
-				vpkgfound = true;
-
-			if ((xbps_pkgpattern_match(pkgver, sd->patterns[x])) ||
-			    (strcasestr(pkgver, sd->patterns[x])) ||
-			    (strcasestr(desc, sd->patterns[x])) || vpkgfound) {
-				xbps_array_add_cstring_nocopy(sd->results, pkgver);
-				xbps_array_add_cstring_nocopy(sd->results, desc);
-			}
-		}
-	}
+	rv = xbps_array_foreach_cb(repo->xhp, allkeys, repo->idx, search_array_cb, sd);
 	xbps_object_release(allkeys);
 
-	return 0;
+	return rv;
 }
 
 int
@@ -141,6 +151,7 @@ repo_search(struct xbps_handle *xhp, int npatterns, char **patterns)
 	struct search_data sd;
 	int rv;
 
+	pthread_mutex_init(&sd.mtx, NULL);
 	sd.npatterns = npatterns;
 	sd.patterns = patterns;
 	sd.maxcols = get_maxcols();
@@ -152,6 +163,8 @@ repo_search(struct xbps_handle *xhp, int npatterns, char **patterns)
 		    strerror(rv));
 
 	print_results(xhp, &sd);
+	pthread_mutex_destroy(&sd.mtx);
+	xbps_object_release(sd.results);
 
 	return rv;
 }