xbps-repo: faster 'genindex' target implementation for index-files.plist.

bin/xbps-repo/index-files.c

@@ -33,18 +33,54 @@
 #include "defs.h"
 
 struct index_files_data {
+	prop_dictionary_t idxdict;
 	prop_array_t idxfiles;
 	const char *pkgdir;
+	bool flush;
+	bool new;
 };
 
+static int
+rmobsoletes_files_cb(prop_object_t obj, void *arg, bool *done)
+{
+	prop_array_t array;
+	struct index_files_data *ifd = arg;
+	const char *pkgver;
+	char *buf;
+
+	(void)done;
+
+	prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+	buf = strdup(pkgver);
+	if (buf == NULL)
+		return ENOMEM;
+
+	array = prop_dictionary_get(ifd->idxdict, "packages");
+	if (xbps_find_pkg_in_array_by_pkgver(array, pkgver)) {
+		/* pkg found, do nothing */
+		free(buf);
+		return 0;
+	}
+	/* remove obsolete pkg dictionary */
+	if (!xbps_remove_pkg_from_array_by_pkgver(ifd->idxfiles, pkgver)) {
+		free(buf);
+		return EINVAL;
+	}
+	printf("Removed obsolete entry for `%s'.\n", buf);
+	free(buf);
+	ifd->flush = true;
+
+	return 0;
+}
+
 static int
 genindex_files_cb(prop_object_t obj, void *arg, bool *done)
 {
-	prop_dictionary_t pkg_filesd, pkgd;
+	prop_dictionary_t pkg_filesd, pkgd, regpkgd;
 	prop_array_t array;
 	struct index_files_data *ifd = arg;
-	const char *binpkg, *pkgver;
+	const char *binpkg, *pkgver, *rpkgver, *version;
-	char *file;
+	char *file, *pkgname, *pattern;
 	bool found = false;
 
 	(void)done;
@@ -52,6 +88,42 @@ genindex_files_cb(prop_object_t obj, void *arg, bool *done)
 	prop_dictionary_get_cstring_nocopy(obj, "filename", &binpkg);
 	prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
 
+	if (ifd->new)
+		goto start;
+
+	pkgname = xbps_pkg_name(pkgver);
+	if (pkgname == NULL)
+		return ENOMEM;
+	version = xbps_pkg_version(pkgver);
+	if (version == NULL) {
+		free(pkgname);
+		return EINVAL;
+	}
+	pattern = xbps_xasprintf("%s>=0", pkgname);
+	if (pattern == NULL) {
+		free(pkgname);
+		return ENOMEM;
+	}
+	free(pkgname);
+	regpkgd = xbps_find_pkg_in_array_by_pattern(ifd->idxfiles, pattern);
+	if (regpkgd) {
+		/*
+		 * pkg already registered, check if same version
+		 * is registered.
+		 */
+		prop_dictionary_get_cstring_nocopy(regpkgd, "pkgver", &rpkgver);
+		if (strcmp(pkgver, rpkgver) == 0) {
+			/* same pkg */
+			xbps_warn_printf("skipping `%s', already registered.\n",
+			    rpkgver);
+			return 0;
+		}
+		/* pkgver does not match, remove it from index-files */
+		if (!xbps_remove_pkg_from_array_by_pkgver(ifd->idxfiles, rpkgver))
+			return EINVAL;
+	}
+
+start:
 	file = xbps_xasprintf("%s/%s", ifd->pkgdir, binpkg);
 	if (file == NULL)
 		return ENOMEM;
@@ -117,6 +189,8 @@ genindex_files_cb(prop_object_t obj, void *arg, bool *done)
 		return EINVAL;
 	}
 	prop_object_release(pkgd);
+	ifd->flush = true;
+	printf("Registered `%s' in repository files index.\n", pkgver);
 
 	return 0;
 }
@@ -129,7 +203,7 @@ repo_genindex_files(const char *pkgdir)
 {
 	prop_dictionary_t idxdict;
 	struct index_files_data *ifd;
-	char *plist, *files_plist;
+	char *plist;
 	int rv;
 
 	plist = xbps_pkg_index_plist(pkgdir);
@@ -142,44 +216,63 @@ repo_genindex_files(const char *pkgdir)
 		free(plist);
 		return errno;
 	}
+	free(plist);
 
-	ifd = malloc(sizeof(*ifd));
+	/* internalize repository index-files plist (if exists) */
+	plist = xbps_pkg_index_files_plist(pkgdir);
+	if (plist == NULL) {
+		rv = ENOMEM;
+		goto out;
+	}
+	ifd = calloc(1, sizeof(*ifd));
 	if (ifd == NULL) {
-		prop_object_release(idxdict);
+		rv = ENOMEM;
-		free(plist);
+		goto out;
-		return ENOMEM;
 	}
 	ifd->pkgdir = pkgdir;
-	ifd->idxfiles = prop_array_create();
+	ifd->idxfiles = prop_array_internalize_from_zfile(plist);
-
+	ifd->idxdict = prop_dictionary_copy(idxdict);
-	printf("Creating repository's index files cache...\n");
+	if (ifd->idxfiles == NULL) {
+		/* missing file, create new one */
+		ifd->idxfiles = prop_array_create();
+		ifd->new = true;
+	}
 
 	/* iterate over index.plist packages array */
 	rv = xbps_callback_array_iter_in_dict(idxdict,
 	    "packages", genindex_files_cb, ifd);
-	prop_object_release(idxdict);
+	if (rv != 0)
-	free(plist);
+		goto out;
-	if (rv != 0) {
-		prop_object_release(ifd->idxfiles);
-		free(ifd);
-		return rv;
-	}
-	files_plist = xbps_pkg_index_files_plist(pkgdir);
-	if (files_plist == NULL) {
-		prop_object_release(ifd->idxfiles);
-		free(ifd);
-		return ENOMEM;
-	}
-	/* externalize index-files dictionary to the plist file */
-	if (!prop_array_externalize_to_zfile(ifd->idxfiles, files_plist)) {
-		free(files_plist);
-		prop_object_release(ifd->idxfiles);
-		free(ifd);
-		return errno;
-	}
-	free(files_plist);
-	prop_object_release(ifd->idxfiles);
-	free(ifd);
 
-	return 0;
+	/* remove obsolete pkg entries */
+	if (!ifd->new) {
+		rv = xbps_callback_array_iter(ifd->idxfiles,
+		    rmobsoletes_files_cb, ifd);
+		if (rv != 0)
+			goto out;
+	}
+	if (!ifd->flush)
+		goto out;
+
+	/* externalize index-files dictionary to the plist file */
+	if (!prop_array_externalize_to_zfile(ifd->idxfiles, plist)) {
+		rv = errno;
+		goto out;
+	}
+out:
+	if (rv == 0)
+		printf("%u packages registered in repository files index.\n",
+		    prop_array_count(ifd->idxfiles));
+	if (ifd->idxfiles != NULL)
+		prop_object_release(ifd->idxfiles);
+	if (ifd->idxdict != NULL)
+		prop_object_release(ifd->idxdict);
+	if (plist != NULL)
+		free(plist);
+	if (ifd != NULL)
+		free(ifd);
+	if (idxdict != NULL)
+		prop_object_release(idxdict);
+
+	return rv;
 }

bin/xbps-repo/index.c

@@ -160,8 +160,8 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 	prop_dictionary_t newpkgd, curpkgd;
 	prop_array_t pkgar;
 	struct stat st;
-	const char *pkgname, *version, *regver, *oldfilen;
+	const char *pkgname, *version, *regver, *oldfilen, *oldpkgver;
-	char *sha256, *filen, *tmpfilen, *tmpstr, *oldfilepath;
+	char *sha256, *filen, *tmpfilen, *oldfilepath;
 	int rv = 0;
 
 	if (idxdict == NULL || file == NULL)
@@ -179,8 +179,8 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 
 	newpkgd = xbps_dictionary_metadata_plist_by_url(file, XBPS_PKGPROPS);
 	if (newpkgd == NULL) {
-		xbps_error_printf("xbps-repo: can't read %s %s metadata "
+		xbps_error_printf("failed to read %s metadata for `%s',"
-		    "file, skipping!\n", file, XBPS_PKGPROPS);
+		    " skipping!\n", XBPS_PKGPROPS, file);
 		goto out;
 	}
 	prop_dictionary_get_cstring_nocopy(newpkgd, "pkgname", &pkgname);
@@ -201,7 +201,7 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 	} else if (curpkgd) {
 		prop_dictionary_get_cstring_nocopy(curpkgd, "version", &regver);
 		if (xbps_cmpver(version, regver) <= 0) {
-			xbps_warn_printf("skipping %s. %s-%s already "
+			xbps_warn_printf("skipping `%s', `%s-%s' already "
 			    "registered.\n", filen, pkgname, regver);
 			prop_object_release(newpkgd);
 			rv = EEXIST;
@@ -214,6 +214,8 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 		 */
 		prop_dictionary_get_cstring_nocopy(curpkgd,
 		    "filename", &oldfilen);
+		prop_dictionary_get_cstring_nocopy(curpkgd,
+		    "pkgver", &oldpkgver);
 		oldfilepath = xbps_xasprintf("%s/%s", filedir, oldfilen);
 		if (oldfilepath == NULL) {
 			prop_object_release(newpkgd);
@@ -221,7 +223,7 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 			goto out;
 		}
 		if (remove(oldfilepath) == -1) {
-			xbps_error_printf("xbps-repo: couldn't remove old "
+			xbps_error_printf("failed to remove old "
 			    "package file `%s': %s\n", oldfilepath,
 			    strerror(errno));
 			free(oldfilepath);
@@ -230,23 +232,15 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 			goto out;
 		}
 		free(oldfilepath);
-		tmpstr = strdup(oldfilen);
-		if (tmpstr == NULL) {
-			prop_object_release(newpkgd);
-			rv = errno;
-			goto out;
-		}
 		if (!xbps_remove_pkg_from_dict_by_name(idxdict,
 		    "packages", pkgname)) {
-			xbps_error_printf("xbps-repo: couldn't remove `%s' "
+			xbps_error_printf("failed to remove `%s' "
 			    "from plist index: %s\n", pkgname, strerror(errno));
 			prop_object_release(newpkgd);
-			free(tmpstr);
 			goto out;
 		}
-		xbps_warn_printf("xbps-repo: removed outdated binpkg file for "
+		printf("Removed obsole package entry/binpkg for `%s'.\n",
-		    "'%s'.\n", tmpstr);
+		    oldpkgver);
-		free(tmpstr);
 	}
 
 	/*
@@ -297,7 +291,7 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 		rv = EINVAL;
 		goto out;
 	}
-	printf("Registered %s-%s (%s) in package index.\n",
+	printf("Registered `%s-%s' (%s) in repository index.\n",
 	    pkgname, version, filen);
 
 	if (!prop_dictionary_set_uint64(idxdict, "total-pkgs",
@@ -380,7 +374,7 @@ repo_genindex(const char *pkgdir)
 		 * Show total count registered packages.
 		 */
 		prop_dictionary_get_uint64(idxdict, "total-pkgs", &npkgcnt);
-		printf("%ju packages registered in package index.\n", npkgcnt);
+		printf("%ju packages registered in repository index.\n", npkgcnt);
 		/*
 		 * Don't write plist file if no packages were registered.
 		 */

include/xbps_api.h

@@ -56,7 +56,7 @@
  */
 #define XBPS_PKGINDEX_VERSION	"1.3"
 
-#define XBPS_API_VERSION	"20120116-1"
+#define XBPS_API_VERSION	"20120117"
 #define XBPS_VERSION		"0.12"
 
 /**
@@ -984,7 +984,7 @@ bool xbps_match_any_virtualpkg_in_rundeps(prop_array_t rundeps,
  * @param[in] array The proplib array where to look for.
  * @param[in] name The package name to match.
  *
- * @return true on success, false otherwise and errno is set appropiately.
+ * @return The package dictionary, otherwise NULL is returned.
  */
 prop_dictionary_t xbps_find_pkg_in_array_by_name(prop_array_t array,
 						 const char *name);
@@ -995,11 +995,23 @@ prop_dictionary_t xbps_find_pkg_in_array_by_name(prop_array_t array,
  * @param[in] array The proplib array where to look for.
  * @param[in] pattern The package pattern to match.
  *
- * @return true on success, false otherwise and errno is set appropiately.
+ * @return The package dictionary, otherwise NULL is returned.
  */
 prop_dictionary_t xbps_find_pkg_in_array_by_pattern(prop_array_t array,
 						    const char *pattern);
 
+/**
+ * Finds a package dictionary in a proplib array by matching a \a pkgver
+ * object.
+ *
+ * @param[in] array The proplib array where to look for.
+ * @param[in] pkgver The package name/version tuple to match, i.e `foo-1.0'.
+ *
+ * @return The package dictionary, otherwise NULL is returned.
+ */
+prop_dictionary_t xbps_find_pkg_in_array_by_pkgver(prop_array_t array,
+						   const char *pkgver);
+
 /**
  * Finds a virtual package dictionary in a proplib array by matching a
  * package name.
@@ -1007,7 +1019,7 @@ prop_dictionary_t xbps_find_pkg_in_array_by_pattern(prop_array_t array,
  * @param[in] array The proplib array where to look for.
  * @param[in] name The virtual package name to match.
  *
- * @return true on success, false otherwise and errno is set appropiately.
+ * @return The package dictionary, otherwise NULL is returned.
  */
 prop_dictionary_t xbps_find_virtualpkg_in_array_by_name(prop_array_t array,
 							const char *name);
@@ -1019,7 +1031,7 @@ prop_dictionary_t xbps_find_virtualpkg_in_array_by_name(prop_array_t array,
  * @param[in] array The proplib array where to look for.
  * @param[in] pattern The virtual package pattern to match.
  *
- * @return true on success, false otherwise and errno is set appropiately.
+ * @return The package dictionary, otherwise NULL is returned.
  */
 prop_dictionary_t xbps_find_virtualpkg_in_array_by_pattern(prop_array_t array,
 							   const char *pattern);
@@ -1113,6 +1125,17 @@ bool xbps_remove_pkg_dict_from_plist_by_name(const char *name,
  */
 bool xbps_remove_pkg_from_array_by_name(prop_array_t array, const char *name);
 
+/**
+ * Removes the package's proplib dictionary matching the \a pkgver
+ * object in a proplib array of dictionaries.
+ *
+ * @param[in] array Proplib array where to look for.
+ * @param[in] pkgver Package name/version tuple to match, i.e `foo-1.0'.
+ *
+ * @return true on success, false otherwise and errno is set appropiately.
+ */
+bool xbps_remove_pkg_from_array_by_pkgver(prop_array_t array, const char *pkgver);
+
 /**
  * Removes the package's proplib dictionary matching \a pkgname,
  * in an array with key \a key stored in a proplib dictionary.

lib/plist_find.c

@@ -105,6 +105,37 @@ xbps_find_pkg_in_array_by_pattern(prop_array_t array, const char *pattern)
 	return find_pkg_in_array(array, pattern, true, false);
 }
 
+prop_dictionary_t
+xbps_find_pkg_in_array_by_pkgver(prop_array_t array, const char *pkgver)
+{
+	prop_object_iterator_t iter;
+	prop_object_t obj = NULL;
+	const char *rpkgver;
+	bool found = false;
+
+	assert(prop_object_type(array) == PROP_TYPE_ARRAY);
+	assert(pkgver != NULL);
+
+	iter = prop_array_iterator(array);
+	if (iter == NULL)
+		return NULL;
+
+	while ((obj = prop_object_iterator_next(iter))) {
+		if (!prop_dictionary_get_cstring_nocopy(obj,
+		    "pkgver", &rpkgver))
+			continue;
+		if (strcmp(pkgver, rpkgver) == 0) {
+			found = true;
+			break;
+		}
+	}
+	prop_object_iterator_release(iter);
+	if (found)
+		return obj;
+
+	return NULL;
+}
+
 prop_dictionary_t
 xbps_find_virtualpkg_in_array_by_name(prop_array_t array, const char *name)
 {

lib/plist_remove.c

@@ -82,6 +82,14 @@ remove_string_from_array(prop_array_t array, const char *str, int mode)
 				found = true;
 				break;
 			}
+		} else if (mode == 3) {
+			/* match by pkgver, obj is a dictionary */
+			prop_dictionary_get_cstring_nocopy(obj,
+			    "pkgver", &curname);
+			if (strcmp(curname, str) == 0) {
+				found = true;
+				break;
+			}
 		}
 		idx++;
 	}
@@ -114,6 +122,12 @@ xbps_remove_pkg_from_array_by_name(prop_array_t array, const char *name)
 	return remove_string_from_array(array, name, 2);
 }
 
+bool
+xbps_remove_pkg_from_array_by_pkgver(prop_array_t array, const char *pkgver)
+{
+	return remove_string_from_array(array, pkgver, 3);
+}
+
 bool
 xbps_remove_pkg_from_dict_by_name(prop_dictionary_t dict,
 				  const char *key,