xbps-repo: faster 'genindex' target implementation for index-files.plist.

bin/xbps-repo/index-files.c

@@ -33,18 +33,54 @@
 #include "defs.h"
 
 struct index_files_data {
+	prop_dictionary_t idxdict;
 	prop_array_t idxfiles;
 	const char *pkgdir;
+	bool flush;
+	bool new;
 };
 
+static int
+rmobsoletes_files_cb(prop_object_t obj, void *arg, bool *done)
+{
+	prop_array_t array;
+	struct index_files_data *ifd = arg;
+	const char *pkgver;
+	char *buf;
+
+	(void)done;
+
+	prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+	buf = strdup(pkgver);
+	if (buf == NULL)
+		return ENOMEM;
+
+	array = prop_dictionary_get(ifd->idxdict, "packages");
+	if (xbps_find_pkg_in_array_by_pkgver(array, pkgver)) {
+		/* pkg found, do nothing */
+		free(buf);
+		return 0;
+	}
+	/* remove obsolete pkg dictionary */
+	if (!xbps_remove_pkg_from_array_by_pkgver(ifd->idxfiles, pkgver)) {
+		free(buf);
+		return EINVAL;
+	}
+	printf("Removed obsolete entry for `%s'.\n", buf);
+	free(buf);
+	ifd->flush = true;
+
+	return 0;
+}
+
 static int
 genindex_files_cb(prop_object_t obj, void *arg, bool *done)
 {
-	prop_dictionary_t pkg_filesd, pkgd;
+	prop_dictionary_t pkg_filesd, pkgd, regpkgd;
 	prop_array_t array;
 	struct index_files_data *ifd = arg;
-	const char *binpkg, *pkgver;
-	char *file;
+	const char *binpkg, *pkgver, *rpkgver, *version;
+	char *file, *pkgname, *pattern;
 	bool found = false;
 
 	(void)done;
@@ -52,6 +88,42 @@ genindex_files_cb(prop_object_t obj, void *arg, bool *done)
 	prop_dictionary_get_cstring_nocopy(obj, "filename", &binpkg);
 	prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
 
+	if (ifd->new)
+		goto start;
+
+	pkgname = xbps_pkg_name(pkgver);
+	if (pkgname == NULL)
+		return ENOMEM;
+	version = xbps_pkg_version(pkgver);
+	if (version == NULL) {
+		free(pkgname);
+		return EINVAL;
+	}
+	pattern = xbps_xasprintf("%s>=0", pkgname);
+	if (pattern == NULL) {
+		free(pkgname);
+		return ENOMEM;
+	}
+	free(pkgname);
+	regpkgd = xbps_find_pkg_in_array_by_pattern(ifd->idxfiles, pattern);
+	if (regpkgd) {
+		/*
+		 * pkg already registered, check if same version
+		 * is registered.
+		 */
+		prop_dictionary_get_cstring_nocopy(regpkgd, "pkgver", &rpkgver);
+		if (strcmp(pkgver, rpkgver) == 0) {
+			/* same pkg */
+			xbps_warn_printf("skipping `%s', already registered.\n",
+			    rpkgver);
+			return 0;
+		}
+		/* pkgver does not match, remove it from index-files */
+		if (!xbps_remove_pkg_from_array_by_pkgver(ifd->idxfiles, rpkgver))
+			return EINVAL;
+	}
+
+start:
 	file = xbps_xasprintf("%s/%s", ifd->pkgdir, binpkg);
 	if (file == NULL)
 		return ENOMEM;
@@ -117,6 +189,8 @@ genindex_files_cb(prop_object_t obj, void *arg, bool *done)
 		return EINVAL;
 	}
 	prop_object_release(pkgd);
+	ifd->flush = true;
+	printf("Registered `%s' in repository files index.\n", pkgver);
 
 	return 0;
 }
@@ -129,7 +203,7 @@ repo_genindex_files(const char *pkgdir)
 {
 	prop_dictionary_t idxdict;
 	struct index_files_data *ifd;
-	char *plist, *files_plist;
+	char *plist;
 	int rv;
 
 	plist = xbps_pkg_index_plist(pkgdir);
@@ -142,44 +216,63 @@ repo_genindex_files(const char *pkgdir)
 		free(plist);
 		return errno;
 	}
+	free(plist);
 
-	ifd = malloc(sizeof(*ifd));
+	/* internalize repository index-files plist (if exists) */
+	plist = xbps_pkg_index_files_plist(pkgdir);
+	if (plist == NULL) {
+		rv = ENOMEM;
+		goto out;
+	}
+	ifd = calloc(1, sizeof(*ifd));
 	if (ifd == NULL) {
-		prop_object_release(idxdict);
-		free(plist);
-		return ENOMEM;
+		rv = ENOMEM;
+		goto out;
 	}
 	ifd->pkgdir = pkgdir;
-	ifd->idxfiles = prop_array_create();
-
-	printf("Creating repository's index files cache...\n");
+	ifd->idxfiles = prop_array_internalize_from_zfile(plist);
+	ifd->idxdict = prop_dictionary_copy(idxdict);
+	if (ifd->idxfiles == NULL) {
+		/* missing file, create new one */
+		ifd->idxfiles = prop_array_create();
+		ifd->new = true;
+	}
 
 	/* iterate over index.plist packages array */
 	rv = xbps_callback_array_iter_in_dict(idxdict,
 	    "packages", genindex_files_cb, ifd);
-	prop_object_release(idxdict);
-	free(plist);
-	if (rv != 0) {
-		prop_object_release(ifd->idxfiles);
-		free(ifd);
-		return rv;
-	}
-	files_plist = xbps_pkg_index_files_plist(pkgdir);
-	if (files_plist == NULL) {
-		prop_object_release(ifd->idxfiles);
-		free(ifd);
-		return ENOMEM;
-	}
-	/* externalize index-files dictionary to the plist file */
-	if (!prop_array_externalize_to_zfile(ifd->idxfiles, files_plist)) {
-		free(files_plist);
-		prop_object_release(ifd->idxfiles);
-		free(ifd);
-		return errno;
-	}
-	free(files_plist);
-	prop_object_release(ifd->idxfiles);
-	free(ifd);
+	if (rv != 0)
+		goto out;
 
-	return 0;
+	/* remove obsolete pkg entries */
+	if (!ifd->new) {
+		rv = xbps_callback_array_iter(ifd->idxfiles,
+		    rmobsoletes_files_cb, ifd);
+		if (rv != 0)
+			goto out;
+	}
+	if (!ifd->flush)
+		goto out;
+
+	/* externalize index-files dictionary to the plist file */
+	if (!prop_array_externalize_to_zfile(ifd->idxfiles, plist)) {
+		rv = errno;
+		goto out;
+	}
+out:
+	if (rv == 0)
+		printf("%u packages registered in repository files index.\n",
+		    prop_array_count(ifd->idxfiles));
+	if (ifd->idxfiles != NULL)
+		prop_object_release(ifd->idxfiles);
+	if (ifd->idxdict != NULL)
+		prop_object_release(ifd->idxdict);
+	if (plist != NULL)
+		free(plist);
+	if (ifd != NULL)
+		free(ifd);
+	if (idxdict != NULL)
+		prop_object_release(idxdict);
+
+	return rv;
 }

bin/xbps-repo/index.c

@@ -160,8 +160,8 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 	prop_dictionary_t newpkgd, curpkgd;
 	prop_array_t pkgar;
 	struct stat st;
-	const char *pkgname, *version, *regver, *oldfilen;
-	char *sha256, *filen, *tmpfilen, *tmpstr, *oldfilepath;
+	const char *pkgname, *version, *regver, *oldfilen, *oldpkgver;
+	char *sha256, *filen, *tmpfilen, *oldfilepath;
 	int rv = 0;
 
 	if (idxdict == NULL || file == NULL)
@@ -179,8 +179,8 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 
 	newpkgd = xbps_dictionary_metadata_plist_by_url(file, XBPS_PKGPROPS);
 	if (newpkgd == NULL) {
-		xbps_error_printf("xbps-repo: can't read %s %s metadata "
-		    "file, skipping!\n", file, XBPS_PKGPROPS);
+		xbps_error_printf("failed to read %s metadata for `%s',"
+		    " skipping!\n", XBPS_PKGPROPS, file);
 		goto out;
 	}
 	prop_dictionary_get_cstring_nocopy(newpkgd, "pkgname", &pkgname);
@@ -201,7 +201,7 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 	} else if (curpkgd) {
 		prop_dictionary_get_cstring_nocopy(curpkgd, "version", &regver);
 		if (xbps_cmpver(version, regver) <= 0) {
-			xbps_warn_printf("skipping %s. %s-%s already "
+			xbps_warn_printf("skipping `%s', `%s-%s' already "
 			    "registered.\n", filen, pkgname, regver);
 			prop_object_release(newpkgd);
 			rv = EEXIST;
@@ -214,6 +214,8 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 		 */
 		prop_dictionary_get_cstring_nocopy(curpkgd,
 		    "filename", &oldfilen);
+		prop_dictionary_get_cstring_nocopy(curpkgd,
+		    "pkgver", &oldpkgver);
 		oldfilepath = xbps_xasprintf("%s/%s", filedir, oldfilen);
 		if (oldfilepath == NULL) {
 			prop_object_release(newpkgd);
@@ -221,7 +223,7 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 			goto out;
 		}
 		if (remove(oldfilepath) == -1) {
-			xbps_error_printf("xbps-repo: couldn't remove old "
+			xbps_error_printf("failed to remove old "
 			    "package file `%s': %s\n", oldfilepath,
 			    strerror(errno));
 			free(oldfilepath);
@@ -230,23 +232,15 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 			goto out;
 		}
 		free(oldfilepath);
-		tmpstr = strdup(oldfilen);
-		if (tmpstr == NULL) {
-			prop_object_release(newpkgd);
-			rv = errno;
-			goto out;
-		}
 		if (!xbps_remove_pkg_from_dict_by_name(idxdict,
 		    "packages", pkgname)) {
-			xbps_error_printf("xbps-repo: couldn't remove `%s' "
+			xbps_error_printf("failed to remove `%s' "
 			    "from plist index: %s\n", pkgname, strerror(errno));
 			prop_object_release(newpkgd);
-			free(tmpstr);
 			goto out;
 		}
-		xbps_warn_printf("xbps-repo: removed outdated binpkg file for "
-		    "'%s'.\n", tmpstr);
-		free(tmpstr);
+		printf("Removed obsole package entry/binpkg for `%s'.\n",
+		    oldpkgver);
 	}
 
 	/*
@@ -297,7 +291,7 @@ add_binpkg_to_index(prop_dictionary_t idxdict,
 		rv = EINVAL;
 		goto out;
 	}
-	printf("Registered %s-%s (%s) in package index.\n",
+	printf("Registered `%s-%s' (%s) in repository index.\n",
 	    pkgname, version, filen);
 
 	if (!prop_dictionary_set_uint64(idxdict, "total-pkgs",
@@ -380,7 +374,7 @@ repo_genindex(const char *pkgdir)
 		 * Show total count registered packages.
 		 */
 		prop_dictionary_get_uint64(idxdict, "total-pkgs", &npkgcnt);
-		printf("%ju packages registered in package index.\n", npkgcnt);
+		printf("%ju packages registered in repository index.\n", npkgcnt);
 		/*
 		 * Don't write plist file if no packages were registered.
 		 */