xbps-repo: replaced 'genindex' target with 'index-{add,clean}'.

See the NEWS file for more information.

NEWS

@@ -1,3 +1,15 @@
+xbps-0.16.6 (???):
+
+ * xbps-repo(8): the 'genindex' target has been replaced by the 'index-add' and
+   'index-clean' targets. The 'index-add' expects a list of binary packages to
+   be added to repository's index:
+
+	$ xbps-repo index-add /path/to/repo/*.xbps
+
+   The 'index-clean' removes obsolete entries in repository's index:
+
+	$ xbps-repo index-clean /path/to/repo
+
 xbps-0.16.5 (2012-07-14):
 
  * xbps.conf: remove obsolete remote repositories.

bin/xbps-repo/Makefile

@@ -3,7 +3,7 @@ TOPDIR = ../..
 
 BIN =	xbps-repo
 OBJS =	main.o index.o show.o find-files.o list.o
-OBJS += index-files.o clean.o
+OBJS += index-files.o index-lock.o clean.o
 OBJS += ../xbps-bin/fetch_cb.o ../xbps-bin/util.o
 OBJS += ../xbps-bin/state_cb.o ../xbps-bin/list.o
 MAN = $(BIN).8

bin/xbps-repo/defs.h

@@ -35,10 +35,16 @@ struct repo_search_data {
 };
 
 /* From index.c */
-int	repo_genindex(struct xbps_handle *, const char *);
+int	repo_index_add(struct xbps_handle *, int, char **);
+int	repo_index_clean(struct xbps_handle *, const char *);
 
 /* From index-files.c */
-int	repo_genindex_files(struct xbps_handle *, const char *);
+int	repo_index_files_add(struct xbps_handle *, int, char **);
+int	repo_index_files_clean(struct xbps_handle *, const char *);
+
+/* From index-common.c */
+int	acquire_repo_lock(const char *, char **);
+void	release_repo_lock(char **, int);
 
 /* From find-files.c */
 int	repo_find_files_in_packages(struct xbps_handle *, int, char **);

bin/xbps-repo/index-files.c

@@ -28,90 +28,203 @@
 #include <string.h>
 #include <stdbool.h>
 #include <errno.h>
+#include <libgen.h>
+#include <assert.h>
 
 #include <xbps_api.h>
 #include "defs.h"
 
-struct index_files_data {
+int
-	prop_array_t idx;
+repo_index_files_clean(struct xbps_handle *xhp, const char *repodir)
-	prop_array_t idxfiles;
-	prop_array_t obsoletes;
-	const char *pkgdir;
-	bool flush;
-	bool new;
-};
-
-static int
-rmobsoletes_files_cb(struct xbps_handle *xhp,
-		     prop_object_t obj,
-		     void *arg,
-		     bool *done)
 {
-	struct index_files_data *ifd = arg;
+	prop_object_t obj;
-	const char *pkgver, *arch;
+	prop_array_t idx, idxfiles, obsoletes;
-	char *str;
+	char *plist, *plistf, *plistf_lock, *pkgver, *str;
+	const char *p, *arch, *ipkgver, *iarch;
+	size_t x, i;
+	int rv = 0, fdlock;
+	bool flush = false;
 
-	(void)xhp;
+	plist = plistf = plistf_lock = pkgver = str = NULL;
-	(void)done;
+	idx = idxfiles = obsoletes = NULL;
 
-	prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+	/* Internalize index-files.plist if found */
-	prop_dictionary_get_cstring_nocopy(obj, "architecture", &arch);
+	if ((plistf = xbps_pkg_index_files_plist(xhp, repodir)) == NULL)
-	if (xbps_find_pkg_in_array_by_pkgver(xhp, ifd->idx, pkgver, arch)) {
-		/* pkg found, do nothing */
-		return 0;
-	}
-	if ((str = xbps_xasprintf("%s,%s", pkgver, arch)) == NULL)
-		return ENOMEM;
-
-	if (!prop_array_add_cstring(ifd->obsoletes, str)) {
-		free(str);
 		return EINVAL;
-	}
+	if ((idxfiles = prop_array_internalize_from_zfile(plistf)) == NULL) {
-	free(str);
+		free(plistf);
-	ifd->flush = true;
-
 		return 0;
 	}
+	/* Acquire exclusive file lock */
+	if ((fdlock = acquire_repo_lock(plistf, &plistf_lock)) == -1) {
+		free(plistf);
+		prop_object_release(idxfiles);
+		return -1;
+	}
 
-static int
+	/* Internalize index.plist */
-genindex_files_cb(struct xbps_handle *xhp,
+	if ((plist = xbps_pkg_index_plist(xhp, repodir)) == NULL) {
-		  prop_object_t obj,
+		rv = EINVAL;
-		  void *arg,
+		goto out;
-		  bool *done)
+	}
+	if ((idx = prop_array_internalize_from_zfile(plist)) == NULL) {
+		release_repo_lock(&plistf_lock, fdlock);
+		rv = EINVAL;
+		goto out;
+	}
+	printf("Cleaning `%s' index-files, please wait...\n", repodir);
+	/*
+	 * Iterate over index-files array to find obsolete entries.
+	 */
+	for (x = 0; x < prop_array_count(idx); x++) {
+		obj = prop_array_get(idx, x);
+		prop_dictionary_get_cstring_nocopy(obj, "pkgver", &ipkgver);
+		prop_dictionary_get_cstring_nocopy(obj, "architecture", &iarch);
+		if (xbps_find_pkg_in_array_by_pkgver(xhp, idx, ipkgver, iarch)) {
+			/* pkg found, do nothing */
+			continue;
+		}
+		if ((str = xbps_xasprintf("%s,%s", ipkgver, iarch)) == NULL) {
+			rv = ENOMEM;
+			goto out;
+		}
+		if (!prop_array_add_cstring(obsoletes, str)) {
+			free(str);
+			rv = EINVAL;
+			goto out;
+		}
+		free(str);
+	}
+	/*
+	 * Iterate over the obsoletes and array and remove entries
+	 * from index-files array.
+	 */
+	for (i = 0; i < prop_array_count(obsoletes); i++) {
+		prop_array_get_cstring_nocopy(obsoletes, i, &p);
+		pkgver = strdup(p);
+		for (x = 0; x < strlen(p); x++) {
+			if ((pkgver[x] = p[x]) == ',') {
+				pkgver[x] = '\0';
+				break;
+			}
+		}
+		arch = strchr(p, ',') + 1;
+		if (!xbps_remove_pkg_from_array_by_pkgver(
+		    xhp, idxfiles, pkgver, arch)) {
+			free(pkgver);
+			rv = EINVAL;
+			goto out;
+		}
+		printf("index-files: removed obsolete entry `%s' "
+		    "(%s)\n", pkgver, arch);
+		free(pkgver);
+		flush = true;
+	}
+	/* Externalize index-files array to plist when necessary */
+	if (flush && !prop_array_externalize_to_zfile(idxfiles, plistf))
+		rv = errno;
+
+	printf("index-files: %u packages registered.\n",
+	    prop_array_count(idxfiles));
+
+out:
+	release_repo_lock(&plistf_lock, fdlock);
+
+	if (obsoletes)
+		prop_object_release(obsoletes);
+	if (idx)
+		prop_object_release(idx);
+	if (idxfiles)
+		prop_object_release(idxfiles);
+	if (plist)
+		free(plist);
+	if (plistf)
+		free(plistf);
+
+	return rv;
+}
+
+int
+repo_index_files_add(struct xbps_handle *xhp, int argc, char **argv)
 {
-	prop_object_t obj2, fileobj;
+	prop_array_t idxfiles = NULL;
-	prop_dictionary_t pkg_filesd, pkgd;
+	prop_object_t obj, fileobj;
+	prop_dictionary_t pkgprops, pkg_filesd, pkgd;
 	prop_array_t files, pkg_cffiles, pkg_files, pkg_links;
-	struct index_files_data *ifd = arg;
 	const char *binpkg, *pkgver, *arch;
-	char *file;
+	char *plist, *repodir, *p, *plist_lock;
-	bool found = false;
+	size_t x;
-	size_t i;
+	int i, fdlock = -1, rv = 0;
+	bool found, flush;
 
-	(void)xhp;
+	found = flush = false;
-	(void)done;
+	plist = plist_lock = repodir = p = NULL;
+	obj = fileobj = NULL;
+	pkgprops = pkg_filesd = pkgd = NULL;
+	files = NULL;
 
-	prop_dictionary_get_cstring_nocopy(obj, "filename", &binpkg);
+        if ((p = strdup(argv[1])) == NULL) {
-	prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
+		rv = ENOMEM;
-	prop_dictionary_get_cstring_nocopy(obj, "architecture", &arch);
+		goto out;
+	}
+	repodir = dirname(p);
+	if ((plist = xbps_pkg_index_files_plist(xhp, repodir)) == NULL) {
+		rv = ENOMEM;
+		goto out;
+	}
+	/* Acquire exclusive file lock or wait for it.
+	 */
+	if ((fdlock = acquire_repo_lock(plist, &plist_lock)) == -1) {
+		free(p);
+		free(plist);
+		return -1;
+	}
+	/*
+	 * Internalize index-files.plist if found and process argv.
+	 */
+	if ((idxfiles = prop_array_internalize_from_zfile(plist)) == NULL) {
+		if (errno == ENOENT) {
+			idxfiles = prop_array_create();
+			assert(idxfiles);
+		} else {
+			rv = errno;
+			goto out;
+		}
+	}
 
-	if (xbps_find_pkg_in_array_by_pkgver(xhp, ifd->idxfiles, pkgver, arch))  {
+	for (i = 1; i < argc; i++) {
+		found = false;
+		pkgprops = xbps_dictionary_metadata_plist_by_url(argv[i],
+				"./props.plist");
+		if (pkgprops == NULL) {
+			fprintf(stderr, "index-files: cannot internalize "
+			    "%s props.plist: %s\n", argv[i], strerror(errno));
+			continue;
+		}
+		prop_dictionary_get_cstring_nocopy(pkgprops,
+		    "filename", &binpkg);
+		prop_dictionary_get_cstring_nocopy(pkgprops,
+		    "pkgver", &pkgver);
+		prop_dictionary_get_cstring_nocopy(pkgprops,
+		    "architecture", &arch);
+
+		if (xbps_find_pkg_in_array_by_pkgver(xhp, idxfiles,
+		    pkgver, arch))  {
 			fprintf(stderr, "index-files: skipping `%s' (%s), "
 			    "already registered.\n", pkgver, arch);
-		return 0;
+			prop_object_release(pkgprops);
+			pkgprops = NULL;
+			continue;
 		}
 
-	file = xbps_xasprintf("%s/%s/%s", ifd->pkgdir, arch, binpkg);
-	if (file == NULL)
-		return ENOMEM;
-
 		/* internalize files.plist from binary package archive */
-	pkg_filesd = xbps_dictionary_metadata_plist_by_url(file, "./files.plist");
+		pkg_filesd = xbps_dictionary_metadata_plist_by_url(argv[i],
+				"./files.plist");
 		if (pkg_filesd == NULL) {
-		free(file);
+			prop_object_release(pkgprops);
-		return EINVAL;
+			rv = EINVAL;
+			goto out;
 		}
-	free(file);
 
 		/* Find out if binary pkg stored in index contain any file */
 		pkg_cffiles = prop_dictionary_get(pkg_filesd, "conf_files");
@@ -134,184 +247,129 @@ genindex_files_cb(struct xbps_handle *xhp,
 
 		/* If pkg does not contain any file, ignore it */
 		if (!found) {
+			prop_object_release(pkgprops);
 			prop_object_release(pkg_filesd);
-		return 0;
+			continue;
 		}
-
 		/* create pkg dictionary */
 		if ((pkgd = prop_dictionary_create()) == NULL) {
+			prop_object_release(pkgprops);
 			prop_object_release(pkg_filesd);
-		return ENOMEM;
+			rv = EINVAL;
+			goto out;
 		}
 		/* add pkgver and architecture objects into pkg dictionary */
 		if (!prop_dictionary_set_cstring(pkgd, "architecture", arch)) {
+			prop_object_release(pkgprops);
 			prop_object_release(pkg_filesd);
 			prop_object_release(pkgd);
-		return EINVAL;
+			rv = EINVAL;
+			goto out;
 		}
 		if (!prop_dictionary_set_cstring(pkgd, "pkgver", pkgver)) {
+			prop_object_release(pkgprops);
 			prop_object_release(pkg_filesd);
 			prop_object_release(pkgd);
-		return EINVAL;
+			rv = EINVAL;
+			goto out;
 		}
 		/* add files array obj into pkg dictionary */
 		if ((files = prop_array_create()) == NULL) {
+			prop_object_release(pkgprops);
 			prop_object_release(pkg_filesd);
 			prop_object_release(pkgd);
-		return EINVAL;
+			rv = EINVAL;
+			goto out;
 		}
 		if (!prop_dictionary_set(pkgd, "files", files)) {
+			prop_object_release(pkgprops);
 			prop_object_release(pkg_filesd);
+			prop_object_release(files);
 			prop_object_release(pkgd);
-		return EINVAL;
+			rv = EINVAL;
+			goto out;
 		}
-
 		/* add conf_files in pkgd */
 		if (pkg_cffiles != NULL) {
-		for (i = 0; i < prop_array_count(pkg_cffiles); i++) {
+			for (x = 0; x < prop_array_count(pkg_cffiles); x++) {
-			obj2 = prop_array_get(pkg_cffiles, i);
+				obj = prop_array_get(pkg_cffiles, x);
-			fileobj = prop_dictionary_get(obj2, "file");
+				fileobj = prop_dictionary_get(obj, "file");
 				if (!prop_array_add(files, fileobj)) {
-				prop_object_release(pkgd);
+					prop_object_release(pkgprops);
 					prop_object_release(pkg_filesd);
-				return EINVAL;
+					prop_object_release(files);
+					prop_object_release(pkgd);
+					rv = EINVAL;
+					goto out;
 				}
 			}
 		}
 		/* add files array in pkgd */
 		if (pkg_files != NULL) {
-		for (i = 0; i < prop_array_count(pkg_files); i++) {
+			for (x = 0; x < prop_array_count(pkg_files); x++) {
-			obj2 = prop_array_get(pkg_files, i);
+				obj = prop_array_get(pkg_files, x);
-			fileobj = prop_dictionary_get(obj2, "file");
+				fileobj = prop_dictionary_get(obj, "file");
 				if (!prop_array_add(files, fileobj)) {
-				prop_object_release(pkgd);
+					prop_object_release(pkgprops);
 					prop_object_release(pkg_filesd);
-				return EINVAL;
+					prop_object_release(files);
+					prop_object_release(pkgd);
+					rv = EINVAL;
+					goto out;
 				}
 			}
 		}
 		/* add links array in pkgd */
 		if (pkg_links != NULL) {
-		for (i = 0; i < prop_array_count(pkg_links); i++) {
+			for (x = 0; x < prop_array_count(pkg_links); x++) {
-			obj2 = prop_array_get(pkg_links, i);
+				obj = prop_array_get(pkg_links, x);
-			fileobj = prop_dictionary_get(obj2, "file");
+				fileobj = prop_dictionary_get(obj, "file");
 				if (!prop_array_add(files, fileobj)) {
-				prop_object_release(pkgd);
+					prop_object_release(pkgprops);
 					prop_object_release(pkg_filesd);
-				return EINVAL;
+					prop_object_release(files);
-			}
-		}
-	}
-	prop_object_release(pkg_filesd);
-	/* add pkgd into provided array */
-	if (!prop_array_add(ifd->idxfiles, pkgd)) {
 					prop_object_release(pkgd);
-		return EINVAL;
-	}
-	printf("index-files: added `%s' (%s)\n", pkgver, arch);
-	prop_object_release(pkgd);
-	ifd->flush = true;
-
-	return 0;
-}
-
-/*
- * Create the index files cache for all packages in repository.
- */
-int
-repo_genindex_files(struct xbps_handle *xhp, const char *pkgdir)
-{
-	prop_array_t idx;
-	struct index_files_data *ifd = NULL;
-	size_t i, x;
-	const char *p, *arch;
-	char *plist, *pkgver;
-	int rv;
-
-	plist = xbps_pkg_index_plist(xhp, pkgdir);
-	if (plist == NULL)
-		return ENOMEM;
-
-	/* internalize repository index plist */
-	idx = prop_array_internalize_from_zfile(plist);
-	if (idx == NULL) {
-		free(plist);
-		return errno;
-	}
-	free(plist);
-
-	/* internalize repository index-files plist (if exists) */
-	plist = xbps_pkg_index_files_plist(xhp, pkgdir);
-	if (plist == NULL) {
-		rv = ENOMEM;
-		goto out;
-	}
-	ifd = calloc(1, sizeof(*ifd));
-	if (ifd == NULL) {
-		rv = ENOMEM;
-		goto out;
-	}
-	ifd->pkgdir = pkgdir;
-	ifd->idxfiles = prop_array_internalize_from_zfile(plist);
-	ifd->idx = idx;
-	ifd->obsoletes = prop_array_create();
-	if (ifd->idxfiles == NULL) {
-		/* missing file, create new one */
-		ifd->idxfiles = prop_array_create();
-		ifd->new = true;
-	}
-
-	/* remove obsolete pkg entries */
-	if (!ifd->new) {
-		rv = xbps_callback_array_iter(xhp, ifd->idxfiles,
-		    rmobsoletes_files_cb, ifd);
-		if (rv != 0)
-			goto out;
-		for (i = 0; i < prop_array_count(ifd->obsoletes); i++) {
-			prop_array_get_cstring_nocopy(ifd->obsoletes, i, &p);
-			pkgver = strdup(p);
-			for (x = 0; x < strlen(p); x++) {
-				if ((pkgver[x] = p[x]) == ',') {
-					pkgver[x] = '\0';
-					break;
-				}
-			}
-			arch = strchr(p, ',') + 1;
-			if (!xbps_remove_pkg_from_array_by_pkgver(
-			    xhp, ifd->idxfiles, pkgver, arch)) {
-				free(pkgver);
 					rv = EINVAL;
 					goto out;
 				}
-			printf("index-files: removed obsolete entry `%s' "
-			    "(%s)\n", pkgver, arch);
-			free(pkgver);
 			}
 		}
-	/* iterate over index.plist array */
+		/* add pkgd into the index-files array */
-	if ((rv = xbps_callback_array_iter(xhp, idx, genindex_files_cb, ifd)) != 0)
+		if (!prop_array_add(idxfiles, pkgd)) {
+			prop_object_release(pkgprops);
+			prop_object_release(pkg_filesd);
+			prop_object_release(files);
+			prop_object_release(pkgd);
+			rv = EINVAL;
 			goto out;
+		}
+		flush = true;
+		printf("index-files: added `%s' (%s)\n", pkgver, arch);
+		prop_object_release(pkgprops);
+		prop_object_release(pkg_filesd);
+		prop_object_release(files);
+		prop_object_release(pkgd);
+		pkgprops = pkg_filesd = pkgd = NULL;
+		files = NULL;
+	}
 
-	if (!ifd->flush)
+	if (flush && !prop_array_externalize_to_zfile(idxfiles, plist)) {
-		goto out;
+		fprintf(stderr, "failed to externalize %s: %s\n",
-
+		    plist, strerror(errno));
-	/* externalize index-files array */
-	if (!prop_array_externalize_to_zfile(ifd->idxfiles, plist)) {
 		rv = errno;
-		goto out;
 	}
-out:
-	if (rv == 0)
 	printf("index-files: %u packages registered.\n",
-		    prop_array_count(ifd->idxfiles));
+	    prop_array_count(idxfiles));
-	if (ifd->idxfiles != NULL)
+
-		prop_object_release(ifd->idxfiles);
+out:
-	if (plist != NULL)
+	release_repo_lock(&plist_lock, fdlock);
+
+	if (p)
+		free(p);
+	if (plist)
 		free(plist);
-	if (ifd != NULL)
+	if (idxfiles)
-		free(ifd);
+		prop_object_release(idxfiles);
-	if (idx != NULL)
-		prop_object_release(idx);
 
 	return rv;
 }

bin/xbps-repo/index-lock.c

@@ -0,0 +1,103 @@
+/*-
+ * Copyright (c) 2012 Juan Romero Pardines.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <dirent.h>
+#include <libgen.h>
+#include <assert.h>
+
+#include <xbps_api.h>
+#include "defs.h"
+
+int
+acquire_repo_lock(const char *plist, char **plist_lock)
+{
+	int try = 0, fd = -1;
+
+	*plist_lock = xbps_xasprintf("%s.lock", plist);
+	assert(*plist_lock);
+
+	fd = open(*plist_lock, O_RDWR);
+	if (fd == -1) {
+		if (errno == ENOENT) {
+			fd = creat(*plist_lock, 0640);
+			if (fd == -1) {
+				fprintf(stderr, "Failed to create "
+				    "repository file lock: %s\n",
+				strerror(errno));
+				return -1;
+			}
+		} else {
+			fprintf(stderr, "Failed to open repository "
+			     "file lock: %s\n", strerror(errno));
+			return -1;
+		}
+	}
+	/*
+	 * Acquire the the exclusive file lock or wait until
+	 * it's available.
+	 */
+#define WAIT_SECONDS 30
+	while (lockf(fd, F_TLOCK, 0) < 0) {
+		if (errno == EAGAIN || errno == EACCES) {
+			if (++try < WAIT_SECONDS) {
+				fprintf(stderr,"Repository index file "
+				    "is busy! retrying in 5 sec...\n");
+				sleep(5);
+				continue;
+			}
+		}
+		fprintf(stderr, "Failed to acquire repository "
+		    "file lock in %d seconds!\n", WAIT_SECONDS);
+		close(fd);
+		return -1;
+	}
+	return fd;
+}
+
+void
+release_repo_lock(char **plist_lock, int fd)
+{
+	assert(*plist_lock);
+
+	if (fd == -1)
+		return;
+	if (lockf(fd, F_ULOCK, 0) == -1) {
+		fprintf(stderr, "failed to unlock file lock: %s\n",
+		     strerror(errno));
+		close(fd);
+		exit(EXIT_FAILURE);
+	}
+	close(fd);
+	unlink(*plist_lock);
+	free(*plist_lock);
+}

bin/xbps-repo/index.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2012 Juan Romero Pardines.
+ * Copyright (c) 2012 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -23,6 +23,7 @@
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <sys/stat.h>
 #include <stdio.h>
 #include <stdbool.h>
 #include <stdlib.h>
@@ -31,50 +32,51 @@
 #include <dirent.h>
 #include <libgen.h>
 #include <assert.h>
-#include <sys/stat.h>
 
 #include <xbps_api.h>
 #include "defs.h"
 
-#ifndef __arraycount
-#define __arraycount(a) (sizeof(a) / sizeof(*a))
-#endif
-
-static const char *archs[] = { "noarch", "i686", "x86_64" };
-
 /*
  * Removes stalled pkg entries in repository's index.plist file, if any
  * binary package cannot be read (unavailable, not enough perms, etc).
  */
-static int
+int
-remove_missing_binpkg_entries(struct xbps_handle *xhp, const char *repodir)
+repo_index_clean(struct xbps_handle *xhp, const char *repodir)
 {
 	prop_array_t array;
 	prop_dictionary_t pkgd;
 	const char *filen, *pkgver, *arch;
-	char *binpkg, *plist;
+	char *binpkg, *plist, *plist_lock;
-	size_t i;
+	size_t i, idx = 0;
-	int rv = 0;
+	int fdlock, rv = 0;
-	bool found = false;
+	bool flush = false;
 
-	plist = xbps_pkg_index_plist(xhp, repodir);
+	if ((plist = xbps_pkg_index_plist(xhp, repodir)) == NULL)
-	if (plist == NULL)
 		return -1;
 
+	if ((fdlock = acquire_repo_lock(plist, &plist_lock)) == -1) {
+		free(plist);
+		return -1;
+	}
+
 	array = prop_array_internalize_from_zfile(plist);
 	if (array == NULL) {
 		if (errno != ENOENT) {
 			xbps_error_printf("xbps-repo: cannot read `%s': %s\n",
 			    plist, strerror(errno));
-			exit(EXIT_FAILURE);
+			free(plist);
+			release_repo_lock(&plist_lock, fdlock);
+			return -1;
 		} else {
+			release_repo_lock(&plist_lock, fdlock);
 			free(plist);
 			return 0;
 		}
 	}
+	printf("Cleaning `%s' index, please wait...\n", repodir);
 
 again:
-	for (i = 0; i < prop_array_count(array); i++) {
+	for (i = idx; i < prop_array_count(array); i++) {
 		pkgd = prop_array_get(array, i);
 		prop_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver);
 		prop_dictionary_get_cstring_nocopy(pkgd, "filename", &filen);
@@ -90,123 +92,160 @@ again:
 			    pkgver, arch);
 			prop_array_remove(array, i);
 			free(binpkg);
-			found = true;
+			flush = true;
+			idx = i;
 			goto again;
 		}
 		free(binpkg);
 	}
-	if (found && !prop_array_externalize_to_zfile(array, plist))
+	if (flush && !prop_array_externalize_to_zfile(array, plist))
 		rv = errno;
 
 	free(plist);
+	printf("index: %u packages registered.\n", prop_array_count(array));
+	prop_object_release(array);
+	release_repo_lock(&plist_lock, fdlock);
 
 	return rv;
 }
 
-static prop_array_t
-repoidx_get(struct xbps_handle *xhp, const char *pkgdir)
-{
-	prop_array_t array;
-	char *plist;
-	int rv;
 /*
-	 * Remove entries in repositories index for unexistent
+ * Adds a binary package into the index and removes old binary package
-	 * packages, i.e dangling entries.
+ * and entry when it's necessary.
  */
-	if ((rv = remove_missing_binpkg_entries(xhp, pkgdir)) != 0)
+int
-		return NULL;
+repo_index_add(struct xbps_handle *xhp, int argc, char **argv)
-
-	plist = xbps_pkg_index_plist(xhp, pkgdir);
-	if (plist == NULL)
-		return NULL;
-
-	array = prop_array_internalize_from_zfile(plist);
-	free(plist);
-	if (array == NULL)
-		array = prop_array_create();
-
-	return array;
-}
-
-static int
-add_binpkg_to_index(struct xbps_handle *xhp,
-		    prop_array_t idx,
-		    const char *repodir,
-		    const char *file)
 {
-	prop_dictionary_t newpkgd, curpkgd;
+	prop_array_t idx = NULL;
+	prop_dictionary_t newpkgd = NULL, curpkgd;
 	struct stat st;
 	const char *pkgname, *version, *regver, *oldfilen, *oldpkgver;
 	const char *arch, *oldarch;
-	char *sha256, *filen, *tmpfilen, *oldfilepath, *buf;
+	char *sha256, *filen, *repodir, *oldfilepath, *buf;
-	int ret = 0, rv = 0;
+	char *tmpfilen = NULL, *tmprepodir = NULL, *plist = NULL;
+	char *plist_lock = NULL;
+	int i, ret = 0, rv = 0, fdlock = -1;
+	bool flush = false;
 
-	tmpfilen = strdup(file);
+	if ((tmprepodir = strdup(argv[1])) == NULL) {
-	if (tmpfilen == NULL)
+		rv = ENOMEM;
-		return errno;
+		goto out;
+	}
+	repodir = dirname(tmprepodir);
 
-	filen = basename(tmpfilen);
+	/* Internalize plist file or create it if doesn't exist */
-	if (strcmp(tmpfilen, filen) == 0) {
+	if ((plist = xbps_pkg_index_plist(xhp, repodir)) == NULL)
-		rv = EINVAL;
+		return -1;
+
+	/* Acquire exclusive file lock */
+	if ((fdlock = acquire_repo_lock(plist, &plist_lock)) == -1) {
+		rv = fdlock;
 		goto out;
 	}
 
-	newpkgd = xbps_dictionary_metadata_plist_by_url(file, "./props.plist");
+	if ((idx = prop_array_internalize_from_zfile(plist)) == NULL) {
+		if (errno != ENOENT) {
+			xbps_error_printf("xbps-repo: cannot read `%s': %s\n",
+			    plist, strerror(errno));
+			rv = -1;
+			goto out;
+		} else {
+			idx = prop_array_create();
+			assert(idx);
+		}
+	}
+
+	/*
+	 * Process all packages specified in argv.
+	 */
+	for (i = 1; i < argc; i++) {
+		if ((tmpfilen = strdup(argv[i])) == NULL) {
+			rv = ENOMEM;
+			goto out;
+		}
+		filen = basename(tmpfilen);
+		/*
+		 * Read metadata props plist dictionary from binary package.
+		 */
+		newpkgd = xbps_dictionary_metadata_plist_by_url(argv[i],
+		    "./props.plist");
 		if (newpkgd == NULL) {
 			xbps_error_printf("failed to read %s metadata for `%s',"
-		    " skipping!\n", XBPS_PKGPROPS, file);
+			    " skipping!\n", XBPS_PKGPROPS, argv[i]);
-		goto out;
+			free(tmpfilen);
+			filen = NULL;
+			continue;
 		}
-	prop_dictionary_get_cstring_nocopy(newpkgd, "pkgname", &pkgname);
+		prop_dictionary_get_cstring_nocopy(newpkgd, "pkgname",
-	prop_dictionary_get_cstring_nocopy(newpkgd, "version", &version);
+		    &pkgname);
-	prop_dictionary_get_cstring_nocopy(newpkgd, "architecture", &arch);
+		prop_dictionary_get_cstring_nocopy(newpkgd, "version",
+		    &version);
+		prop_dictionary_get_cstring_nocopy(newpkgd, "architecture",
+		    &arch);
 		/*
 		 * Check if this package exists already in the index, but first
 		 * checking the version. If current package version is greater
 		 * than current registered package, update the index; otherwise
 		 * pass to the next one.
 		 */
-	curpkgd = xbps_find_pkg_in_array_by_name(xhp, idx, pkgname, arch);
+		curpkgd =
+		    xbps_find_pkg_in_array_by_name(xhp, idx, pkgname, arch);
 		if (curpkgd == NULL) {
 			if (errno && errno != ENOENT) {
 				prop_object_release(newpkgd);
+				free(tmpfilen);
 				rv = errno;
 				goto out;
 			}
 		} else {
-		prop_dictionary_get_cstring_nocopy(curpkgd, "version", &regver);
+			prop_dictionary_get_cstring_nocopy(curpkgd,
+			    "version", &regver);
 			ret = xbps_cmpver(version, regver);
 			if (ret == 0) {
-			/* same version */
+				/* Same version */
-			fprintf(stderr, "index: skipping `%s-%s' (%s), `%s-%s' already "
+				fprintf(stderr, "index: skipping `%s-%s' "
-			    "registered.\n", pkgname, version,
+				    "(%s), `%s-%s' already registered.\n",
-			    arch, pkgname, regver);
+				    pkgname, version, arch, pkgname, regver);
 				prop_object_release(newpkgd);
-			rv = EEXIST;
+				free(tmpfilen);
-			goto out;
+				newpkgd = NULL;
+				filen = NULL;
+				continue;
 			} else if (ret == -1) {
-			/* idx version is greater, remove current binpkg */
+				/*
+				 * Index version is greater, remove current
+				 * package.
+				 */
 				oldfilepath = xbps_xasprintf("%s/%s/%s",
 					repodir, arch, filen);
 				assert(oldfilepath != NULL);
 				if (remove(oldfilepath) == -1) {
 					rv = errno;
-				xbps_error_printf("failed to remove old binpkg "
+					xbps_error_printf("failed to remove "
-				    "`%s': %s\n", oldfilepath, strerror(rv));
+					    "old binpkg `%s': %s\n",
-				free(oldfilepath);
+					    oldfilepath, strerror(rv));
 					prop_object_release(newpkgd);
+					free(tmpfilen);
+					free(oldfilepath);
 					goto out;
 				}
 				free(oldfilepath);
-			buf = xbps_xasprintf("`%s-%s' (%s)", pkgname, version, arch);
+				buf = xbps_xasprintf("`%s-%s' (%s)",
+				    pkgname, version, arch);
 				assert(buf != NULL);
-			prop_object_release(newpkgd);
+				printf("index: removed obsolete binpkg %s.\n",
-			printf("index: removed obsolete binpkg %s.\n", buf);
+				    buf);
 				free(buf);
-			rv = EEXIST;
+				prop_object_release(newpkgd);
-			goto out;
+				free(tmpfilen);
+				newpkgd = NULL;
+				filen = NULL;
+				continue;
 			}
-		/* current binpkg is greater than idx version */
+			/*
+			 * Current package version is greater than
+			 * index version.
+			 */
 			prop_dictionary_get_cstring_nocopy(curpkgd,
 			    "filename", &oldfilen);
 			prop_dictionary_get_cstring_nocopy(curpkgd,
@@ -214,9 +253,7 @@ add_binpkg_to_index(struct xbps_handle *xhp,
 			prop_dictionary_get_cstring_nocopy(curpkgd,
 			    "architecture", &oldarch);
 
-		buf = strdup(oldpkgver);
+			if ((buf = strdup(oldpkgver)) == NULL) {
-		if (buf == NULL) {
-			prop_object_release(newpkgd);
 				rv = ENOMEM;
 				goto out;
 			}
@@ -224,8 +261,9 @@ add_binpkg_to_index(struct xbps_handle *xhp,
 			    repodir, oldarch, oldfilen);
 			if (oldfilepath == NULL) {
 				rv = errno;
-			prop_object_release(newpkgd);
 				free(buf);
+				prop_object_release(newpkgd);
+				free(tmpfilen);
 				goto out;
 			}
 			if (remove(oldfilepath) == -1) {
@@ -234,165 +272,100 @@ add_binpkg_to_index(struct xbps_handle *xhp,
 				    "package file `%s': %s\n", oldfilepath,
 				    strerror(errno));
 				free(oldfilepath);
-			prop_object_release(newpkgd);
 				free(buf);
+				prop_object_release(newpkgd);
+				free(tmpfilen);
 				goto out;
 			}
 			free(oldfilepath);
 			if (!xbps_remove_pkg_from_array_by_pkgver(xhp, idx,
 			    buf, oldarch)) {
 				xbps_error_printf("failed to remove `%s' "
-			    "from plist index: %s\n", buf, strerror(errno));
+				    "from plist index: %s\n", buf,
-			prop_object_release(newpkgd);
+				    strerror(errno));
+				rv = errno;
 				free(buf);
+				prop_object_release(newpkgd);
+				free(tmpfilen);
 				goto out;
 			}
 			printf("index: removed obsolete entry/binpkg `%s' "
 			     "(%s).\n", buf, arch);
 			free(buf);
 		}
-
 		/*
 		 * We have the dictionary now, add the required
 		 * objects for the index.
 		 */
 		if (!prop_dictionary_set_cstring(newpkgd, "filename", filen)) {
-		prop_object_release(newpkgd);
 			rv = errno;
+			prop_object_release(newpkgd);
+			free(tmpfilen);
 			goto out;
 		}
-	if ((sha256 = xbps_file_hash(file)) == NULL) {
+		if ((sha256 = xbps_file_hash(argv[i])) == NULL) {
-		prop_object_release(newpkgd);
 			rv = errno;
+			prop_object_release(newpkgd);
+			free(tmpfilen);
 			goto out;
 		}
-	if (!prop_dictionary_set_cstring(newpkgd, "filename-sha256", sha256)) {
+		if (!prop_dictionary_set_cstring(newpkgd, "filename-sha256",
-		prop_object_release(newpkgd);
+		    sha256)) {
 			free(sha256);
+			prop_object_release(newpkgd);
+			free(tmpfilen);
 			rv = errno;
 			goto out;
 		}
 		free(sha256);
-	if (stat(file, &st) == -1) {
+		if (stat(argv[i], &st) == -1) {
 			prop_object_release(newpkgd);
+			free(tmpfilen);
 			rv = errno;
 			goto out;
 		}
 		if (!prop_dictionary_set_uint64(newpkgd, "filename-size",
 		    (uint64_t)st.st_size)) {
 			prop_object_release(newpkgd);
+			free(tmpfilen);
 			rv = errno;
 			goto out;
 		}
 		/*
-	 * Add dictionary into the index and update package count.
+		 * Add new pkg dictionary into the index.
 		 */
-	if (!xbps_add_obj_to_array(idx, newpkgd)) {
+		if (!prop_array_add(idx, newpkgd)) {
+			prop_object_release(newpkgd);
+			free(tmpfilen);
 			rv = EINVAL;
 			goto out;
 		}
+		flush = true;
 		printf("index: added `%s-%s' (%s).\n", pkgname, version, arch);
-
-out:
-	if (tmpfilen)
 		free(tmpfilen);
-
+		prop_object_release(newpkgd);
-	return rv;
+		newpkgd = NULL;
+		sha256 = NULL;
+		filen = NULL;
+		oldfilen = oldarch = oldpkgver = NULL;
+		pkgname = version = arch = NULL;
 	}
 
-int
+	if (flush && !prop_array_externalize_to_zfile(idx, plist)) {
-repo_genindex(struct xbps_handle *xhp, const char *pkgdir)
+		xbps_error_printf("failed to externalize plist: %s\n",
-{
+		    strerror(errno));
-	prop_array_t idx = NULL;
-	struct dirent *dp;
-	DIR *dirp;
-	size_t i;
-	char *curdir;
-	char *binfile, *plist;
-	int rv = 0;
-	bool registered_newpkgs = false, foundpkg = false;
-
-	/*
-	 * Create or read existing package index plist file.
-	 */
-	idx = repoidx_get(xhp, pkgdir);
-	if (idx == NULL)
-		return errno;
-
-	plist = xbps_pkg_index_plist(xhp, pkgdir);
-	if (plist == NULL) {
-		prop_object_release(idx);
-		return errno;
-	}
-
-	for (i = 0; i < __arraycount(archs); i++) {
-		curdir = xbps_xasprintf("%s/%s", pkgdir, archs[i]);
-		assert(curdir != NULL);
-
-		dirp = opendir(curdir);
-		if (dirp == NULL) {
-			if (errno == ENOENT) {
-				free(curdir);
-				continue;
-			}
-			xbps_error_printf("xbps-repo: cannot open `%s': %s\n",
-			    curdir, strerror(errno));
-			exit(EXIT_FAILURE);
-		}
-		while ((dp = readdir(dirp)) != NULL) {
-			if ((strcmp(dp->d_name, ".") == 0) ||
-			    (strcmp(dp->d_name, "..") == 0))
-				continue;
-			/* Ignore unknown files */
-			if (strstr(dp->d_name, ".xbps") == NULL)
-				continue;
-
-			foundpkg = true;
-			binfile = xbps_xasprintf("%s/%s", curdir, dp->d_name);
-			if (binfile == NULL) {
-				(void)closedir(dirp);
-				rv = errno;
-				goto out;
-			}
-			rv = add_binpkg_to_index(xhp, idx, pkgdir, binfile);
-			free(binfile);
-			if (rv == EEXIST) {
-				rv = 0;
-				continue;
-			} else if (rv != 0) {
-				(void)closedir(dirp);
-				free(curdir);
-				goto out;
-			}
-			registered_newpkgs = true;
-		}
-		(void)closedir(dirp);
-		free(curdir);
-	}
-
-	if (foundpkg == false) {
-		/* No packages were found in directory */
-		rv = ENOENT;
-	} else {
-		/*
-		 * Show total count registered packages.
-		 */
-		printf("index: %zu packages registered.\n",
-		    (size_t)prop_array_count(idx));
-		/*
-		 * Don't write plist file if no packages were registered.
-		 */
-		if (registered_newpkgs == false)
-			goto out;
-		/*
-		 * If any package was registered in package index, write
-		 * plist file to storage.
-		 */
-		if (!prop_array_externalize_to_zfile(idx, plist))
 		rv = errno;
 	}
+	printf("index: %u packages registered.\n", prop_array_count(idx));
+
 out:
+	release_repo_lock(&plist_lock, fdlock);
+
+	if (tmprepodir)
+		free(tmprepodir);
+	if (plist)
 		free(plist);
+	if (idx)
 		prop_object_release(idx);
 
 	return rv;

bin/xbps-repo/main.c

@@ -260,14 +260,25 @@ main(int argc, char **argv)
 			xbps_error_printf("xbps-repo: no repositories "
 			    "currently registered!\n");
 		}
-	} else if (strcasecmp(argv[0], "genindex") == 0) {
+	} else if (strcasecmp(argv[0], "index-add") == 0) {
-		/* Generates a package repository index plist file. */
+		/* Registers a binary package into the repository's index. */
+		if (argc < 2)
+			usage(true);
+
+		if ((rv = repo_index_add(&xh, argc, argv)) != 0)
+			goto out;
+		if ((rv = repo_index_files_add(&xh, argc, argv)) != 0)
+			goto out;
+
+	} else if (strcasecmp(argv[0], "index-clean") == 0) {
+		/* Removes obsolete pkg entries from index in a repository */
 		if (argc != 2)
 			usage(true);
 
-		rv = repo_genindex(&xh, argv[1]);
+		if ((rv = repo_index_clean(&xh, argv[1])) != 0)
-		if (rv == 0)
+			goto out;
-			rv = repo_genindex_files(&xh, argv[1]);
+
+		rv = repo_index_files_clean(&xh, argv[1]);
 
 	} else if (strcasecmp(argv[0], "sync") == 0) {
 		/* Syncs the pkg index for all registered remote repos */

bin/xbps-repo/xbps-repo.8

@@ -1,4 +1,4 @@
-.Dd June 5, 2012
+.Dd July 17, 2012
 .Os Void GNU/Linux
 .Dt xbps-repo 8
 .Sh NAME
@@ -64,13 +64,12 @@ Prints the name of
 .Em package(s)
 matching the pattern on its file list by looking in all repositories index files.
 Multiple patterns can be specified as arguments.
-.It Sy genindex Pa /path/to/local/repo
+.It Sy index-add Ar /path/to/local/repository/foo-1.0.arch.xbps
-Generates the package index files for a local repository as specified in its argument.
+This will register the binary package into the local repository's index files, and remove
-It will look for archives with the
+old entry and binary package if any old version exists.
-.Sy .xbps
+Multiple binary packages can be specified.
-extension and will only add them onto the index if version is newer than the one
+.It Sy index-clean Ar /path/to/local/repository
-currently stored. If a newer package is available, the old binary package will be
+This will remove any obsolete entry found in the local repository's index files.
-removed automatically.
 .It Sy list
 Lists all working repositories in repository pool.
 .It Sy pkg-list Op repository

include/xbps_api.h

@@ -57,7 +57,7 @@
 #define XBPS_PKGINDEX_VERSION	"1.5"
 
 #define XBPS_API_VERSION	"20120714"
-#define XBPS_VERSION		"0.16.5"
+#define XBPS_VERSION		"0.16.6"
 
 /**
  * @def XBPS_RELVER