Use a single file to store repository data.

This is just the starting point to extend repositories with PGP signatures.

bin/xbps-rindex/Makefile

@@ -2,7 +2,7 @@ TOPDIR = ../..
 -include $(TOPDIR)/config.mk
 
 BIN =	xbps-rindex
-OBJS =	main.o index-add.o index-clean.o remove-obsoletes.o
+OBJS =	main.o index-add.o index-clean.o remove-obsoletes.o repoflush.o
 MAN = $(BIN).8
 
 include $(TOPDIR)/mk/prog.mk

bin/xbps-rindex/defs.h

@@ -41,4 +41,8 @@ int	index_files_clean(struct xbps_handle *, const char *);
 /* From remove-obsoletes.c */
 int	remove_obsoletes(struct xbps_handle *, const char *);
 
+/* From repoflush.c */
+int	repodata_flush(struct xbps_handle *, const char *,
+		prop_dictionary_t, prop_dictionary_t);
+
 #endif /* !_XBPS_RINDEX_DEFS_H_ */

bin/xbps-rindex/index-add.c

@@ -32,8 +32,11 @@
 #include <dirent.h>
 #include <libgen.h>
 #include <assert.h>
+#include <fcntl.h>
 
 #include <xbps_api.h>
+#include <archive.h>
+#include <archive_entry.h>
 #include "defs.h"
 
 /*
@@ -45,56 +48,37 @@ index_add(struct xbps_handle *xhp, int argc, char **argv, bool force)
 {
 	prop_array_t filespkgar, pkg_files, pkg_links, pkg_cffiles;
 	prop_dictionary_t idx, idxfiles, newpkgd, newpkgfilesd, curpkgd;
-	prop_dictionary_t filespkgd;
 	prop_object_t obj, fileobj;
+	struct xbps_repo *repo;
 	struct stat st;
 	const char *oldpkgver, *arch, *oldarch;
 	char *pkgver, *pkgname, *sha256, *repodir, *buf;
-	char *tmprepodir, *plist, *plistf;
+	char *tmprepodir;
 	size_t x;
-	int i, ret = 0;
-	bool files_flush = false, found = false, flush = false;
+	int i, rv, ret = 0;
+	bool flush = false, found = false;
 
 	idx = idxfiles = newpkgd = newpkgfilesd = curpkgd = NULL;
-	tmprepodir = plist = plistf = NULL;
 
 	if ((tmprepodir = strdup(argv[0])) == NULL)
 		return ENOMEM;
 
+	/*
+	 * Read the repository data or create index dictionaries otherwise.
+	 */
 	repodir = dirname(tmprepodir);
 
-	/* Internalize index or create it if doesn't exist */
-	if ((plist = xbps_pkg_index_plist(xhp, repodir)) == NULL) {
-		free(tmprepodir);
-		return -1;
-	}
-
-	if ((idx = prop_dictionary_internalize_from_zfile(plist)) == NULL) {
-		if (errno != ENOENT) {
-			fprintf(stderr, "index: cannot read `%s': %s\n",
-			    plist, strerror(errno));
-			return -1;
-		} else {
-			idx = prop_dictionary_create();
-			assert(idx);
-		}
-	}
-	/* Internalize index-files or create it if doesn't exist */
-	if ((plistf = xbps_pkg_index_files_plist(xhp, repodir)) == NULL)
-		return -1;
-
-	free(tmprepodir);
-
-	if ((idxfiles = prop_dictionary_internalize_from_zfile(plistf)) == NULL) {
-		if (errno != ENOENT) {
-			fprintf(stderr, "index: cannot read `%s': %s\n",
-			    plistf, strerror(errno));
-			return -1;
-		} else {
-			idxfiles = prop_dictionary_create();
-			assert(idx);
-		}
+	repo = xbps_repo_open(xhp, repodir);
+	if (repo != NULL) {
+		idx = xbps_repo_get_plist(repo, XBPS_PKGINDEX);
+		idxfiles = xbps_repo_get_plist(repo, XBPS_PKGINDEX_FILES);
 	}
+	if (idx == NULL)
+		idx = prop_dictionary_create();
+	if (idxfiles == NULL)
+		idxfiles = prop_dictionary_create();
+	if (repo != NULL)
+		xbps_repo_close(repo);
 
 	/*
 	 * Process all packages specified in argv.
@@ -212,7 +196,6 @@ index_add(struct xbps_handle *xhp, int argc, char **argv, bool force)
 			free(pkgname);
 			return EINVAL;
 		}
-
 		flush = true;
 		printf("index: added `%s' (%s).\n", pkgver, arch);
 		/*
@@ -284,43 +267,29 @@ index_add(struct xbps_handle *xhp, int argc, char **argv, bool force)
 		}
 		prop_object_release(newpkgfilesd);
 
-		/* create pkg dictionary */
-		filespkgd = prop_dictionary_create();
-		assert(filespkgd);
-
-		/* add pkg files array into pkg dictionary */
-		prop_dictionary_set(filespkgd, "files", filespkgar);
+		/* add pkg files array into index-files */
+		prop_dictionary_set(idxfiles, pkgver, filespkgar);
 		prop_object_release(filespkgar);
 
-		/* set pkgver obj into pkg dictionary */
-		prop_dictionary_set_cstring(filespkgd, "pkgver", pkgver);
-
-		/* add pkg dictionary into index-files */
-		prop_dictionary_set(idxfiles, pkgname, filespkgd);
-		prop_object_release(filespkgd);
-
 		printf("index-files: added `%s' (%s)\n", pkgver, arch);
-		files_flush = true;
 		prop_object_release(newpkgd);
 		free(pkgver);
 		free(pkgname);
 	}
-
-	if (flush && !prop_dictionary_externalize_to_zfile(idx, plist)) {
-		fprintf(stderr, "index: failed to externalize plist: %s\n",
-		    strerror(errno));
-		return -1;
-	}
-	if (files_flush &&
-	    !prop_dictionary_externalize_to_zfile(idxfiles, plistf)) {
-		fprintf(stderr, "index-files: failed to externalize "
-		    "plist: %s\n", strerror(errno));
-		return -1;
+	/*
+	 * Generate repository data file.
+	 */
+	if (flush) {
+		rv = repodata_flush(xhp, repodir, idx, idxfiles);
+		if (rv != 0)
+			return rv;
 	}
 	printf("index: %u packages registered.\n",
 	    prop_dictionary_count(idx));
 	printf("index-files: %u packages registered.\n",
 	    prop_dictionary_count(idxfiles));
+	prop_object_release(idx);
+	prop_object_release(idxfiles);
 
 	return 0;
 }

bin/xbps-rindex/index-clean.c

@@ -33,6 +33,7 @@
 #include <libgen.h>
 #include <assert.h>
 #include <pthread.h>
+#include <fcntl.h>
 
 #include <xbps_api.h>
 #include "defs.h"
@@ -99,8 +100,10 @@ cleaner_files_thread(void *arg)
 {
 	prop_object_t obj;
 	prop_array_t array;
+	prop_dictionary_t ipkgd;
 	struct thread_data *thd = arg;
-	const char *pkgname;
+	const char *pkgver, *ipkgver;
+	char *pkgname;
 	unsigned int i;
 
 	/* process pkgs from start until end */
@@ -108,10 +111,20 @@ cleaner_files_thread(void *arg)
 
 	for (i = thd->start; i < thd->end; i++) {
 		obj = prop_array_get(array, i);
-		pkgname = prop_dictionary_keysym_cstring_nocopy(obj);
+		pkgver = prop_dictionary_keysym_cstring_nocopy(obj);
+		pkgname = xbps_pkg_name(pkgver);
+		assert(pkgname);
+		ipkgd = prop_dictionary_get(thd->idx, pkgname);
 		/* If pkg is not registered in index, remove it */
-		if (!prop_dictionary_get(thd->idx, pkgname))
-			prop_array_add_cstring_nocopy(thd->result_files, pkgname);
+		if (ipkgd == NULL)
+			prop_array_add_cstring_nocopy(thd->result_files, pkgver);
+		/* if another version is registered in index, remove it */
+		else {
+			prop_dictionary_get_cstring_nocopy(ipkgd, "pkgver", &ipkgver);
+			if (strcmp(ipkgver, pkgver))
+				prop_array_add_cstring_nocopy(thd->result_files, pkgver);
+		}
+		free(pkgname);
 	}
 	prop_object_release(array);
 
@@ -125,36 +138,28 @@ cleaner_files_thread(void *arg)
 int
 index_clean(struct xbps_handle *xhp, const char *repodir)
 {
+	struct xbps_repo *repo;
 	struct thread_data *thd;
 	prop_dictionary_t idx, idxfiles;
 	const char *keyname;
-	char *plist, *plistf, *pkgname;
+	char *pkgname;
 	size_t x, pkgcount, slicecount;
 	int i, maxthreads, rv = 0;
 	bool flush = false;
 
-	plist = xbps_pkg_index_plist(xhp, repodir);
-	assert(plist);
-	plistf = xbps_pkg_index_files_plist(xhp, repodir);
-	assert(plistf);
-
-	idx = prop_dictionary_internalize_from_zfile(plist);
-	if (idx == NULL) {
-		if (errno != ENOENT) {
-			fprintf(stderr, "index: cannot read `%s': %s\n",
-			    plist, strerror(errno));
-			return -1;
-		} else
+	repo = xbps_repo_open(xhp, repodir);
+	if (repo == NULL) {
+		if (errno == ENOENT)
 			return 0;
+		fprintf(stderr, "index: cannot read repository data: %s\n", strerror(errno));
+		return -1;
 	}
-	idxfiles = prop_dictionary_internalize_from_zfile(plistf);
-	if (idxfiles == NULL) {
-		if (errno != ENOENT) {
-			fprintf(stderr, "index: cannot read `%s': %s\n",
-			    plistf, strerror(errno));
-			return -1;
-		} else
-			return 0;
+	idx = xbps_repo_get_plist(repo, XBPS_PKGINDEX);
+	idxfiles = xbps_repo_get_plist(repo, XBPS_PKGINDEX_FILES);
+	xbps_repo_close(repo);
+	if (idx == NULL || idxfiles == NULL) {
+		fprintf(stderr, "incomplete repository data file!");
+		return -1;
 	}
 	if (chdir(repodir) == -1) {
 		fprintf(stderr, "index: cannot chdir to %s: %s\n",
@@ -216,7 +221,7 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
 			printf("index: removed entry %s\n", keyname);
 			pkgname = xbps_pkg_name(keyname);
 			prop_dictionary_remove(idx, pkgname);
-			prop_dictionary_remove(idxfiles, pkgname);
+			prop_dictionary_remove(idxfiles, keyname);
 			free(pkgname);
 			flush = true;
 		}
@@ -228,23 +233,17 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
 			flush = true;
 		}
 	}
-
-	if (!flush)
-		goto out;
-
-	if (!prop_dictionary_externalize_to_zfile(idx, plist))
-		fprintf(stderr, "index: failed to externalize %s: %s\n",
-		    plist, strerror(errno));
-
-	if (!prop_dictionary_externalize_to_zfile(idxfiles, plistf))
-		fprintf(stderr, "index-files: failed to externalize %s: %s\n",
-		    plistf, strerror(errno));
-
-out:
+	if (flush) {
+		rv = repodata_flush(xhp, repodir, idx, idxfiles);
+		if (rv != 0)
+			return rv;
+	}
 	printf("index: %u packages registered.\n",
 	    prop_dictionary_count(idx));
 	printf("index-files: %u packages registered.\n",
 	    prop_dictionary_count(idxfiles));
+	prop_object_release(idx);
+	prop_object_release(idxfiles);
 
 	return rv;
 }

bin/xbps-rindex/remove-obsoletes.c

@@ -40,7 +40,7 @@
 struct thread_data {
 	pthread_t thread;
 	prop_array_t array;
-	struct xbps_rindex *ri;
+	struct xbps_repo *repo;
 	unsigned int start;
 	unsigned int end;
 	int thread_num;
@@ -95,7 +95,7 @@ cleaner_thread(void *arg)
 		prop_array_get_cstring_nocopy(thd->array, i, &binpkg);
 		pkgd = xbps_get_pkg_plist_from_binpkg(binpkg, "./props.plist");
 		if (pkgd == NULL) {
-			rv = remove_pkg(thd->ri->uri, arch, binpkg);
+			rv = remove_pkg(thd->repo->uri, arch, binpkg);
 			if (rv != 0) {
 				prop_object_release(pkgd);
 				continue;
@@ -105,17 +105,17 @@ cleaner_thread(void *arg)
 		prop_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver);
 		prop_dictionary_get_cstring_nocopy(pkgd, "architecture", &arch);
 		/* ignore pkgs from other archs */
-		if (!xbps_pkg_arch_match(thd->ri->xhp, arch, NULL)) {
+		if (!xbps_pkg_arch_match(thd->repo->xhp, arch, NULL)) {
 			prop_object_release(pkgd);
 			continue;
 		}
-		xbps_dbg_printf(thd->ri->xhp, "thread[%d] checking %s (%s)\n",
+		xbps_dbg_printf(thd->repo->xhp, "thread[%d] checking %s (%s)\n",
 		    thd->thread_num, pkgver, binpkg);
 		/*
 		 * If binpkg is not registered in index, remove binpkg.
 		 */
-		if (!xbps_rindex_get_pkg(thd->ri, pkgver)) {
-			rv = remove_pkg(thd->ri->uri, arch, binpkg);
+		if (!xbps_repo_get_pkg(thd->repo, pkgver)) {
+			rv = remove_pkg(thd->repo->uri, arch, binpkg);
 			if (rv != 0) {
 				prop_object_release(pkgd);
 				continue;
@@ -131,33 +131,28 @@ cleaner_thread(void *arg)
 int
 remove_obsoletes(struct xbps_handle *xhp, const char *repodir)
 {
-	prop_dictionary_t idx;
 	prop_array_t array = NULL;
-	struct xbps_rindex ri;
+	struct xbps_repo *repo;
 	struct thread_data *thd;
 	DIR *dirp;
 	struct dirent *dp;
-	char *plist, *ext;
+	char *ext;
 	int i, maxthreads, rv = 0;
 	size_t slicecount, pkgcount;
 
-	if ((plist = xbps_pkg_index_plist(xhp, repodir)) == NULL)
-		return -1;
-
-	idx = prop_dictionary_internalize_from_zfile(plist);
-	if (idx == NULL) {
+	repo = xbps_repo_open(xhp, repodir);
+	if (repo == NULL) {
 		if (errno != ENOENT) {
-			fprintf(stderr, "xbps-rindex: cannot read `%s': %s\n",
-			    plist, strerror(errno));
+			fprintf(stderr, "xbps-rindex: cannot read repository data: %s\n",
+			    strerror(errno));
 			return -1;
-		} else
-			return 0;
+		}
+		return 0;
+	}
+	if ((repo->idx = xbps_repo_get_plist(repo, XBPS_PKGINDEX)) == NULL) {
+		xbps_repo_close(repo);
+		return -1;
 	}
-	/* initialize repository index */
-	ri.repod = idx;
-	ri.uri = repodir;
-	ri.xhp = xhp;
-
 	if (chdir(repodir) == -1) {
 		fprintf(stderr, "xbps-rindex: cannot chdir to %s: %s\n",
 		    repodir, strerror(errno));
@@ -191,7 +186,7 @@ remove_obsoletes(struct xbps_handle *xhp, const char *repodir)
 	for (i = 0; i < maxthreads; i++) {
 		thd[i].thread_num = i;
 		thd[i].array = array;
-		thd[i].ri = &ri;
+		thd[i].repo = repo;
 		thd[i].start = pkgcount;
 		if (i + 1 >= maxthreads)
 			thd[i].end = prop_array_count(array);
@@ -205,5 +200,7 @@ remove_obsoletes(struct xbps_handle *xhp, const char *repodir)
 	for (i = 0; i < maxthreads; i++)
 		pthread_join(thd[i].thread, NULL);
 
+	xbps_repo_close(repo);
+
 	return rv;
 }

bin/xbps-rindex/repoflush.c

@@ -0,0 +1,94 @@
+/*-
+ * Copyright (c) 2013 Juan Romero Pardines.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/stat.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <dirent.h>
+#include <libgen.h>
+#include <assert.h>
+#include <fcntl.h>
+
+#include <xbps_api.h>
+#include "defs.h"
+
+int
+repodata_flush(struct xbps_handle *xhp, const char *repodir,
+	prop_dictionary_t idx, prop_dictionary_t idxfiles)
+{
+	struct archive *ar;
+	mode_t myumask;
+	char *repofile, *tname, *xml;
+	int repofd;
+
+	/* Create a tempfile for our repository archive */
+	repofile = xbps_repo_path(xhp, repodir);
+	tname = xbps_xasprintf("%s.XXXXXXXXXX", repofile);
+	if ((repofd = mkstemp(tname)) == -1)
+		return errno;
+
+	/* Create and write our repository archive */
+	ar = archive_write_new();
+	assert(ar);
+	archive_write_add_filter_gzip(ar);
+	archive_write_set_format_pax_restricted(ar);
+	archive_write_set_options(ar, "compression-level=9");
+	archive_write_open_fd(ar, repofd);
+
+	xml = prop_dictionary_externalize(idx);
+	assert(xml);
+	if (xbps_archive_append_buf(ar, xml, strlen(xml),
+	    XBPS_PKGINDEX, 0644, "root", "root") != 0) {
+		free(xml);
+		return -1;
+	}
+	free(xml);
+
+	xml = prop_dictionary_externalize(idxfiles);
+	assert(xml);
+	if (xbps_archive_append_buf(ar, xml, strlen(xml),
+	    XBPS_PKGINDEX_FILES, 0644, "root", "root") != 0) {
+		free(xml);
+		return -1;
+	}
+	free(xml);
+
+	archive_write_free(ar);
+
+	/* Write data to tempfile and rename */
+	fdatasync(repofd);
+	myumask = umask(0);
+	(void)umask(myumask);
+	assert(fchmod(repofd, 0666 & ~myumask) != -1);
+	close(repofd);
+	rename(tname, repofile);
+	free(repofile);
+	free(tname);
+
+	return 0;
+}