Get rid of repodata index-files; the archive is now 8x smaller.

See the NEWS file for more information.

NEWS

@@ -1,5 +1,13 @@
 xbps-0.42 (???):
 
+ * Get rid of repodata index-files, that contained all pkg files of a repository
+   in the archive. That file was so huge that incresed the repository archive
+   size by 8x. That means that `xbps-query -Ro` will now make N parallel connections
+   to the remote server to inspect the binary packages remotely; if the binary
+   package is in cachedir, no network connection to the remote server will be
+   established. This reduces the size of the main x86_64 repository (4000 packages)
+   to 700KB, compared to 4.5MB previously.
+
  * xbps-create(8): fixed issue #64 "xbps-create(8) incorrect installed-size with hardlinks".
    https://github.com/voidlinux/xbps/issues/64
 

bin/xbps-query/ownedby.c

@@ -119,30 +119,37 @@ ownedby_pkgdb_cb(struct xbps_handle *xhp,
 
 
 static int
-repo_match_cb(struct xbps_handle *xhp _unused,
+repo_match_cb(struct xbps_handle *xhp,
 		xbps_object_t obj,
-		const char *key,
+		const char *key _unused,
 		void *arg,
 		bool *done _unused)
 {
+	xbps_dictionary_t filesd;
+	xbps_array_t files_keys;
 	struct ffdata *ffd = arg;
-	const char *filestr;
+	const char *pkgver;
-	regex_t regex;
+	char *bfile;
 
-	for (unsigned int i = 0; i < xbps_array_count(obj); i++) {
+	xbps_dictionary_set_cstring_nocopy(obj, "repository", ffd->repouri);
-		xbps_array_get_cstring_nocopy(obj, i, &filestr);
+	xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
-		if (ffd->regex) {
+
-			if (regcomp(&regex, ffd->pat, REG_EXTENDED|REG_NOSUB) != 0)
+	bfile = xbps_repository_pkg_path(xhp, obj);
-				return errno;
+	assert(bfile);
-			if (regexec(&regex, filestr, 0, 0, 0) == 0) {
+	filesd = xbps_get_pkg_plist_from_binpkg(bfile, "./files.plist");
-				printf("%s: %s (%s)\n", key, filestr, ffd->repouri);
+	if (filesd == NULL) {
-			}
+		xbps_dbg_printf(xhp, "%s: couldn't fetch files.plist from %s: %s\n",
-			regfree(&regex);
+		    pkgver, bfile, strerror(errno));
-		} else {
+		return EINVAL;
-			if ((fnmatch(ffd->pat, filestr, FNM_PERIOD)) == 0)
-				printf("%s: %s (%s)\n", key, filestr, ffd->repouri);
-		}
 	}
+	files_keys = xbps_dictionary_all_keys(filesd);
+	for (unsigned int i = 0; i < xbps_array_count(files_keys); i++) {
+		match_files_by_pattern(filesd,
+		    xbps_array_get(files_keys, i), ffd, pkgver);
+	}
+	xbps_object_release(files_keys);
+	xbps_object_release(filesd);
+	free(bfile);
 
 	return 0;
 }
@@ -154,13 +161,9 @@ repo_ownedby_cb(struct xbps_repo *repo, void *arg, bool *done _unused)
 	struct ffdata *ffd = arg;
 	int rv;
 
-	xbps_repo_open_idxfiles(repo);
-	if (repo->idxfiles == NULL)
-		return 0;
-
 	ffd->repouri = repo->uri;
-	allkeys = xbps_dictionary_all_keys(repo->idxfiles);
+	allkeys = xbps_dictionary_all_keys(repo->idx);
-	rv = xbps_array_foreach_cb(repo->xhp, allkeys, repo->idxfiles, repo_match_cb, ffd);
+	rv = xbps_array_foreach_cb_multi(repo->xhp, allkeys, repo->idx, repo_match_cb, ffd);
 	xbps_object_release(allkeys);
 
 	return rv;

bin/xbps-rindex/defs.h

@@ -81,6 +81,6 @@ int	sign_repo(struct xbps_handle *, const char *, const char *,
 
 /* From repoflush.c */
 bool	repodata_flush(struct xbps_handle *, const char *,
-		xbps_dictionary_t, xbps_dictionary_t, xbps_dictionary_t);
+		xbps_dictionary_t, xbps_dictionary_t);
 
 #endif /* !_XBPS_RINDEX_DEFS_H_ */

bin/xbps-rindex/index-add.c

@@ -40,14 +40,12 @@
 int
 index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force)
 {
-	xbps_array_t array, pkg_files, pkg_links, pkg_cffiles;
+	xbps_dictionary_t idx, idxmeta, binpkgd, curpkgd;
-	xbps_dictionary_t idx, idxmeta, idxfiles, binpkgd, pkg_filesd, curpkgd;
-	xbps_object_t obj, fileobj;
 	struct xbps_repo *repo = NULL;
 	struct stat st;
 	char *tmprepodir = NULL, *repodir = NULL;
 	int rv = 0, ret = 0;
-	bool flush = false, found = false;
+	bool flush = false;
 
 	assert(argv);
 	/*
@@ -64,15 +62,12 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
 		rv = -1;
 		goto out;
 	}
-	if (repo && repo->idx) {
+	if (repo) {
-		xbps_repo_open_idxfiles(repo);
 		idx = xbps_dictionary_copy(repo->idx);
 		idxmeta = xbps_dictionary_copy(repo->idxmeta);
-		idxfiles = xbps_dictionary_copy(repo->idxfiles);
 	} else {
 		idx = xbps_dictionary_create();
 		idxmeta = NULL;
-		idxfiles = xbps_dictionary_create();
 	}
 	/*
 	 * Process all packages specified in argv.
@@ -153,7 +148,6 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
 			 */
 			printf("index: removed obsolete entry `%s' (%s).\n", opkgver, oarch);
 			xbps_dictionary_remove(idx, pkgname);
-			xbps_dictionary_remove(idxfiles, opkgver);
 			free(opkgver);
 			free(oarch);
 		}
@@ -205,87 +199,19 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
 		printf("index: added `%s' (%s).\n", pkgver, arch);
 		xbps_object_release(binpkgd);
 		free(pkgname);
-		/*
-		 * Add new pkg dictionary into the index-files.
-		 */
-		found = false;
-		pkg_filesd = xbps_get_pkg_plist_from_binpkg(pkg, "./files.plist");
-		if (pkg_filesd == NULL) {
-			free(pkgver);
-			rv = EINVAL;
-			goto out;
-		}
-
-		pkg_cffiles = xbps_dictionary_get(pkg_filesd, "conf_files");
-		if (xbps_array_count(pkg_cffiles))
-			found = true;
-		else
-			pkg_cffiles = NULL;
-
-		pkg_files = xbps_dictionary_get(pkg_filesd, "files");
-		if (xbps_array_count(pkg_files))
-			found = true;
-		else
-			pkg_files = NULL;
-
-		pkg_links = xbps_dictionary_get(pkg_filesd, "links");
-		if (xbps_array_count(pkg_links))
-			found = true;
-		else
-			pkg_links = NULL;
-
-		/* If pkg does not contain any file, ignore it */
-		if (!found) {
-			xbps_object_release(pkg_filesd);
-			free(pkgver);
-			continue;
-		}
-		/* create pkg files array */
-		array = xbps_array_create();
-		assert(array);
-
-		/* add conf_files in pkg files array */
-		if (pkg_cffiles != NULL) {
-			for (unsigned int x = 0; x < xbps_array_count(pkg_cffiles); x++) {
-				obj = xbps_array_get(pkg_cffiles, x);
-				fileobj = xbps_dictionary_get(obj, "file");
-				xbps_array_add(array, fileobj);
-			}
-		}
-		/* add files array in pkg files array */
-		if (pkg_files != NULL) {
-			for (unsigned int x = 0; x < xbps_array_count(pkg_files); x++) {
-				obj = xbps_array_get(pkg_files, x);
-				fileobj = xbps_dictionary_get(obj, "file");
-				xbps_array_add(array, fileobj);
-			}
-		}
-		/* add links array in pkg files array */
-		if (pkg_links != NULL) {
-			for (unsigned int x = 0; x < xbps_array_count(pkg_links); x++) {
-				obj = xbps_array_get(pkg_links, x);
-				fileobj = xbps_dictionary_get(obj, "file");
-				xbps_array_add(array, fileobj);
-			}
-		}
-		/* add pkg files array into index-files */
-		xbps_dictionary_set(idxfiles, pkgver, array);
-		xbps_object_release(array);
-		xbps_object_release(pkg_filesd);
 		free(pkgver);
 	}
 	/*
 	 * Generate repository data files.
 	 */
 	if (flush) {
-		if (!repodata_flush(xhp, repodir, idx, idxfiles, idxmeta)) {
+		if (!repodata_flush(xhp, repodir, idx, idxmeta)) {
 			fprintf(stderr, "%s: failed to write repodata: %s\n",
 			    _XBPS_RINDEX, strerror(errno));
 			goto out;
 		}
 	}
 	printf("index: %u packages registered.\n", xbps_dictionary_count(idx));
-	printf("index-files: %u packages registered.\n", xbps_dictionary_count(idxfiles));
 
 out:
 	if (repo)

bin/xbps-rindex/index-clean.c

@@ -85,32 +85,6 @@ idx_cleaner_cb(struct xbps_handle *xhp,
 	return 0;
 }
 
-static int
-idxfiles_cleaner_cb(struct xbps_handle *xhp _unused, xbps_object_t obj _unused,
-		const char *key, void *arg, bool *done _unused)
-{
-	xbps_dictionary_t pkg;
-	struct cbdata *cbd = arg;
-	char *pkgname;
-	const char *pkgver;
-
-	/* Find out entries on index-files that aren't registered on index */
-	if ((pkgname = xbps_pkg_name(key)) == NULL) {
-		xbps_dbg_printf(xhp, "%s: invalid entry found on index-files: %s\n", cbd->repourl, key);
-		return 0;
-	}
-	if ((pkg = xbps_dictionary_get(cbd->idx, pkgname))) {
-		xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver);
-		if (strcmp(pkgver, key)) {
-			pthread_mutex_lock(&cbd->mtx);
-			xbps_array_add_cstring_nocopy(cbd->result, key);
-			pthread_mutex_unlock(&cbd->mtx);
-		}
-	}
-	free(pkgname);
-
-	return 0;
-}
 /*
  * Removes stalled pkg entries in repository's XBPS_REPOIDX file, if any
  * binary package cannot be read (unavailable, not enough perms, etc).
@@ -119,7 +93,7 @@ int
 index_clean(struct xbps_handle *xhp, const char *repodir)
 {
 	xbps_array_t allkeys;
-	xbps_dictionary_t idx = NULL, idxmeta = NULL, idxfiles = NULL;
+	xbps_dictionary_t idx = NULL, idxmeta = NULL;
 	struct xbps_repo *repo;
 	struct cbdata cbd;
 	int rv = 0;
@@ -135,11 +109,9 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
 		    _XBPS_RINDEX, strerror(errno));
 		return rv;
 	}
-	xbps_repo_open_idxfiles(repo);
 	idx = xbps_dictionary_copy(repo->idx);
 	idxmeta = xbps_dictionary_copy(repo->idxmeta);
-	idxfiles = xbps_dictionary_copy(repo->idxfiles);
+	if (idx == NULL) {
-	if (idx == NULL || idxfiles == NULL) {
 		fprintf(stderr, "%s: incomplete repository data file!\n", _XBPS_RINDEX);
 		rv = EINVAL;
 		goto out;
@@ -159,41 +131,20 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
 		char *keyname = NULL, *pkgname = NULL;
 
 		xbps_array_get_cstring(cbd.result, x, &keyname);
-		printf("index-files: removed entry %s\n", keyname);
 		printf("index: removed entry %s\n", keyname);
 		pkgname = xbps_pkg_name(keyname);
 		assert(pkgname);
-		xbps_dictionary_remove(idxfiles, keyname);
 		xbps_dictionary_remove(idx, pkgname);
 		free(pkgname);
 		free(keyname);
 		flush = true;
 	}
-	/*
-	 * Second pass: find out obsolete entries on index-files.
-	 */
-	xbps_object_release(cbd.result);
-	xbps_object_release(allkeys);
-	cbd.idx = idx;
-	cbd.result = xbps_array_create();
-	allkeys = xbps_dictionary_all_keys(idxfiles);
-	(void)xbps_array_foreach_cb_multi(xhp, allkeys, idxfiles, idxfiles_cleaner_cb, &cbd);
-	for (unsigned int x = 0; x < xbps_array_count(cbd.result); x++) {
-		char *keyname = NULL;
-
-		xbps_array_get_cstring(cbd.result, x, &keyname);
-		printf("index-files: removed entry %s\n", keyname);
-		xbps_dictionary_remove(idxfiles, keyname);
-		free(keyname);
-		flush = true;
-	}
-
 	pthread_mutex_destroy(&cbd.mtx);
 	xbps_object_release(cbd.result);
 	xbps_object_release(allkeys);
 
 	if (flush) {
-		if (!repodata_flush(xhp, repodir, idx, idxfiles, idxmeta)) {
+		if (!repodata_flush(xhp, repodir, idx, idxmeta)) {
 			rv = errno;
 			fprintf(stderr, "failed to write repodata: %s\n",
 			    strerror(errno));
@@ -202,8 +153,6 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
 	}
 	printf("index: %u packages registered.\n",
 			xbps_dictionary_count(idx));
-	printf("index-files: %u packages registered.\n",
-			xbps_dictionary_count(idxfiles));
 
 out:
 	xbps_repo_close(repo, true);
@@ -211,8 +160,6 @@ out:
 		xbps_object_release(idx);
 	if (idxmeta)
 		xbps_object_release(idxmeta);
-	if (idxfiles)
-		xbps_object_release(idxfiles);
 
 	return rv;
 }

bin/xbps-rindex/repoflush.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2013 Juan Romero Pardines.
+ * Copyright (c) 2013-2014 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,8 +39,7 @@
 
 bool
 repodata_flush(struct xbps_handle *xhp, const char *repodir,
-	xbps_dictionary_t idx, xbps_dictionary_t idxfiles,
+	xbps_dictionary_t idx, xbps_dictionary_t meta)
-	xbps_dictionary_t meta)
 {
 	struct archive *ar;
 	char *repofile, *tname, *buf;
@@ -82,15 +81,6 @@ repodata_flush(struct xbps_handle *xhp, const char *repodir,
 	if (rv != 0)
 		return false;
 
-	/* XBPS_REPOIDX_FILES */
-	buf = xbps_dictionary_externalize(idxfiles);
-	assert(buf);
-	rv = xbps_archive_append_buf(ar, buf, strlen(buf),
-	    XBPS_REPOIDX_FILES, 0644, "root", "root");
-	free(buf);
-	if (rv != 0)
-		return false;
-
 	/* Write data to tempfile and rename */
 	archive_write_finish(ar);
 	fdatasync(repofd);

bin/xbps-rindex/sign.c

@@ -153,7 +153,6 @@ sign_repo(struct xbps_handle *xhp, const char *repodir,
 		rv = EINVAL;
 		goto out;
 	}
-	xbps_repo_open_idxfiles(repo);
 	/*
 	 * If privkey not set, default to ~/.ssh/id_rsa.
 	 */
@@ -286,7 +285,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir,
 	xbps_object_release(data);
 	data = NULL;
 
-	if (!repodata_flush(xhp, repodir, repo->idx, repo->idxfiles, meta)) {
+	if (!repodata_flush(xhp, repodir, repo->idx, meta)) {
 		fprintf(stderr, "failed to write repodata: %s\n", strerror(errno));
 		goto out;
 	}

include/xbps.h.in

@@ -48,7 +48,7 @@
  *
  * This header documents the full API for the XBPS Library.
  */
-#define XBPS_API_VERSION	"20141113"
+#define XBPS_API_VERSION	"20141113-1"
 
 #ifndef XBPS_VERSION
  #define XBPS_VERSION		"UNSET"
@@ -112,12 +112,6 @@
  */
 #define XBPS_REPOIDX		"index.plist"
 
-/**
- * @def XBPS_REPOIDX_FILES
- * Filename for the repository index-files property list.
- */
-#define XBPS_REPOIDX_FILES	"index-files.plist"
-
 /**
  * @def XBPS_REPOIDX_META
  * Filename for the repository index metadata property list.
@@ -1188,12 +1182,6 @@ struct xbps_repo {
 	 * Proplib dictionary associated with the repository index-meta.
 	 */
 	xbps_dictionary_t idxmeta;
-	/**
-	 * @var idxfiles
-	 *
-	 * Proplib dictionary associated with the repository index-files.
-	 */
-	xbps_dictionary_t idxfiles;
 	/**
 	 * @var uri
 	 * 
@@ -1345,15 +1333,6 @@ struct xbps_repo *xbps_repo_open(struct xbps_handle *xhp, const char *url, bool
  */
 void xbps_repo_close(struct xbps_repo *repo, bool lock);
 
-
-/**
- * Prepares the repository index-files.plist to have access to it.
- * The repository must be opened previously with \a xbps_repo_open().
- *
- * @param[in] repo The repository object to use.
- */
-void xbps_repo_open_idxfiles(struct xbps_repo *repo);
-
 /**
  *
  * Returns a heap-allocated string with the repository local path.
@@ -1668,6 +1647,23 @@ bool xbps_binpkg_exists(struct xbps_handle *xhp, xbps_dictionary_t pkgd);
  */
 bool xbps_repository_is_remote(const char *uri);
 
+/*
+ * Returns an allocated string with the full path to the binary package
+ * matching \a pkgd.
+ *
+ * The \a pkgd dictionary must contain the following objects:
+ *  - architecture (string)
+ *  - pkgver (string)
+ *  - repository (string)
+ *
+ * @param[in] xhp The pointer to an xbps_handle struct.
+ * @param[in] pkgd The package dictionary to match.
+ *
+ * @return A malloc(3)ed buffer with the full path, NULL otherwise.
+ * The buffer should be free(3)d when it's no longer needed.
+ */
+char *xbps_repository_pkg_path(struct xbps_handle *xhp, xbps_dictionary_t pkgd);
+
 /**
  * Gets the name of a package string. Package strings are composed
  * by a @<pkgname@>/@<version@> pair and separated by the <em>minus</em>

include/xbps_api_impl.h

@@ -127,12 +127,6 @@ bool HIDDEN xbps_remove_pkg_from_array_by_name(xbps_array_t, const char *);
 bool HIDDEN xbps_remove_pkg_from_array_by_pattern(xbps_array_t, const char *);
 bool HIDDEN xbps_remove_pkg_from_array_by_pkgver(xbps_array_t, const char *);
 
-/**
- * @private
- * From lib/util.c
- */
-char HIDDEN *xbps_repository_pkg_path(struct xbps_handle *, xbps_dictionary_t);
-
 /**
  * @private
  * From lib/download.c

lib/repo.c

@@ -202,13 +202,6 @@ out:
 	return NULL;
 }
 
-void
-xbps_repo_open_idxfiles(struct xbps_repo *repo)
-{
-	assert(repo);
-	repo->idxfiles = repo_get_dict(repo);
-}
-
 void
 xbps_repo_close(struct xbps_repo *repo, bool lock)
 {
@@ -225,10 +218,6 @@ xbps_repo_close(struct xbps_repo *repo, bool lock)
 		xbps_object_release(repo->idxmeta);
 		repo->idxmeta = NULL;
 	}
-	if (repo->idxfiles != NULL) {
-		xbps_object_release(repo->idxfiles);
-		repo->idxfiles = NULL;
-	}
         if (lock && lockf(repo->fd, F_ULOCK, 0) == -1)
 		xbps_dbg_printf(repo->xhp, "[repo] failed to unlock %s: %s\n", repo->uri, strerror(errno));
 

lib/util.c

@@ -280,7 +280,7 @@ xbps_pkgpattern_version(const char *pkg)
 	return strpbrk(pkg, "><*?[]");
 }
 
-char HIDDEN *
+char *
 xbps_repository_pkg_path(struct xbps_handle *xhp, xbps_dictionary_t pkg_repod)
 {
 	const char *pkgver, *arch, *repoloc;

tests/xbps/xbps-rindex/clean_test.sh

@@ -22,63 +22,7 @@ noremove_body() {
 	atf_check_equal ${result} 1
 }
 
-# 2nd test: make sure that entries are also removed from index-files.
+# xbps issue #19.
-atf_test_case filesclean
-
-filesclean_head() {
-	atf_set "descr" "xbps-rindex(8) -c: index-files clean test"
-}
-
-filesclean_body() {
-	mkdir -p some_repo pkg_A
-	touch -f pkg_A/file00
-	cd some_repo
-	xbps-create -A noarch -n foo-1.0_1 -s "foo pkg" ../pkg_A
-	atf_check_equal $? 0
-	xbps-rindex -d -a $PWD/*.xbps
-	atf_check_equal $? 0
-	rm *.xbps
-	cd ..
-	xbps-rindex -c some_repo
-	atf_check_equal $? 0
-	result=$(xbps-query -r root -C empty.conf --repository=some_repo -o \*)
-	test -z "${result}"
-	atf_check_equal $? 0
-}
-
-# 3nd test: make sure that entries are removed from index-files on updates.
-atf_test_case filesclean2
-
-filesclean2_head() {
-	atf_set "descr" "xbps-rindex(8) -c: index-files clean test on updates"
-}
-
-filesclean2_body() {
-	mkdir -p some_repo pkg_A
-	touch -f pkg_A/file00
-	cd some_repo
-	xbps-create -A noarch -n foo-1.0_1 -s "foo pkg" ../pkg_A
-	atf_check_equal $? 0
-	xbps-rindex -d -a $PWD/*.xbps
-	atf_check_equal $? 0
-	xbps-create -A noarch -n foo-1.1_1 -s "foo pkg" ../pkg_A
-	atf_check_equal $? 0
-	xbps-rindex -d -a $PWD/*.xbps
-	atf_check_equal $? 0
-	cd ..
-	xbps-rindex -c some_repo
-	atf_check_equal $? 0
-	result="$(xbps-query -r root -C empty.conf --repository=some_repo -o \*)"
-	expected="foo-1.1_1: /file00 (some_repo)"
-	rv=0
-	if [ "$result" != "$expected" ]; then
-		rv=1
-	fi
-	atf_check_equal $rv 0
-
-}
-
-# 4th test: xbps issue #19.
 # How to reproduce it:
 #	Generate pkg foo-1.0_1.
 #	Add it to the index of a local repo.
@@ -109,7 +53,5 @@ issue19_body() {
 
 atf_init_test_cases() {
 	atf_add_test_case noremove
-	atf_add_test_case filesclean
-	atf_add_test_case filesclean2
 	atf_add_test_case issue19
 }