From 3afb9d709dc615b9fbd9f2f3fb9d1fca32371e86 Mon Sep 17 00:00:00 2001 From: Juan RP Date: Thu, 13 Nov 2014 17:09:43 +0100 Subject: [PATCH] Get rid of repodata index-files; the archive is now 8x smaller. See the NEWS file for more information. --- NEWS | 8 +++ bin/xbps-query/ownedby.c | 49 +++++++++-------- bin/xbps-rindex/defs.h | 2 +- bin/xbps-rindex/index-add.c | 82 ++-------------------------- bin/xbps-rindex/index-clean.c | 59 +------------------- bin/xbps-rindex/repoflush.c | 14 +---- bin/xbps-rindex/sign.c | 3 +- include/xbps.h.in | 40 ++++++-------- include/xbps_api_impl.h | 6 -- lib/repo.c | 11 ---- lib/util.c | 2 +- tests/xbps/xbps-rindex/clean_test.sh | 60 +------------------- 12 files changed, 65 insertions(+), 271 deletions(-) diff --git a/NEWS b/NEWS index cbbe04a6..1338ffac 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,13 @@ xbps-0.42 (???): + * Get rid of repodata index-files, that contained all pkg files of a repository + in the archive. That file was so huge that incresed the repository archive + size by 8x. That means that `xbps-query -Ro` will now make N parallel connections + to the remote server to inspect the binary packages remotely; if the binary + package is in cachedir, no network connection to the remote server will be + established. This reduces the size of the main x86_64 repository (4000 packages) + to 700KB, compared to 4.5MB previously. + * xbps-create(8): fixed issue #64 "xbps-create(8) incorrect installed-size with hardlinks". https://github.com/voidlinux/xbps/issues/64 diff --git a/bin/xbps-query/ownedby.c b/bin/xbps-query/ownedby.c index aebf2228..2ad70df9 100644 --- a/bin/xbps-query/ownedby.c +++ b/bin/xbps-query/ownedby.c @@ -119,30 +119,37 @@ ownedby_pkgdb_cb(struct xbps_handle *xhp, static int -repo_match_cb(struct xbps_handle *xhp _unused, +repo_match_cb(struct xbps_handle *xhp, xbps_object_t obj, - const char *key, + const char *key _unused, void *arg, bool *done _unused) { + xbps_dictionary_t filesd; + xbps_array_t files_keys; struct ffdata *ffd = arg; - const char *filestr; - regex_t regex; + const char *pkgver; + char *bfile; - for (unsigned int i = 0; i < xbps_array_count(obj); i++) { - xbps_array_get_cstring_nocopy(obj, i, &filestr); - if (ffd->regex) { - if (regcomp(®ex, ffd->pat, REG_EXTENDED|REG_NOSUB) != 0) - return errno; - if (regexec(®ex, filestr, 0, 0, 0) == 0) { - printf("%s: %s (%s)\n", key, filestr, ffd->repouri); - } - regfree(®ex); - } else { - if ((fnmatch(ffd->pat, filestr, FNM_PERIOD)) == 0) - printf("%s: %s (%s)\n", key, filestr, ffd->repouri); - } + xbps_dictionary_set_cstring_nocopy(obj, "repository", ffd->repouri); + xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); + + bfile = xbps_repository_pkg_path(xhp, obj); + assert(bfile); + filesd = xbps_get_pkg_plist_from_binpkg(bfile, "./files.plist"); + if (filesd == NULL) { + xbps_dbg_printf(xhp, "%s: couldn't fetch files.plist from %s: %s\n", + pkgver, bfile, strerror(errno)); + return EINVAL; } + files_keys = xbps_dictionary_all_keys(filesd); + for (unsigned int i = 0; i < xbps_array_count(files_keys); i++) { + match_files_by_pattern(filesd, + xbps_array_get(files_keys, i), ffd, pkgver); + } + xbps_object_release(files_keys); + xbps_object_release(filesd); + free(bfile); return 0; } @@ -154,13 +161,9 @@ repo_ownedby_cb(struct xbps_repo *repo, void *arg, bool *done _unused) struct ffdata *ffd = arg; int rv; - xbps_repo_open_idxfiles(repo); - if (repo->idxfiles == NULL) - return 0; - ffd->repouri = repo->uri; - allkeys = xbps_dictionary_all_keys(repo->idxfiles); - rv = xbps_array_foreach_cb(repo->xhp, allkeys, repo->idxfiles, repo_match_cb, ffd); + allkeys = xbps_dictionary_all_keys(repo->idx); + rv = xbps_array_foreach_cb_multi(repo->xhp, allkeys, repo->idx, repo_match_cb, ffd); xbps_object_release(allkeys); return rv; diff --git a/bin/xbps-rindex/defs.h b/bin/xbps-rindex/defs.h index e9ad0de3..2862b77e 100644 --- a/bin/xbps-rindex/defs.h +++ b/bin/xbps-rindex/defs.h @@ -81,6 +81,6 @@ int sign_repo(struct xbps_handle *, const char *, const char *, /* From repoflush.c */ bool repodata_flush(struct xbps_handle *, const char *, - xbps_dictionary_t, xbps_dictionary_t, xbps_dictionary_t); + xbps_dictionary_t, xbps_dictionary_t); #endif /* !_XBPS_RINDEX_DEFS_H_ */ diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 8271fd1c..1fc8feb2 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -40,14 +40,12 @@ int index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force) { - xbps_array_t array, pkg_files, pkg_links, pkg_cffiles; - xbps_dictionary_t idx, idxmeta, idxfiles, binpkgd, pkg_filesd, curpkgd; - xbps_object_t obj, fileobj; + xbps_dictionary_t idx, idxmeta, binpkgd, curpkgd; struct xbps_repo *repo = NULL; struct stat st; char *tmprepodir = NULL, *repodir = NULL; int rv = 0, ret = 0; - bool flush = false, found = false; + bool flush = false; assert(argv); /* @@ -64,15 +62,12 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force rv = -1; goto out; } - if (repo && repo->idx) { - xbps_repo_open_idxfiles(repo); + if (repo) { idx = xbps_dictionary_copy(repo->idx); idxmeta = xbps_dictionary_copy(repo->idxmeta); - idxfiles = xbps_dictionary_copy(repo->idxfiles); } else { idx = xbps_dictionary_create(); idxmeta = NULL; - idxfiles = xbps_dictionary_create(); } /* * Process all packages specified in argv. @@ -153,7 +148,6 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force */ printf("index: removed obsolete entry `%s' (%s).\n", opkgver, oarch); xbps_dictionary_remove(idx, pkgname); - xbps_dictionary_remove(idxfiles, opkgver); free(opkgver); free(oarch); } @@ -205,87 +199,19 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force printf("index: added `%s' (%s).\n", pkgver, arch); xbps_object_release(binpkgd); free(pkgname); - /* - * Add new pkg dictionary into the index-files. - */ - found = false; - pkg_filesd = xbps_get_pkg_plist_from_binpkg(pkg, "./files.plist"); - if (pkg_filesd == NULL) { - free(pkgver); - rv = EINVAL; - goto out; - } - - pkg_cffiles = xbps_dictionary_get(pkg_filesd, "conf_files"); - if (xbps_array_count(pkg_cffiles)) - found = true; - else - pkg_cffiles = NULL; - - pkg_files = xbps_dictionary_get(pkg_filesd, "files"); - if (xbps_array_count(pkg_files)) - found = true; - else - pkg_files = NULL; - - pkg_links = xbps_dictionary_get(pkg_filesd, "links"); - if (xbps_array_count(pkg_links)) - found = true; - else - pkg_links = NULL; - - /* If pkg does not contain any file, ignore it */ - if (!found) { - xbps_object_release(pkg_filesd); - free(pkgver); - continue; - } - /* create pkg files array */ - array = xbps_array_create(); - assert(array); - - /* add conf_files in pkg files array */ - if (pkg_cffiles != NULL) { - for (unsigned int x = 0; x < xbps_array_count(pkg_cffiles); x++) { - obj = xbps_array_get(pkg_cffiles, x); - fileobj = xbps_dictionary_get(obj, "file"); - xbps_array_add(array, fileobj); - } - } - /* add files array in pkg files array */ - if (pkg_files != NULL) { - for (unsigned int x = 0; x < xbps_array_count(pkg_files); x++) { - obj = xbps_array_get(pkg_files, x); - fileobj = xbps_dictionary_get(obj, "file"); - xbps_array_add(array, fileobj); - } - } - /* add links array in pkg files array */ - if (pkg_links != NULL) { - for (unsigned int x = 0; x < xbps_array_count(pkg_links); x++) { - obj = xbps_array_get(pkg_links, x); - fileobj = xbps_dictionary_get(obj, "file"); - xbps_array_add(array, fileobj); - } - } - /* add pkg files array into index-files */ - xbps_dictionary_set(idxfiles, pkgver, array); - xbps_object_release(array); - xbps_object_release(pkg_filesd); free(pkgver); } /* * Generate repository data files. */ if (flush) { - if (!repodata_flush(xhp, repodir, idx, idxfiles, idxmeta)) { + if (!repodata_flush(xhp, repodir, idx, idxmeta)) { fprintf(stderr, "%s: failed to write repodata: %s\n", _XBPS_RINDEX, strerror(errno)); goto out; } } printf("index: %u packages registered.\n", xbps_dictionary_count(idx)); - printf("index-files: %u packages registered.\n", xbps_dictionary_count(idxfiles)); out: if (repo) diff --git a/bin/xbps-rindex/index-clean.c b/bin/xbps-rindex/index-clean.c index 87fc640a..9d90e649 100644 --- a/bin/xbps-rindex/index-clean.c +++ b/bin/xbps-rindex/index-clean.c @@ -85,32 +85,6 @@ idx_cleaner_cb(struct xbps_handle *xhp, return 0; } -static int -idxfiles_cleaner_cb(struct xbps_handle *xhp _unused, xbps_object_t obj _unused, - const char *key, void *arg, bool *done _unused) -{ - xbps_dictionary_t pkg; - struct cbdata *cbd = arg; - char *pkgname; - const char *pkgver; - - /* Find out entries on index-files that aren't registered on index */ - if ((pkgname = xbps_pkg_name(key)) == NULL) { - xbps_dbg_printf(xhp, "%s: invalid entry found on index-files: %s\n", cbd->repourl, key); - return 0; - } - if ((pkg = xbps_dictionary_get(cbd->idx, pkgname))) { - xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver); - if (strcmp(pkgver, key)) { - pthread_mutex_lock(&cbd->mtx); - xbps_array_add_cstring_nocopy(cbd->result, key); - pthread_mutex_unlock(&cbd->mtx); - } - } - free(pkgname); - - return 0; -} /* * Removes stalled pkg entries in repository's XBPS_REPOIDX file, if any * binary package cannot be read (unavailable, not enough perms, etc). @@ -119,7 +93,7 @@ int index_clean(struct xbps_handle *xhp, const char *repodir) { xbps_array_t allkeys; - xbps_dictionary_t idx = NULL, idxmeta = NULL, idxfiles = NULL; + xbps_dictionary_t idx = NULL, idxmeta = NULL; struct xbps_repo *repo; struct cbdata cbd; int rv = 0; @@ -135,11 +109,9 @@ index_clean(struct xbps_handle *xhp, const char *repodir) _XBPS_RINDEX, strerror(errno)); return rv; } - xbps_repo_open_idxfiles(repo); idx = xbps_dictionary_copy(repo->idx); idxmeta = xbps_dictionary_copy(repo->idxmeta); - idxfiles = xbps_dictionary_copy(repo->idxfiles); - if (idx == NULL || idxfiles == NULL) { + if (idx == NULL) { fprintf(stderr, "%s: incomplete repository data file!\n", _XBPS_RINDEX); rv = EINVAL; goto out; @@ -159,41 +131,20 @@ index_clean(struct xbps_handle *xhp, const char *repodir) char *keyname = NULL, *pkgname = NULL; xbps_array_get_cstring(cbd.result, x, &keyname); - printf("index-files: removed entry %s\n", keyname); printf("index: removed entry %s\n", keyname); pkgname = xbps_pkg_name(keyname); assert(pkgname); - xbps_dictionary_remove(idxfiles, keyname); xbps_dictionary_remove(idx, pkgname); free(pkgname); free(keyname); flush = true; } - /* - * Second pass: find out obsolete entries on index-files. - */ - xbps_object_release(cbd.result); - xbps_object_release(allkeys); - cbd.idx = idx; - cbd.result = xbps_array_create(); - allkeys = xbps_dictionary_all_keys(idxfiles); - (void)xbps_array_foreach_cb_multi(xhp, allkeys, idxfiles, idxfiles_cleaner_cb, &cbd); - for (unsigned int x = 0; x < xbps_array_count(cbd.result); x++) { - char *keyname = NULL; - - xbps_array_get_cstring(cbd.result, x, &keyname); - printf("index-files: removed entry %s\n", keyname); - xbps_dictionary_remove(idxfiles, keyname); - free(keyname); - flush = true; - } - pthread_mutex_destroy(&cbd.mtx); xbps_object_release(cbd.result); xbps_object_release(allkeys); if (flush) { - if (!repodata_flush(xhp, repodir, idx, idxfiles, idxmeta)) { + if (!repodata_flush(xhp, repodir, idx, idxmeta)) { rv = errno; fprintf(stderr, "failed to write repodata: %s\n", strerror(errno)); @@ -202,8 +153,6 @@ index_clean(struct xbps_handle *xhp, const char *repodir) } printf("index: %u packages registered.\n", xbps_dictionary_count(idx)); - printf("index-files: %u packages registered.\n", - xbps_dictionary_count(idxfiles)); out: xbps_repo_close(repo, true); @@ -211,8 +160,6 @@ out: xbps_object_release(idx); if (idxmeta) xbps_object_release(idxmeta); - if (idxfiles) - xbps_object_release(idxfiles); return rv; } diff --git a/bin/xbps-rindex/repoflush.c b/bin/xbps-rindex/repoflush.c index f47961ee..877560a7 100644 --- a/bin/xbps-rindex/repoflush.c +++ b/bin/xbps-rindex/repoflush.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2013 Juan Romero Pardines. + * Copyright (c) 2013-2014 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,8 +39,7 @@ bool repodata_flush(struct xbps_handle *xhp, const char *repodir, - xbps_dictionary_t idx, xbps_dictionary_t idxfiles, - xbps_dictionary_t meta) + xbps_dictionary_t idx, xbps_dictionary_t meta) { struct archive *ar; char *repofile, *tname, *buf; @@ -82,15 +81,6 @@ repodata_flush(struct xbps_handle *xhp, const char *repodir, if (rv != 0) return false; - /* XBPS_REPOIDX_FILES */ - buf = xbps_dictionary_externalize(idxfiles); - assert(buf); - rv = xbps_archive_append_buf(ar, buf, strlen(buf), - XBPS_REPOIDX_FILES, 0644, "root", "root"); - free(buf); - if (rv != 0) - return false; - /* Write data to tempfile and rename */ archive_write_finish(ar); fdatasync(repofd); diff --git a/bin/xbps-rindex/sign.c b/bin/xbps-rindex/sign.c index f2a154e1..6efea04a 100644 --- a/bin/xbps-rindex/sign.c +++ b/bin/xbps-rindex/sign.c @@ -153,7 +153,6 @@ sign_repo(struct xbps_handle *xhp, const char *repodir, rv = EINVAL; goto out; } - xbps_repo_open_idxfiles(repo); /* * If privkey not set, default to ~/.ssh/id_rsa. */ @@ -286,7 +285,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir, xbps_object_release(data); data = NULL; - if (!repodata_flush(xhp, repodir, repo->idx, repo->idxfiles, meta)) { + if (!repodata_flush(xhp, repodir, repo->idx, meta)) { fprintf(stderr, "failed to write repodata: %s\n", strerror(errno)); goto out; } diff --git a/include/xbps.h.in b/include/xbps.h.in index aa10a95e..7ff67e10 100644 --- a/include/xbps.h.in +++ b/include/xbps.h.in @@ -48,7 +48,7 @@ * * This header documents the full API for the XBPS Library. */ -#define XBPS_API_VERSION "20141113" +#define XBPS_API_VERSION "20141113-1" #ifndef XBPS_VERSION #define XBPS_VERSION "UNSET" @@ -112,12 +112,6 @@ */ #define XBPS_REPOIDX "index.plist" -/** - * @def XBPS_REPOIDX_FILES - * Filename for the repository index-files property list. - */ -#define XBPS_REPOIDX_FILES "index-files.plist" - /** * @def XBPS_REPOIDX_META * Filename for the repository index metadata property list. @@ -1188,12 +1182,6 @@ struct xbps_repo { * Proplib dictionary associated with the repository index-meta. */ xbps_dictionary_t idxmeta; - /** - * @var idxfiles - * - * Proplib dictionary associated with the repository index-files. - */ - xbps_dictionary_t idxfiles; /** * @var uri * @@ -1345,15 +1333,6 @@ struct xbps_repo *xbps_repo_open(struct xbps_handle *xhp, const char *url, bool */ void xbps_repo_close(struct xbps_repo *repo, bool lock); - -/** - * Prepares the repository index-files.plist to have access to it. - * The repository must be opened previously with \a xbps_repo_open(). - * - * @param[in] repo The repository object to use. - */ -void xbps_repo_open_idxfiles(struct xbps_repo *repo); - /** * * Returns a heap-allocated string with the repository local path. @@ -1668,6 +1647,23 @@ bool xbps_binpkg_exists(struct xbps_handle *xhp, xbps_dictionary_t pkgd); */ bool xbps_repository_is_remote(const char *uri); +/* + * Returns an allocated string with the full path to the binary package + * matching \a pkgd. + * + * The \a pkgd dictionary must contain the following objects: + * - architecture (string) + * - pkgver (string) + * - repository (string) + * + * @param[in] xhp The pointer to an xbps_handle struct. + * @param[in] pkgd The package dictionary to match. + * + * @return A malloc(3)ed buffer with the full path, NULL otherwise. + * The buffer should be free(3)d when it's no longer needed. + */ +char *xbps_repository_pkg_path(struct xbps_handle *xhp, xbps_dictionary_t pkgd); + /** * Gets the name of a package string. Package strings are composed * by a @/@ pair and separated by the minus diff --git a/include/xbps_api_impl.h b/include/xbps_api_impl.h index f185f547..a8b00c62 100644 --- a/include/xbps_api_impl.h +++ b/include/xbps_api_impl.h @@ -127,12 +127,6 @@ bool HIDDEN xbps_remove_pkg_from_array_by_name(xbps_array_t, const char *); bool HIDDEN xbps_remove_pkg_from_array_by_pattern(xbps_array_t, const char *); bool HIDDEN xbps_remove_pkg_from_array_by_pkgver(xbps_array_t, const char *); -/** - * @private - * From lib/util.c - */ -char HIDDEN *xbps_repository_pkg_path(struct xbps_handle *, xbps_dictionary_t); - /** * @private * From lib/download.c diff --git a/lib/repo.c b/lib/repo.c index 60d2bb82..8d835033 100644 --- a/lib/repo.c +++ b/lib/repo.c @@ -202,13 +202,6 @@ out: return NULL; } -void -xbps_repo_open_idxfiles(struct xbps_repo *repo) -{ - assert(repo); - repo->idxfiles = repo_get_dict(repo); -} - void xbps_repo_close(struct xbps_repo *repo, bool lock) { @@ -225,10 +218,6 @@ xbps_repo_close(struct xbps_repo *repo, bool lock) xbps_object_release(repo->idxmeta); repo->idxmeta = NULL; } - if (repo->idxfiles != NULL) { - xbps_object_release(repo->idxfiles); - repo->idxfiles = NULL; - } if (lock && lockf(repo->fd, F_ULOCK, 0) == -1) xbps_dbg_printf(repo->xhp, "[repo] failed to unlock %s: %s\n", repo->uri, strerror(errno)); diff --git a/lib/util.c b/lib/util.c index 9be13c87..5a753c4a 100644 --- a/lib/util.c +++ b/lib/util.c @@ -280,7 +280,7 @@ xbps_pkgpattern_version(const char *pkg) return strpbrk(pkg, "><*?[]"); } -char HIDDEN * +char * xbps_repository_pkg_path(struct xbps_handle *xhp, xbps_dictionary_t pkg_repod) { const char *pkgver, *arch, *repoloc; diff --git a/tests/xbps/xbps-rindex/clean_test.sh b/tests/xbps/xbps-rindex/clean_test.sh index 4ac96a05..487123d4 100644 --- a/tests/xbps/xbps-rindex/clean_test.sh +++ b/tests/xbps/xbps-rindex/clean_test.sh @@ -22,63 +22,7 @@ noremove_body() { atf_check_equal ${result} 1 } -# 2nd test: make sure that entries are also removed from index-files. -atf_test_case filesclean - -filesclean_head() { - atf_set "descr" "xbps-rindex(8) -c: index-files clean test" -} - -filesclean_body() { - mkdir -p some_repo pkg_A - touch -f pkg_A/file00 - cd some_repo - xbps-create -A noarch -n foo-1.0_1 -s "foo pkg" ../pkg_A - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 - rm *.xbps - cd .. - xbps-rindex -c some_repo - atf_check_equal $? 0 - result=$(xbps-query -r root -C empty.conf --repository=some_repo -o \*) - test -z "${result}" - atf_check_equal $? 0 -} - -# 3nd test: make sure that entries are removed from index-files on updates. -atf_test_case filesclean2 - -filesclean2_head() { - atf_set "descr" "xbps-rindex(8) -c: index-files clean test on updates" -} - -filesclean2_body() { - mkdir -p some_repo pkg_A - touch -f pkg_A/file00 - cd some_repo - xbps-create -A noarch -n foo-1.0_1 -s "foo pkg" ../pkg_A - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 - xbps-create -A noarch -n foo-1.1_1 -s "foo pkg" ../pkg_A - atf_check_equal $? 0 - xbps-rindex -d -a $PWD/*.xbps - atf_check_equal $? 0 - cd .. - xbps-rindex -c some_repo - atf_check_equal $? 0 - result="$(xbps-query -r root -C empty.conf --repository=some_repo -o \*)" - expected="foo-1.1_1: /file00 (some_repo)" - rv=0 - if [ "$result" != "$expected" ]; then - rv=1 - fi - atf_check_equal $rv 0 - -} - -# 4th test: xbps issue #19. +# xbps issue #19. # How to reproduce it: # Generate pkg foo-1.0_1. # Add it to the index of a local repo. @@ -109,7 +53,5 @@ issue19_body() { atf_init_test_cases() { atf_add_test_case noremove - atf_add_test_case filesclean - atf_add_test_case filesclean2 atf_add_test_case issue19 }