diff --git a/NEWS b/NEWS index c1609493..93714834 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,13 @@ xbps-0.12.0 (???): + * xbps-repo(8): the 'genindex' target now creates a plist file to cache + all files provided by binary packages in a repository. This makes + the 'find-files' target marginally faster, because this avoids having + to fetch data from network in remote repositories. The new file + 'index-files.plist' will be downloaded every time the 'sync' target + is issued but only iff the file has been modified. + This fixes Issue 21 "`xbps-repo find-files` is awfully slow". + * It's now possible to fetch files thru xbps_fetch_file() from HTTP servers that don't set the Content-Length header. This fixes issue 19: "xbps-src cannot fetch source with unknown Content-Length" reported diff --git a/bin/xbps-repo/Makefile b/bin/xbps-repo/Makefile index b9c6f22f..bbbe27ab 100644 --- a/bin/xbps-repo/Makefile +++ b/bin/xbps-repo/Makefile @@ -3,6 +3,7 @@ TOPDIR = ../.. BIN = xbps-repo OBJS = main.o index.o show.o find-files.o list.o +OBJS += index-files.o OBJS += ../xbps-bin/fetch_cb.o ../xbps-bin/util.o OBJS += ../xbps-bin/state_cb.o ../xbps-bin/list.o MAN = $(BIN).8 diff --git a/bin/xbps-repo/defs.h b/bin/xbps-repo/defs.h index 26d37f78..95851998 100644 --- a/bin/xbps-repo/defs.h +++ b/bin/xbps-repo/defs.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2009-2011 Juan Romero Pardines. + * Copyright (c) 2009-2012 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,6 +41,9 @@ struct repo_search_data { /* From index.c */ int repo_genindex(const char *); +/* From index-files.c */ +int repo_genindex_files(const char *); + /* From find-files.c */ int repo_find_files_in_packages(int, char **); diff --git a/bin/xbps-repo/find-files.c b/bin/xbps-repo/find-files.c index de46a7d6..ea6e029a 100644 --- a/bin/xbps-repo/find-files.c +++ b/bin/xbps-repo/find-files.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2010 Juan Romero Pardines. + * Copyright (c) 2010-2012 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,100 +37,81 @@ struct ffdata { char **patterns; }; -static int -match_files_by_pattern(prop_dictionary_t pkg_filesd, - prop_dictionary_keysym_t key, - struct ffdata *ffd, - const char *pkgver) +static void +match_files_by_pattern(prop_dictionary_t pkg_filesd, struct ffdata *ffd) { prop_object_iterator_t iter; - prop_array_t array; + prop_array_t array, allkeys; prop_object_t obj; - const char *keyname, *filestr, *typestr; - int i; + prop_dictionary_keysym_t key; + const char *keyname, *filestr, *typestr, *pkgver; + size_t i; + int x; - keyname = prop_dictionary_keysym_cstring_nocopy(key); - array = prop_dictionary_get_keysym(pkg_filesd, key); - if (prop_object_type(array) != PROP_TYPE_ARRAY) - return 0; + allkeys = prop_dictionary_all_keys(pkg_filesd); + for (i = 0; i < prop_array_count(allkeys); i++) { + key = prop_array_get(allkeys, i); + keyname = prop_dictionary_keysym_cstring_nocopy(key); + array = prop_dictionary_get_keysym(pkg_filesd, key); + if (prop_object_type(array) != PROP_TYPE_ARRAY) + break; - if (strcmp(keyname, "files") == 0) - typestr = "regular file"; - else if (strcmp(keyname, "dirs") == 0) - typestr = "directory"; - else if (strcmp(keyname, "links") == 0) - typestr = "link"; - else - typestr = "configuration file"; + if (strcmp(keyname, "files") == 0) + typestr = "regular file"; + else if (strcmp(keyname, "links") == 0) + typestr = "link"; + else + typestr = "configuration file"; - iter = prop_array_iterator(array); - while ((obj = prop_object_iterator_next(iter))) { - prop_dictionary_get_cstring_nocopy(obj, "file", &filestr); - for (i = 1; i < ffd->npatterns; i++) { - if ((strcmp(filestr, ffd->patterns[i]) == 0) || - (strstr(filestr, ffd->patterns[i])) || - (xbps_pkgpattern_match(filestr, - ffd->patterns[i]) == 1)) - printf(" %s: %s (%s)\n", pkgver, filestr, typestr); + iter = prop_array_iterator(array); + while ((obj = prop_object_iterator_next(iter))) { + prop_dictionary_get_cstring_nocopy(obj, "file", &filestr); + for (x = 1; x < ffd->npatterns; x++) { + if ((strcmp(filestr, ffd->patterns[x]) == 0) || + (strstr(filestr, ffd->patterns[x])) || + (xbps_pkgpattern_match(filestr, + ffd->patterns[x]) == 1)) { + prop_dictionary_get_cstring_nocopy( + pkg_filesd, "pkgver", &pkgver); + printf(" %s: %s (%s)\n", + pkgver, filestr, typestr); + } + } } + prop_object_iterator_release(iter); } - prop_object_iterator_release(iter); - return 0; + prop_object_release(allkeys); } static int find_files_in_package(struct repository_pool_index *rpi, void *arg, bool *done) { - prop_dictionary_t pkg_filesd; + prop_dictionary_t idxfilesd; prop_array_t files_keys; - prop_object_t obj; - prop_object_iterator_t iter; struct ffdata *ffd = arg; - const char *pkgname, *pkgver; - char *url; - int rv = 0; - unsigned int i, count; + char *plist; + unsigned int i; (void)done; - iter = xbps_array_iter_from_dict(rpi->rpi_repod, "packages"); - if (iter == NULL) - return -1; - printf("Looking in repository '%s', please wait...\n", rpi->rpi_uri); - while ((obj = prop_object_iterator_next(iter))) { - url = xbps_path_from_repository_uri(obj, rpi->rpi_uri); - if (url == NULL) { - rv = -1; - break; - } - prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname); - prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); - pkg_filesd = xbps_dictionary_metadata_plist_by_url(url, - XBPS_PKGFILES); - free(url); - if (pkg_filesd == NULL) { - xbps_error_printf("xbps-repo: couldn't read '%s' " - "from '%s (%s)': %s\n", XBPS_PKGFILES, pkgname, - rpi->rpi_uri, strerror(errno)); - rv = -1; - break; - } - files_keys = prop_dictionary_all_keys(pkg_filesd); - count = prop_array_count(files_keys); - for (i = 0; i < count; i++) { - rv = match_files_by_pattern(pkg_filesd, - prop_array_get(files_keys, i), ffd, pkgver); - if (rv == -1) - break; - } - prop_object_release(files_keys); - prop_object_release(pkg_filesd); - if (rv == -1) - break; + plist = xbps_pkg_index_files_plist(rpi->rpi_uri); + if (plist == NULL) + return ENOMEM; + + idxfilesd = prop_dictionary_internalize_from_zfile(plist); + if (idxfilesd == NULL) { + free(plist); + return errno; } - prop_object_iterator_release(iter); - return rv; + free(plist); + + files_keys = prop_dictionary_get(idxfilesd, "packages"); + for (i = 0; i < prop_array_count(files_keys); i++) + match_files_by_pattern(prop_array_get(files_keys, i), ffd); + + prop_object_release(idxfilesd); + return 0; } int diff --git a/bin/xbps-repo/index-files.c b/bin/xbps-repo/index-files.c new file mode 100644 index 00000000..835436cf --- /dev/null +++ b/bin/xbps-repo/index-files.c @@ -0,0 +1,196 @@ +/*- + * Copyright (c) 2012 Juan Romero Pardines. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include +#include "defs.h" + +struct index_files_data { + prop_array_t idxfiles; + const char *pkgdir; +}; + +static int +genindex_files_cb(prop_object_t obj, void *arg, bool *done) +{ + prop_dictionary_t pkg_filesd, pkgd; + prop_array_t array; + struct index_files_data *ifd = arg; + const char *binpkg, *pkgver; + char *file; + bool found = false; + + (void)done; + + prop_dictionary_get_cstring_nocopy(obj, "filename", &binpkg); + prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); + + file = xbps_xasprintf("%s/%s", ifd->pkgdir, binpkg); + if (file == NULL) + return ENOMEM; + + /* internalize files.plist from binary package archive */ + pkg_filesd = xbps_dictionary_metadata_plist_by_url(file, XBPS_PKGFILES); + if (pkg_filesd == NULL) { + free(file); + return EINVAL; + } + free(file); + + /* create pkg dictionary */ + pkgd = prop_dictionary_create(); + if (pkgd == NULL) { + prop_object_release(pkg_filesd); + return ENOMEM; + } + /* add conf_files array in pkgd */ + array = prop_dictionary_get(pkg_filesd, "conf_files"); + if (array != NULL && prop_array_count(array)) { + found = true; + if (!prop_dictionary_set(pkgd, "conf_files", array)) { + prop_object_release(pkgd); + prop_object_release(pkg_filesd); + return EINVAL; + } + } + /* add files array in pkgd */ + array = prop_dictionary_get(pkg_filesd, "files"); + if (array != NULL && prop_array_count(array)) { + found = true; + if (!prop_dictionary_set(pkgd, "files", array)) { + prop_object_release(pkgd); + prop_object_release(pkg_filesd); + return EINVAL; + } + } + /* add links array in pkgd */ + array = prop_dictionary_get(pkg_filesd, "links"); + if (array != NULL && prop_array_count(array)) { + found = true; + if (!prop_dictionary_set(pkgd, "links", array)) { + prop_object_release(pkgd); + prop_object_release(pkg_filesd); + return EINVAL; + } + } + prop_object_release(pkg_filesd); + if (!found) { + prop_object_release(pkgd); + return 0; + } + /* pkgver obj in pkgd */ + if (!prop_dictionary_set_cstring(pkgd, "pkgver", pkgver)) { + prop_object_release(pkgd); + return EINVAL; + } + + /* add pkgd into provided array */ + if (!prop_array_add(ifd->idxfiles, pkgd)) { + prop_object_release(pkgd); + return EINVAL; + } + prop_object_release(pkgd); + + return 0; +} + +/* + * Create the index files cache for all packages in repository. + */ +int +repo_genindex_files(const char *pkgdir) +{ + prop_dictionary_t idxdict, idxfilesd; + struct index_files_data *ifd; + char *plist, *files_plist; + int rv; + + plist = xbps_pkg_index_plist(pkgdir); + if (plist == NULL) + return ENOMEM; + + /* internalize repository index plist */ + idxdict = prop_dictionary_internalize_from_zfile(plist); + if (idxdict == NULL) { + free(plist); + return errno; + } + + ifd = malloc(sizeof(*ifd)); + if (ifd == NULL) { + prop_object_release(idxdict); + free(plist); + return ENOMEM; + } + ifd->pkgdir = pkgdir; + ifd->idxfiles = prop_array_create(); + + printf("Creating repository's index files cache...\n"); + + /* iterate over index.plist packages array */ + rv = xbps_callback_array_iter_in_dict(idxdict, + "packages", genindex_files_cb, ifd); + prop_object_release(idxdict); + free(plist); + if (rv != 0) { + prop_object_release(ifd->idxfiles); + free(ifd); + return rv; + } + idxfilesd = prop_dictionary_create(); + /* add array into the index-files dictionary */ + if (!prop_dictionary_set(idxfilesd, "packages", ifd->idxfiles)) { + prop_object_release(ifd->idxfiles); + prop_object_release(idxfilesd); + free(ifd); + return EINVAL; + } + files_plist = xbps_pkg_index_files_plist(pkgdir); + if (files_plist == NULL) { + prop_object_release(ifd->idxfiles); + prop_object_release(idxfilesd); + free(ifd); + return ENOMEM; + } + /* externalize index-files dictionary to the plist file */ + if (!prop_dictionary_externalize_to_zfile(idxfilesd, files_plist)) { + free(files_plist); + prop_object_release(ifd->idxfiles); + prop_object_release(idxfilesd); + free(ifd); + return errno; + } + free(files_plist); + prop_object_release(idxfilesd); + prop_object_release(ifd->idxfiles); + free(ifd); + + return 0; +} diff --git a/bin/xbps-repo/main.c b/bin/xbps-repo/main.c index d6dfe9ba..12831125 100644 --- a/bin/xbps-repo/main.c +++ b/bin/xbps-repo/main.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2008-2011 Juan Romero Pardines. + * Copyright (c) 2008-2012 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -239,6 +239,9 @@ main(int argc, char **argv) usage(xhp); rv = repo_genindex(argv[1]); + if (rv == 0) + rv = repo_genindex_files(argv[1]); + } else if (strcasecmp(argv[0], "sync") == 0) { /* Syncs the pkg index for all registered remote repos */ if (argc != 1) diff --git a/include/xbps_api.h b/include/xbps_api.h index f84a1ad4..52824df6 100644 --- a/include/xbps_api.h +++ b/include/xbps_api.h @@ -56,7 +56,7 @@ */ #define XBPS_PKGINDEX_VERSION "1.3" -#define XBPS_API_VERSION "20120105" +#define XBPS_API_VERSION "20120115" #define XBPS_VERSION "0.12" /** @@ -103,6 +103,12 @@ */ #define XBPS_PKGINDEX "index.plist" +/** + * @def XBPS_PKGINDEX_FILES + * Filename for the repository package index files property list. + */ +#define XBPS_PKGINDEX_FILES "index-files.plist" + /** * @def XBPS_SYSCONF_PATH * Default configuration PATH to find XBPS_CONF_PLIST. @@ -1450,12 +1456,13 @@ prop_dictionary_t * by the \a uri argument (if necessary). * * @param[in] uri URI to a remote repository. + * @param[in] plistf Plist file to sync. * * @return -1 on error (errno is set appropiately), 0 if transfer was * not necessary (local/remote size/mtime matched) or 1 if * downloaded successfully. */ -int xbps_repository_sync_pkg_index(const char *uri); +int xbps_repository_sync_pkg_index(const char *uri, const char *plistf); /*@}*/ @@ -1692,6 +1699,18 @@ char *xbps_path_from_repository_uri(prop_dictionary_t pkgd, const char *repoloc) */ char *xbps_pkg_index_plist(const char *uri); +/** + * Returns the full path to a repository package index files plist file, + * as specified by \a uri. + * + * @param[in] uri Repository URI. + * + * @return A pointer to a malloc(3)ed string, NULL otherwise and + * errno is set appropiately. The pointer should be free(3)d when it's + * no longer needded. + */ +char *xbps_pkg_index_files_plist(const char *uri); + /** * Gets the name of a package string. Package strings are composed * by a @/@ pair and separated by the minus diff --git a/lib/repository_pool.c b/lib/repository_pool.c index 82982b46..10c917a0 100644 --- a/lib/repository_pool.c +++ b/lib/repository_pool.c @@ -196,9 +196,20 @@ xbps_repository_pool_sync(const struct xbps_handle *xhp) continue; } /* - * Fetch repository index file. + * Fetch repository index.plist. */ - rv = xbps_repository_sync_pkg_index(repouri); + rv = xbps_repository_sync_pkg_index(repouri, XBPS_PKGINDEX); + if (rv == -1) { + xbps_dbg_printf("[rpool] `%s' failed to fetch: %s\n", + repouri, fetchLastErrCode == 0 ? + strerror(errno) : xbps_fetch_error_string()); + continue; + } + /* + * Fetch repository index-files.plist. + */ + rv = xbps_repository_sync_pkg_index(repouri, + XBPS_PKGINDEX_FILES); if (rv == -1) { xbps_dbg_printf("[rpool] `%s' failed to fetch: %s\n", repouri, fetchLastErrCode == 0 ? diff --git a/lib/repository_sync_index.c b/lib/repository_sync_index.c index f88337f7..aba26a3c 100644 --- a/lib/repository_sync_index.c +++ b/lib/repository_sync_index.c @@ -86,7 +86,7 @@ xbps_get_remote_repo_string(const char *uri) * size and/or mtime match) and 1 if downloaded successfully. */ int -xbps_repository_sync_pkg_index(const char *uri) +xbps_repository_sync_pkg_index(const char *uri, const char *plistf) { prop_dictionary_t tmpd; struct xbps_handle *xhp; @@ -132,7 +132,7 @@ xbps_repository_sync_pkg_index(const char *uri) /* * Remote repository index.plist full URL. */ - rpidx = xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX); + rpidx = xbps_xasprintf("%s/%s", uri, plistf); if (rpidx == NULL) { rv = -1; goto out; @@ -141,7 +141,7 @@ xbps_repository_sync_pkg_index(const char *uri) * Save temporary file in XBPS_META_PATH, and rename if it * was downloaded successfully. */ - tmp_metafile = xbps_xasprintf("%s/%s", metadir, XBPS_PKGINDEX); + tmp_metafile = xbps_xasprintf("%s/%s", metadir, plistf); if (tmp_metafile == NULL) { rv = -1; goto out; @@ -200,7 +200,7 @@ xbps_repository_sync_pkg_index(const char *uri) } prop_object_release(tmpd); - lrepofile = xbps_xasprintf("%s/%s", lrepodir, XBPS_PKGINDEX); + lrepofile = xbps_xasprintf("%s/%s", lrepodir, plistf); if (lrepofile == NULL) { rv = -1; goto out; diff --git a/lib/util.c b/lib/util.c index 5f1f885a..5f1c6a99 100644 --- a/lib/util.c +++ b/lib/util.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2008-2011 Juan Romero Pardines. + * Copyright (c) 2008-2012 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -201,7 +201,7 @@ xbps_pkgpattern_version(const char *pkg) } static char * -get_pkg_index_remote_plist(const char *uri) +get_pkg_index_remote_plist(const char *uri, const char *plistf) { struct xbps_handle *xhp; char *uri_fixed, *repodir; @@ -215,11 +215,11 @@ get_pkg_index_remote_plist(const char *uri) if (strcmp(xhp->rootdir, "/") == 0) { repodir = xbps_xasprintf("/%s/%s/%s", - XBPS_META_PATH, uri_fixed, XBPS_PKGINDEX); + XBPS_META_PATH, uri_fixed, plistf); } else { repodir = xbps_xasprintf("%s/%s/%s/%s", xhp->rootdir, - XBPS_META_PATH, uri_fixed, XBPS_PKGINDEX); + XBPS_META_PATH, uri_fixed, plistf); } free(uri_fixed); return repodir; @@ -231,11 +231,21 @@ xbps_pkg_index_plist(const char *uri) assert(uri != NULL); if (xbps_check_is_repository_uri_remote(uri)) - return get_pkg_index_remote_plist(uri); + return get_pkg_index_remote_plist(uri, XBPS_PKGINDEX); return xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX); } +char * +xbps_pkg_index_files_plist(const char *uri) +{ + assert(uri != NULL); + if (xbps_check_is_repository_uri_remote(uri)) + return get_pkg_index_remote_plist(uri, XBPS_PKGINDEX_FILES); + + return xbps_xasprintf("%s/%s", uri, XBPS_PKGINDEX_FILES); +} + char * xbps_path_from_repository_uri(prop_dictionary_t pkg_repod, const char *repoloc) {